From 0d949140f475314755b331d3fa270c51af90de38 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Mon, 5 Feb 2024 20:02:54 +0530 Subject: [PATCH 1/3] Fix actor, director tests using freeze time by ignoring transformers package transformers package was causing freeze time to fail during setup --- tests/test_gpt4all_chat_actors.py | 8 ++++---- tests/test_gpt4all_chat_director.py | 9 ++++++--- tests/test_openai_chat_actors.py | 6 +++--- tests/test_openai_chat_director.py | 5 ++--- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/tests/test_gpt4all_chat_actors.py b/tests/test_gpt4all_chat_actors.py index 30209b81..3b1d7249 100644 --- a/tests/test_gpt4all_chat_actors.py +++ b/tests/test_gpt4all_chat_actors.py @@ -41,7 +41,7 @@ freezegun.configure(extend_ignore_list=["transformers"]) # ---------------------------------------------------------------------------------------------------- @pytest.mark.xfail(reason="Search actor isn't very date aware nor capable of formatting") @pytest.mark.chatquality -@freeze_time("1984-04-02") +@freeze_time("1984-04-02", ignore=["transformers"]) def test_extract_question_with_date_filter_from_relative_day(loaded_model): # Act response = extract_questions_offline("Where did I go for dinner yesterday?", loaded_model=loaded_model) @@ -61,7 +61,7 @@ def test_extract_question_with_date_filter_from_relative_day(loaded_model): # ---------------------------------------------------------------------------------------------------- @pytest.mark.xfail(reason="Search actor still isn't very date aware nor capable of formatting") @pytest.mark.chatquality -@freeze_time("1984-04-02") +@freeze_time("1984-04-02", ignore=["transformers"]) def test_extract_question_with_date_filter_from_relative_month(loaded_model): # Act response = extract_questions_offline("Which countries did I visit last month?", loaded_model=loaded_model) @@ -83,7 +83,7 @@ def test_extract_question_with_date_filter_from_relative_month(loaded_model): # ---------------------------------------------------------------------------------------------------- @pytest.mark.xfail(reason="Chat actor still isn't very date aware nor capable of formatting") @pytest.mark.chatquality -@freeze_time("1984-04-02") +@freeze_time("1984-04-02", ignore=["transformers"]) def test_extract_question_with_date_filter_from_relative_year(): # Act response = extract_questions_offline("Which countries have I visited this year?") @@ -102,7 +102,7 @@ def test_extract_question_with_date_filter_from_relative_year(): # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@freeze_time("1984-04-02") +@freeze_time("1984-04-02", ignore=["transformers"]) def test_extract_question_includes_root_question(loaded_model): # Act response = extract_questions_offline("Which countries have I visited this year?", loaded_model=loaded_model) diff --git a/tests/test_gpt4all_chat_director.py b/tests/test_gpt4all_chat_director.py index dd9b2d21..0c6b3b95 100644 --- a/tests/test_gpt4all_chat_director.py +++ b/tests/test_gpt4all_chat_director.py @@ -301,11 +301,14 @@ def test_answer_not_known_using_notes_command(client_offline_chat, default_user2 @pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering time aware questions yet") @pytest.mark.chatquality @pytest.mark.django_db(transaction=True) -@freeze_time("2023-04-01") +@freeze_time("2023-04-01", ignore=["transformers"]) def test_answer_requires_current_date_awareness(client_offline_chat): "Chat actor should be able to answer questions relative to current date using provided notes" + # Arrange + query = urllib.parse.quote("Where did I have lunch today?") + # Act - response = client_offline_chat.get(f'/api/chat?q="Where did I have lunch today?"&stream=true') + response = client_offline_chat.get(f"/api/chat?q={query}&stream=true") response_message = response.content.decode("utf-8") # Assert @@ -320,7 +323,7 @@ def test_answer_requires_current_date_awareness(client_offline_chat): @pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering this question yet") @pytest.mark.chatquality @pytest.mark.django_db(transaction=True) -@freeze_time("2023-04-01") +@freeze_time("2023-04-01", ignore=["transformers"]) def test_answer_requires_date_aware_aggregation_across_provided_notes(client_offline_chat): "Chat director should be able to answer questions that require date aware aggregation across multiple notes" # Act diff --git a/tests/test_openai_chat_actors.py b/tests/test_openai_chat_actors.py index 46f63d0e..201e3fef 100644 --- a/tests/test_openai_chat_actors.py +++ b/tests/test_openai_chat_actors.py @@ -22,7 +22,7 @@ freezegun.configure(extend_ignore_list=["transformers"]) # Test # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@freeze_time("1984-04-02") +@freeze_time("1984-04-02", ignore=["transformers"]) def test_extract_question_with_date_filter_from_relative_day(): # Act response = extract_questions("Where did I go for dinner yesterday?") @@ -41,7 +41,7 @@ def test_extract_question_with_date_filter_from_relative_day(): # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@freeze_time("1984-04-02") +@freeze_time("1984-04-02", ignore=["transformers"]) def test_extract_question_with_date_filter_from_relative_month(): # Act response = extract_questions("Which countries did I visit last month?") @@ -56,7 +56,7 @@ def test_extract_question_with_date_filter_from_relative_month(): # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@freeze_time("1984-04-02") +@freeze_time("1984-04-02", ignore=["transformers"]) def test_extract_question_with_date_filter_from_relative_year(): # Act response = extract_questions("Which countries have I visited this year?") diff --git a/tests/test_openai_chat_director.py b/tests/test_openai_chat_director.py index fc92d4b7..ce5897e6 100644 --- a/tests/test_openai_chat_director.py +++ b/tests/test_openai_chat_director.py @@ -277,7 +277,7 @@ def test_answer_not_known_using_notes_command(chat_client_no_background, default @pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering time aware questions yet") @pytest.mark.django_db(transaction=True) @pytest.mark.chatquality -@freeze_time("2023-04-01") +@freeze_time("2023-04-01", ignore=["transformers"]) def test_answer_requires_current_date_awareness(chat_client): "Chat actor should be able to answer questions relative to current date using provided notes" # Act @@ -295,11 +295,10 @@ def test_answer_requires_current_date_awareness(chat_client): # ---------------------------------------------------------------------------------------------------- @pytest.mark.django_db(transaction=True) @pytest.mark.chatquality -@freeze_time("2023-04-01") +@freeze_time("2023-04-01", ignore=["transformers"]) def test_answer_requires_date_aware_aggregation_across_provided_notes(chat_client): "Chat director should be able to answer questions that require date aware aggregation across multiple notes" # Act - response = chat_client.get(f'/api/chat?q="How much did I spend on dining this year?"&stream=true') response_message = response.content.decode("utf-8") From 035165b534b2cafceb6e924983d8cbff69e0e781 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Mon, 5 Feb 2024 19:58:42 +0530 Subject: [PATCH 2/3] Make offline chat model current date aware. Improve system prompts - Can now expect date awareness chat quality test to pass - Prevent offline chat model from printing verbatim user Notes and special tokens - Make it ask follow-up questions if it needs more context --- .../processor/conversation/offline/chat_model.py | 5 +++-- src/khoj/processor/conversation/prompts.py | 16 ++++++++++++---- tests/test_gpt4all_chat_director.py | 2 -- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/khoj/processor/conversation/offline/chat_model.py b/src/khoj/processor/conversation/offline/chat_model.py index 8361650c..c366b1fd 100644 --- a/src/khoj/processor/conversation/offline/chat_model.py +++ b/src/khoj/processor/conversation/offline/chat_model.py @@ -171,9 +171,10 @@ def converse_offline( ) # Setup Prompt with Primer or Conversation History + current_date = datetime.now().strftime("%Y-%m-%d") messages = generate_chatml_messages_with_context( conversation_primer, - prompts.system_prompt_message_gpt4all, + prompts.system_prompt_message_gpt4all.format(current_date=current_date), conversation_log, model_name=model, max_prompt_size=max_prompt_size, @@ -198,7 +199,7 @@ def llm_thread(g, messages: List[ChatMessage], model: Any): for message in conversation_history ] - stop_words = [""] + stop_words = ["", "INST]", "Notes:"] chat_history = "".join(formatted_messages) templated_system_message = prompts.system_prompt_gpt4all.format(message=system_message.content) templated_user_message = prompts.user_message_gpt4all.format(message=user_message.content) diff --git a/src/khoj/processor/conversation/prompts.py b/src/khoj/processor/conversation/prompts.py index 67d8725f..bb3bd0a9 100644 --- a/src/khoj/processor/conversation/prompts.py +++ b/src/khoj/processor/conversation/prompts.py @@ -5,7 +5,7 @@ from langchain.prompts import PromptTemplate personality = PromptTemplate.from_template( """ You are Khoj, a smart, inquisitive and helpful personal assistant. -Use your general knowledge and the past conversation with the user as context to inform your responses. +Use your general knowledge and past conversation with the user as context to inform your responses. You were created by Khoj Inc. with the following capabilities: - You *CAN REMEMBER ALL NOTES and PERSONAL INFORMATION FOREVER* that the user ever shares with you. @@ -48,9 +48,17 @@ no_entries_found = PromptTemplate.from_template( ## Conversation Prompts for GPT4All Models ## -- -system_prompt_message_gpt4all = f"""You are Khoj, a smart, inquisitive and helpful personal assistant. -Using your general knowledge and our past conversations as context, answer the following question. -If you do not know the answer, say 'I don't know.'""" +system_prompt_message_gpt4all = PromptTemplate.from_template( + """ +You are Khoj, a smart, inquisitive and helpful personal assistant. +- Use your general knowledge and past conversation with the user as context to inform your responses. +- If you do not know the answer, say 'I don't know.' +- Ask crisp follow-up questions to get additional context, when the answer cannot be inferred from the provided notes or past conversations. +- Do not print verbatim Notes unless necessary. + +Today is {current_date} in UTC. + """.strip() +) system_prompt_message_extract_questions_gpt4all = f"""You are Khoj, a kind and intelligent personal assistant. When the user asks you a question, you ask follow-up questions to clarify the necessary information you need in order to answer from the user's perspective. - Write the question as if you can search for the answer on the user's personal notes. diff --git a/tests/test_gpt4all_chat_director.py b/tests/test_gpt4all_chat_director.py index 0c6b3b95..7476c4e6 100644 --- a/tests/test_gpt4all_chat_director.py +++ b/tests/test_gpt4all_chat_director.py @@ -298,7 +298,6 @@ def test_answer_not_known_using_notes_command(client_offline_chat, default_user2 # ---------------------------------------------------------------------------------------------------- -@pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering time aware questions yet") @pytest.mark.chatquality @pytest.mark.django_db(transaction=True) @freeze_time("2023-04-01", ignore=["transformers"]) @@ -336,7 +335,6 @@ def test_answer_requires_date_aware_aggregation_across_provided_notes(client_off # ---------------------------------------------------------------------------------------------------- -@pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering this question yet") @pytest.mark.chatquality @pytest.mark.django_db(transaction=True) def test_answer_general_question_not_in_chat_history_or_retrieved_content(client_offline_chat, default_user2): From dd4cf66be16f53265c4d1f1f6f716c3e80a33777 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Mon, 5 Feb 2024 20:18:14 +0530 Subject: [PATCH 3/3] Improve offline chat system prompt to think step by step --- src/khoj/processor/conversation/prompts.py | 2 +- tests/test_gpt4all_chat_director.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/khoj/processor/conversation/prompts.py b/src/khoj/processor/conversation/prompts.py index bb3bd0a9..9c009326 100644 --- a/src/khoj/processor/conversation/prompts.py +++ b/src/khoj/processor/conversation/prompts.py @@ -53,7 +53,7 @@ system_prompt_message_gpt4all = PromptTemplate.from_template( You are Khoj, a smart, inquisitive and helpful personal assistant. - Use your general knowledge and past conversation with the user as context to inform your responses. - If you do not know the answer, say 'I don't know.' -- Ask crisp follow-up questions to get additional context, when the answer cannot be inferred from the provided notes or past conversations. +- Think step-by-step and ask questions to get the necessary information to answer the user's question. - Do not print verbatim Notes unless necessary. Today is {current_date} in UTC. diff --git a/tests/test_gpt4all_chat_director.py b/tests/test_gpt4all_chat_director.py index 7476c4e6..48520cad 100644 --- a/tests/test_gpt4all_chat_director.py +++ b/tests/test_gpt4all_chat_director.py @@ -426,7 +426,6 @@ def test_answer_chat_history_very_long(client_offline_chat, default_user2): # ---------------------------------------------------------------------------------------------------- -@pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering this question yet") @pytest.mark.chatquality @pytest.mark.django_db(transaction=True) def test_answer_requires_multiple_independent_searches(client_offline_chat):