diff --git a/pyproject.toml b/pyproject.toml index 2669f5ff..d41d7977 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ dependencies = [ "pymupdf >= 1.23.5", "django == 5.0.7", "authlib == 1.2.1", - "llama-cpp-python == 0.2.76", + "llama-cpp-python == 0.2.82", "itsdangerous == 2.1.2", "httpx == 0.25.0", "pgvector == 0.2.4", diff --git a/src/khoj/database/adapters/__init__.py b/src/khoj/database/adapters/__init__.py index 0c0724ee..a0a13e8c 100644 --- a/src/khoj/database/adapters/__init__.py +++ b/src/khoj/database/adapters/__init__.py @@ -559,7 +559,7 @@ class AgentAdapters: if default_conversation_config is None: logger.info("No default conversation config found, skipping default agent creation") return None - default_personality = prompts.personality.format(current_date="placeholder") + default_personality = prompts.personality.format(current_date="placeholder", day_of_week="placeholder") agent = Agent.objects.filter(name=AgentAdapters.DEFAULT_AGENT_NAME).first() diff --git a/src/khoj/processor/conversation/anthropic/anthropic_chat.py b/src/khoj/processor/conversation/anthropic/anthropic_chat.py index 001489ed..d5778885 100644 --- a/src/khoj/processor/conversation/anthropic/anthropic_chat.py +++ b/src/khoj/processor/conversation/anthropic/anthropic_chat.py @@ -36,7 +36,7 @@ def extract_questions_anthropic( # Extract Past User Message and Inferred Questions from Conversation Log chat_history = "".join( [ - f'Q: {chat["intent"]["query"]}\nKhoj: {{"queries": {chat["intent"].get("inferred-queries") or list([chat["intent"]["query"]])}}}\nA: {chat["message"]}\n\n' + f'User: {chat["intent"]["query"]}\nAssistant: {{"queries": {chat["intent"].get("inferred-queries") or list([chat["intent"]["query"]])}}}\nA: {chat["message"]}\n\n' for chat in conversation_log.get("chat", [])[-4:] if chat["by"] == "khoj" and "text-to-image" not in chat["intent"].get("type") ] @@ -135,17 +135,23 @@ def converse_anthropic( Converse with user using Anthropic's Claude """ # Initialize Variables - current_date = datetime.now().strftime("%Y-%m-%d") + current_date = datetime.now() compiled_references = "\n\n".join({f"# {item}" for item in references}) conversation_primer = prompts.query_prompt.format(query=user_query) if agent and agent.personality: system_prompt = prompts.custom_personality.format( - name=agent.name, bio=agent.personality, current_date=current_date + name=agent.name, + bio=agent.personality, + current_date=current_date.strftime("%Y-%m-%d"), + day_of_week=current_date.strftime("%A"), ) else: - system_prompt = prompts.personality.format(current_date=current_date) + system_prompt = prompts.personality.format( + current_date=current_date.strftime("%Y-%m-%d"), + day_of_week=current_date.strftime("%A"), + ) if location_data: location = f"{location_data.city}, {location_data.region}, {location_data.country}" diff --git a/src/khoj/processor/conversation/offline/chat_model.py b/src/khoj/processor/conversation/offline/chat_model.py index edc2d9f0..2da0c186 100644 --- a/src/khoj/processor/conversation/offline/chat_model.py +++ b/src/khoj/processor/conversation/offline/chat_model.py @@ -55,6 +55,7 @@ def extract_questions_offline( chat_history += f"Q: {chat['intent']['query']}\n" chat_history += f"Khoj: {chat['message']}\n\n" + # Get dates relative to today for prompt creation today = datetime.today() yesterday = (today - timedelta(days=1)).strftime("%Y-%m-%d") last_year = today.year - 1 @@ -62,11 +63,13 @@ def extract_questions_offline( query=text, chat_history=chat_history, current_date=today.strftime("%Y-%m-%d"), + day_of_week=today.strftime("%A"), yesterday_date=yesterday, last_year=last_year, this_year=today.year, location=location, ) + messages = generate_chatml_messages_with_context( example_questions, model_name=model, loaded_model=offline_chat_model, max_prompt_size=max_prompt_size ) @@ -74,7 +77,7 @@ def extract_questions_offline( state.chat_lock.acquire() try: response = send_message_to_model_offline( - messages, loaded_model=offline_chat_model, max_prompt_size=max_prompt_size + messages, loaded_model=offline_chat_model, model=model, max_prompt_size=max_prompt_size ) finally: state.chat_lock.release() @@ -96,7 +99,7 @@ def extract_questions_offline( except: logger.warning(f"Llama returned invalid JSON. Falling back to using user message as search query.\n{response}") return all_questions - logger.debug(f"Extracted Questions by Llama: {questions}") + logger.debug(f"Questions extracted by {model}: {questions}") return questions @@ -144,14 +147,20 @@ def converse_offline( offline_chat_model = loaded_model or download_model(model, max_tokens=max_prompt_size) compiled_references_message = "\n\n".join({f"{item['compiled']}" for item in references}) - current_date = datetime.now().strftime("%Y-%m-%d") + current_date = datetime.now() if agent and agent.personality: system_prompt = prompts.custom_system_prompt_offline_chat.format( - name=agent.name, bio=agent.personality, current_date=current_date + name=agent.name, + bio=agent.personality, + current_date=current_date.strftime("%Y-%m-%d"), + day_of_week=current_date.strftime("%A"), ) else: - system_prompt = prompts.system_prompt_offline_chat.format(current_date=current_date) + system_prompt = prompts.system_prompt_offline_chat.format( + current_date=current_date.strftime("%Y-%m-%d"), + day_of_week=current_date.strftime("%A"), + ) conversation_primer = prompts.query_prompt.format(query=user_query) @@ -177,9 +186,9 @@ def converse_offline( if online_results[result].get("webpages"): simplified_online_results[result] = online_results[result]["webpages"] - conversation_primer = f"{prompts.online_search_conversation.format(online_results=str(simplified_online_results))}\n{conversation_primer}" + conversation_primer = f"{prompts.online_search_conversation_offline.format(online_results=str(simplified_online_results))}\n{conversation_primer}" if not is_none_or_empty(compiled_references_message): - conversation_primer = f"{prompts.notes_conversation_offline.format(references=compiled_references_message)}\n{conversation_primer}" + conversation_primer = f"{prompts.notes_conversation_offline.format(references=compiled_references_message)}\n\n{conversation_primer}" # Setup Prompt with Primer or Conversation History messages = generate_chatml_messages_with_context( @@ -192,6 +201,9 @@ def converse_offline( tokenizer_name=tokenizer_name, ) + truncated_messages = "\n".join({f"{message.content[:70]}..." for message in messages}) + logger.debug(f"Conversation Context for {model}: {truncated_messages}") + g = ThreadedGenerator(references, online_results, completion_func=completion_func) t = Thread(target=llm_thread, args=(g, messages, offline_chat_model, max_prompt_size)) t.start() diff --git a/src/khoj/processor/conversation/offline/utils.py b/src/khoj/processor/conversation/offline/utils.py index 05de4b9f..66017b36 100644 --- a/src/khoj/processor/conversation/offline/utils.py +++ b/src/khoj/processor/conversation/offline/utils.py @@ -24,6 +24,8 @@ def download_model(repo_id: str, filename: str = "*Q4_K_M.gguf", max_tokens: int # Add chat format if known if "llama-3" in repo_id.lower(): kwargs["chat_format"] = "llama-3" + elif "gemma-2" in repo_id.lower(): + kwargs["chat_format"] = "gemma" # Check if the model is already downloaded model_path = load_model_from_cache(repo_id, filename) diff --git a/src/khoj/processor/conversation/openai/gpt.py b/src/khoj/processor/conversation/openai/gpt.py index 2f893f22..f1608fba 100644 --- a/src/khoj/processor/conversation/openai/gpt.py +++ b/src/khoj/processor/conversation/openai/gpt.py @@ -125,17 +125,23 @@ def converse( Converse with user using OpenAI's ChatGPT """ # Initialize Variables - current_date = datetime.now().strftime("%Y-%m-%d") + current_date = datetime.now() compiled_references = "\n\n".join({f"# {item['compiled']}" for item in references}) conversation_primer = prompts.query_prompt.format(query=user_query) if agent and agent.personality: system_prompt = prompts.custom_personality.format( - name=agent.name, bio=agent.personality, current_date=current_date + name=agent.name, + bio=agent.personality, + current_date=current_date.strftime("%Y-%m-%d"), + day_of_week=current_date.strftime("%A"), ) else: - system_prompt = prompts.personality.format(current_date=current_date) + system_prompt = prompts.personality.format( + current_date=current_date.strftime("%Y-%m-%d"), + day_of_week=current_date.strftime("%A"), + ) if location_data: location = f"{location_data.city}, {location_data.region}, {location_data.country}" diff --git a/src/khoj/processor/conversation/prompts.py b/src/khoj/processor/conversation/prompts.py index 9b609f75..289bafbc 100644 --- a/src/khoj/processor/conversation/prompts.py +++ b/src/khoj/processor/conversation/prompts.py @@ -19,8 +19,8 @@ You were created by Khoj Inc. with the following capabilities: - Sometimes the user will share personal information that needs to be remembered, like an account ID or a residential address. These can be acknowledged with a simple "Got it" or "Okay". - Provide inline references to quotes from the user's notes or any web pages you refer to in your responses in markdown format. For example, "The farmer had ten sheep. [1](https://example.com)". *ALWAYS CITE YOUR SOURCES AND PROVIDE REFERENCES*. Add them inline to directly support your claim. -Note: More information about you, the company or Khoj apps for download can be found at https://khoj.dev. -Today is {current_date} in UTC. +Note: More information about you, the company or Khoj apps can be found at https://khoj.dev. +Today is {day_of_week}, {current_date} in UTC. """.strip() ) @@ -39,7 +39,7 @@ You were created by Khoj Inc. with the following capabilities: - Ask crisp follow-up questions to get additional context, when the answer cannot be inferred from the provided notes or past conversations. - Sometimes the user will share personal information that needs to be remembered, like an account ID or a residential address. These can be acknowledged with a simple "Got it" or "Okay". -Today is {current_date} in UTC. +Today is {day_of_week}, {current_date} in UTC. Instructions:\n{bio} """.strip() @@ -79,10 +79,12 @@ You are Khoj, a smart, inquisitive and helpful personal assistant. - Use your general knowledge and past conversation with the user as context to inform your responses. - If you do not know the answer, say 'I don't know.' - Think step-by-step and ask questions to get the necessary information to answer the user's question. +- Ask crisp follow-up questions to get additional context, when the answer cannot be inferred from the provided information or past conversations. - Do not print verbatim Notes unless necessary. -Today is {current_date} in UTC. - """.strip() +Note: More information about you, the company or Khoj apps can be found at https://khoj.dev. +Today is {day_of_week}, {current_date} in UTC. +""".strip() ) custom_system_prompt_offline_chat = PromptTemplate.from_template( @@ -91,12 +93,14 @@ You are {name}, a personal agent on Khoj. - Use your general knowledge and past conversation with the user as context to inform your responses. - If you do not know the answer, say 'I don't know.' - Think step-by-step and ask questions to get the necessary information to answer the user's question. +- Ask crisp follow-up questions to get additional context, when the answer cannot be inferred from the provided information or past conversations. - Do not print verbatim Notes unless necessary. -Today is {current_date} in UTC. +Note: More information about you, the company or Khoj apps can be found at https://khoj.dev. +Today is {day_of_week}, {current_date} in UTC. Instructions:\n{bio} - """.strip() +""".strip() ) ## Notes Conversation @@ -106,13 +110,15 @@ notes_conversation = PromptTemplate.from_template( Use my personal notes and our past conversations to inform your response. Ask crisp follow-up questions to get additional context, when a helpful response cannot be provided from the provided notes or past conversations. -Notes: +User's Notes: {references} """.strip() ) notes_conversation_offline = PromptTemplate.from_template( """ +Use my personal notes and our past conversations to inform your response. + User's Notes: {references} """.strip() @@ -174,6 +180,15 @@ Information from the internet: """.strip() ) +online_search_conversation_offline = PromptTemplate.from_template( + """ +Use this up-to-date information from the internet to inform your response. + +Information from the internet: +{online_results} +""".strip() +) + ## Query prompt ## -- query_prompt = PromptTemplate.from_template( @@ -186,15 +201,16 @@ Query: {query}""".strip() ## -- extract_questions_offline = PromptTemplate.from_template( """ -You are Khoj, an extremely smart and helpful search assistant with the ability to retrieve information from the user's notes. Construct search queries to retrieve relevant information to answer the user's question. -- You will be provided past questions(Q) and answers(A) for context. +You are Khoj, an extremely smart and helpful search assistant with the ability to retrieve information from the user's notes. Disregard online search requests. +Construct search queries to retrieve relevant information to answer the user's question. +- You will be provided past questions(Q) and answers(Khoj) for context. - Try to be as specific as possible. Instead of saying "they" or "it" or "he", use proper nouns like name of the person or thing you are referring to. - Add as much context from the previous questions and answers as required into your search queries. - Break messages into multiple search queries when required to retrieve the relevant information. - Add date filters to your search queries from questions and answers when required to retrieve the relevant information. - Share relevant search queries as a JSON list of strings. Do not say anything else. -Current Date: {current_date} +Current Date: {day_of_week}, {current_date} User's Location: {location} Examples: @@ -232,7 +248,8 @@ Q: {query} extract_questions = PromptTemplate.from_template( """ -You are Khoj, an extremely smart and helpful document search assistant with only the ability to retrieve information from the user's notes. Disregard online search requests. Construct search queries to retrieve relevant information to answer the user's question. +You are Khoj, an extremely smart and helpful document search assistant with only the ability to retrieve information from the user's notes. Disregard online search requests. +Construct search queries to retrieve relevant information to answer the user's question. - You will be provided past questions(Q) and answers(A) for context. - Add as much context from the previous questions and answers as required into your search queries. - Break messages into multiple search queries when required to retrieve the relevant information. @@ -282,8 +299,9 @@ Khoj: extract_questions_anthropic_system_prompt = PromptTemplate.from_template( """ -You are Khoj, an extremely smart and helpful document search assistant with only the ability to retrieve information from the user's notes. Disregard online search requests. Construct search queries to retrieve relevant information to answer the user's question. -- You will be provided past questions(Q) and answers(A) for context. +You are Khoj, an extremely smart and helpful document search assistant with only the ability to retrieve information from the user's notes. Disregard online search requests. +Construct search queries to retrieve relevant information to answer the user's question. +- You will be provided past questions(User), extracted queries(Assistant) and answers(A) for context. - Add as much context from the previous questions and answers as required into your search queries. - Break messages into multiple search queries when required to retrieve the relevant information. - Add date filters to your search queries from questions and answers when required to retrieve the relevant information. @@ -297,15 +315,19 @@ Here are some examples of how you can construct search queries to answer the use User: How was my trip to Cambodia? Assistant: {{"queries": ["How was my trip to Cambodia?"]}} +A: The trip was amazing. You went to the Angkor Wat temple and it was beautiful. User: What national parks did I go to last year? Assistant: {{"queries": ["National park I visited in {last_new_year} dt>='{last_new_year_date}' dt<'{current_new_year_date}'"]}} +A: You visited the Grand Canyon and Yellowstone National Park in {last_new_year}. User: How can you help me? Assistant: {{"queries": ["Social relationships", "Physical and mental health", "Education and career", "Personal life goals and habits"]}} +A: I can help you live healthier and happier across work and personal life User: Who all did I meet here yesterday? Assistant: {{"queries": ["Met in {location} on {yesterday_date} dt>='{yesterday_date}' dt<'{current_date}'"]}} +A: Yesterday's note mentions your visit to your local beach with Ram and Shyam. """.strip() ) @@ -319,7 +341,11 @@ Assistant: """.strip() ) -system_prompt_extract_relevant_information = """As a professional analyst, create a comprehensive report of the most relevant information from a web page in response to a user's query. The text provided is directly from within the web page. The report you create should be multiple paragraphs, and it should represent the content of the website. Tell the user exactly what the website says in response to their query, while adhering to these guidelines: +system_prompt_extract_relevant_information = """ +As a professional analyst, create a comprehensive report of the most relevant information from a web page in response to a user's query. +The text provided is directly from within the web page. +The report you create should be multiple paragraphs, and it should represent the content of the website. +Tell the user exactly what the website says in response to their query, while adhering to these guidelines: 1. Answer the user's query as specifically as possible. Include many supporting details from the website. 2. Craft a report that is detailed, thorough, in-depth, and complex, while maintaining clarity. @@ -340,7 +366,11 @@ Collate only relevant information from the website to answer the target query. """.strip() ) -system_prompt_extract_relevant_summary = """As a professional analyst, create a comprehensive report of the most relevant information from the document in response to a user's query. The text provided is directly from within the document. The report you create should be multiple paragraphs, and it should represent the content of the document. Tell the user exactly what the document says in response to their query, while adhering to these guidelines: +system_prompt_extract_relevant_summary = """ +As a professional analyst, create a comprehensive report of the most relevant information from the document in response to a user's query. +The text provided is directly from within the document. +The report you create should be multiple paragraphs, and it should represent the content of the document. +Tell the user exactly what the document says in response to their query, while adhering to these guidelines: 1. Answer the user's query as specifically as possible. Include many supporting details from the document. 2. Craft a report that is detailed, thorough, in-depth, and complex, while maintaining clarity. @@ -363,11 +393,13 @@ Collate only relevant information from the document to answer the target query. pick_relevant_output_mode = PromptTemplate.from_template( """ -You are Khoj, an excellent analyst for selecting the correct way to respond to a user's query. You have access to a limited set of modes for your response. You can only use one of these modes. +You are Khoj, an excellent analyst for selecting the correct way to respond to a user's query. +You have access to a limited set of modes for your response. +You can only use one of these modes. {modes} -Here are some example responses: +Here are some examples: Example: Chat History: @@ -383,7 +415,7 @@ User: I'm having trouble deciding which laptop to get. I want something with at AI: I can help with that. I see online that there is a new model of the Dell XPS 15 that meets your requirements. Q: What are the specs of the new Dell XPS 15? -Khoj: default +Khoj: text Example: Chat History: @@ -391,7 +423,7 @@ User: Where did I go on my last vacation? AI: You went to Jordan and visited Petra, the Dead Sea, and Wadi Rum. Q: Remind me who did I go with on that trip? -Khoj: default +Khoj: text Example: Chat History: @@ -399,7 +431,7 @@ User: How's the weather outside? Current Location: Bali, Indonesia AI: It's currently 28°C and partly cloudy in Bali. Q: Share a painting using the weather for Bali every morning. -Khoj: reminder +Khoj: automation Now it's your turn to pick the mode you would like to use to answer the user's question. Provide your response as a string. @@ -422,7 +454,7 @@ Which of the data sources listed below you would use to answer the user's questi {tools} -Here are some example responses: +Here are some examples: Example: Chat History: @@ -533,10 +565,10 @@ You are Khoj, an advanced google search assistant. You are tasked with construct - Break messages into multiple search queries when required to retrieve the relevant information. - Use site: google search operators when appropriate - You have access to the the whole internet to retrieve information. -- Official, up-to-date information about you, Khoj, is available at site:khoj.dev +- Official, up-to-date information about you, Khoj, is available at site:khoj.dev, github or pypi. What Google searches, if any, will you need to perform to answer the user's question? -Provide search queries as a list of strings in a JSON object. +Provide search queries as a list of strings in a JSON object. Do not wrap the json in a codeblock. Current Date: {current_date} User's Location: {location} @@ -589,7 +621,6 @@ Q: How many oranges would fit in NASA's Saturn V rocket? Khoj: {{"queries": ["volume of an orange", "volume of saturn v rocket"]}} Now it's your turn to construct Google search queries to answer the user's question. Provide them as a list of strings in a JSON object. Do not say anything else. -Now it's your turn to construct a search query for Google to answer the user's question. History: {chat_history} diff --git a/src/khoj/processor/conversation/utils.py b/src/khoj/processor/conversation/utils.py index 5d68d17d..797066d7 100644 --- a/src/khoj/processor/conversation/utils.py +++ b/src/khoj/processor/conversation/utils.py @@ -186,7 +186,7 @@ def generate_chatml_messages_with_context( def truncate_messages( messages: list[ChatMessage], - max_prompt_size, + max_prompt_size: int, model_name: str, loaded_model: Optional[Llama] = None, tokenizer_name=None, @@ -232,7 +232,8 @@ def truncate_messages( tokens = sum([len(encoder.encode(message.content)) for message in messages if type(message.content) == str]) # Drop older messages until under max supported prompt size by model - while (tokens + system_message_tokens) > max_prompt_size and len(messages) > 1: + # Reserves 4 tokens to demarcate each message (e.g <|im_start|>user, <|im_end|>, <|endoftext|> etc.) + while (tokens + system_message_tokens + 4 * len(messages)) > max_prompt_size and len(messages) > 1: messages.pop() tokens = sum([len(encoder.encode(message.content)) for message in messages if type(message.content) == str]) @@ -254,6 +255,8 @@ def truncate_messages( f"Truncate current message to fit within max prompt size of {max_prompt_size} supported by {model_name} model:\n {truncated_message}" ) + if system_message: + system_message.role = "user" if "gemma-2" in model_name else "system" return messages + [system_message] if system_message else messages diff --git a/src/khoj/routers/api.py b/src/khoj/routers/api.py index cbe19891..c9d76ae7 100644 --- a/src/khoj/routers/api.py +++ b/src/khoj/routers/api.py @@ -317,7 +317,7 @@ async def extract_references_and_questions( filters_in_query += " ".join([f'file:"{filter}"' for filter in conversation.file_filters]) using_offline_chat = False - print(f"Filters in query: {filters_in_query}") + logger.debug(f"Filters in query: {filters_in_query}") # Infer search queries from user message with timer("Extracting search queries took", logger): @@ -335,6 +335,7 @@ async def extract_references_and_questions( inferred_queries = extract_questions_offline( defiltered_query, + model=chat_model, loaded_model=loaded_model, conversation_log=meta_log, should_extract_questions=True, diff --git a/src/khoj/routers/helpers.py b/src/khoj/routers/helpers.py index e0f91df7..25d21f29 100644 --- a/src/khoj/routers/helpers.py +++ b/src/khoj/routers/helpers.py @@ -307,6 +307,7 @@ async def aget_relevant_output_modes(query: str, conversation_history: dict, is_ # Check whether the tool exists as a valid ConversationCommand return ConversationCommand(response) + logger.error(f"Invalid output mode selected: {response}. Defaulting to text.") return ConversationCommand.Text except Exception: logger.error(f"Invalid response for determining relevant mode: {response}") @@ -519,9 +520,6 @@ async def send_message_to_model_wrapper( chat_model_option or await ConversationAdapters.aget_default_conversation_config() ) - if conversation_config is None: - raise HTTPException(status_code=500, detail="Contact the server administrator to set a default chat model.") - chat_model = conversation_config.chat_model max_tokens = conversation_config.max_prompt_size tokenizer = conversation_config.tokenizer diff --git a/tests/test_offline_chat_actors.py b/tests/test_offline_chat_actors.py index 67c014ed..ed17cac9 100644 --- a/tests/test_offline_chat_actors.py +++ b/tests/test_offline_chat_actors.py @@ -286,7 +286,7 @@ def test_answer_from_chat_history_and_currently_retrieved_content(loaded_model): # Act response_gen = converse_offline( references=[ - "Testatron was born on 1st April 1984 in Testville." + {"compiled": "Testatron was born on 1st April 1984 in Testville."} ], # Assume context retrieved from notes for the user_query user_query="Where was I born?", conversation_log=populate_chat_history(message_list), @@ -341,14 +341,22 @@ def test_answer_requires_current_date_awareness(loaded_model): "Chat actor should be able to answer questions relative to current date using provided notes" # Arrange context = [ - f"""{datetime.now().strftime("%Y-%m-%d")} "Naco Taco" "Tacos for Dinner" -Expenses:Food:Dining 10.00 USD""", - f"""{datetime.now().strftime("%Y-%m-%d")} "Sagar Ratna" "Dosa for Lunch" -Expenses:Food:Dining 10.00 USD""", - f"""2020-04-01 "SuperMercado" "Bananas" -Expenses:Food:Groceries 10.00 USD""", - f"""2020-01-01 "Naco Taco" "Burittos for Dinner" -Expenses:Food:Dining 10.00 USD""", + { + "compiled": f"""{datetime.now().strftime("%Y-%m-%d")} "Naco Taco" "Tacos for Dinner" +Expenses:Food:Dining 10.00 USD""" + }, + { + "compiled": f"""{datetime.now().strftime("%Y-%m-%d")} "Sagar Ratna" "Dosa for Lunch" +Expenses:Food:Dining 10.00 USD""" + }, + { + "compiled": f"""2020-04-01 "SuperMercado" "Bananas" +Expenses:Food:Groceries 10.00 USD""" + }, + { + "compiled": f"""2020-01-01 "Naco Taco" "Burittos for Dinner" +Expenses:Food:Dining 10.00 USD""" + }, ] # Act @@ -373,14 +381,22 @@ def test_answer_requires_date_aware_aggregation_across_provided_notes(loaded_mod "Chat actor should be able to answer questions that require date aware aggregation across multiple notes" # Arrange context = [ - f"""# {datetime.now().strftime("%Y-%m-%d")} "Naco Taco" "Tacos for Dinner" -Expenses:Food:Dining 10.00 USD""", - f"""{datetime.now().strftime("%Y-%m-%d")} "Sagar Ratna" "Dosa for Lunch" -Expenses:Food:Dining 10.00 USD""", - f"""2020-04-01 "SuperMercado" "Bananas" -Expenses:Food:Groceries 10.00 USD""", - f"""2020-01-01 "Naco Taco" "Burittos for Dinner" -Expenses:Food:Dining 10.00 USD""", + { + "compiled": f"""# {datetime.now().strftime("%Y-%m-%d")} "Naco Taco" "Tacos for Dinner" +Expenses:Food:Dining 10.00 USD""" + }, + { + "compiled": f"""{datetime.now().strftime("%Y-%m-%d")} "Sagar Ratna" "Dosa for Lunch" +Expenses:Food:Dining 10.00 USD""" + }, + { + "compiled": f"""2020-04-01 "SuperMercado" "Bananas" +Expenses:Food:Groceries 10.00 USD""" + }, + { + "compiled": f"""2020-01-01 "Naco Taco" "Burittos for Dinner" +Expenses:Food:Dining 10.00 USD""" + }, ] # Act @@ -430,12 +446,18 @@ def test_ask_for_clarification_if_not_enough_context_in_question(loaded_model): "Chat actor should ask for clarification if question cannot be answered unambiguously with the provided context" # Arrange context = [ - f"""# Ramya -My sister, Ramya, is married to Kali Devi. They have 2 kids, Ravi and Rani.""", - f"""# Fang -My sister, Fang Liu is married to Xi Li. They have 1 kid, Xiao Li.""", - f"""# Aiyla -My sister, Aiyla is married to Tolga. They have 3 kids, Yildiz, Ali and Ahmet.""", + { + "compiled": f"""# Ramya +My sister, Ramya, is married to Kali Devi. They have 2 kids, Ravi and Rani.""" + }, + { + "compiled": f"""# Fang +My sister, Fang Liu is married to Xi Li. They have 1 kid, Xiao Li.""" + }, + { + "compiled": f"""# Aiyla +My sister, Aiyla is married to Tolga. They have 3 kids, Yildiz, Ali and Ahmet.""" + }, ] # Act @@ -459,9 +481,9 @@ def test_agent_prompt_should_be_used(loaded_model, offline_agent): "Chat actor should ask be tuned to think like an accountant based on the agent definition" # Arrange context = [ - f"""I went to the store and bought some bananas for 2.20""", - f"""I went to the store and bought some apples for 1.30""", - f"""I went to the store and bought some oranges for 6.00""", + {"compiled": f"""I went to the store and bought some bananas for 2.20"""}, + {"compiled": f"""I went to the store and bought some apples for 1.30"""}, + {"compiled": f"""I went to the store and bought some oranges for 6.00"""}, ] # Act @@ -499,7 +521,7 @@ def test_chat_does_not_exceed_prompt_size(loaded_model): "Ensure chat context and response together do not exceed max prompt size for the model" # Arrange prompt_size_exceeded_error = "ERROR: The prompt size exceeds the context window size and cannot be processed" - context = [" ".join([f"{number}" for number in range(2043)])] + context = [{"compiled": " ".join([f"{number}" for number in range(2043)])}] # Act response_gen = converse_offline( @@ -530,7 +552,7 @@ def test_filter_questions(): # ---------------------------------------------------------------------------------------------------- @pytest.mark.anyio @pytest.mark.django_db(transaction=True) -async def test_use_default_response_mode(client_offline_chat): +async def test_use_text_response_mode(client_offline_chat): # Arrange user_query = "What's the latest in the Israel/Palestine conflict?" @@ -538,7 +560,7 @@ async def test_use_default_response_mode(client_offline_chat): mode = await aget_relevant_output_modes(user_query, {}) # Assert - assert mode.value == "default" + assert mode.value == "text" # ---------------------------------------------------------------------------------------------------- diff --git a/tests/test_offline_chat_director.py b/tests/test_offline_chat_director.py index a72dae56..43e254e6 100644 --- a/tests/test_offline_chat_director.py +++ b/tests/test_offline_chat_director.py @@ -45,7 +45,6 @@ def create_conversation(message_list, user, agent=None): # Tests # ---------------------------------------------------------------------------------------------------- -@pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering this question yet") @pytest.mark.chatquality @pytest.mark.django_db(transaction=True) def test_offline_chat_with_no_chat_history_or_retrieved_content(client_offline_chat): @@ -179,10 +178,6 @@ def test_answer_from_chat_history_and_previously_retrieved_content(client_offlin # ---------------------------------------------------------------------------------------------------- -@pytest.mark.xfail( - AssertionError, - reason="Chat director not capable of answering this question yet because it requires extract_questions", -) @pytest.mark.chatquality @pytest.mark.django_db(transaction=True) def test_answer_from_chat_history_and_currently_retrieved_content(client_offline_chat, default_user2):