diff --git a/pyproject.toml b/pyproject.toml index 8df96e4f..06338201 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ dependencies = [ "fastapi == 0.77.1", "jinja2 == 3.1.2", "openai >= 0.27.0", + "tiktoken >= 0.3.0", "pillow == 9.3.0", "pydantic == 1.9.1", "pyqt6 == 6.3.1", diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el index 64ebfbe4..239ac9bf 100644 --- a/src/interface/emacs/khoj.el +++ b/src/interface/emacs/khoj.el @@ -109,6 +109,8 @@ (defvar khoj--content-type "org" "The type of content to perform search on.") +(declare-function org-element-property "org-mode" (PROPERTY ELEMENT)) +(declare-function org-element-type "org-mode" (ELEMENT)) (declare-function beancount-mode "beancount" ()) (declare-function markdown-mode "markdown-mode" ()) (declare-function org-music-mode "org-music" ()) @@ -134,6 +136,7 @@ NO-PAGING FILTER)) "C-x M | music\n")))) (defvar khoj--rerank nil "Track when re-rank of results triggered.") +(defvar khoj--reference-count 0 "Track number of references currently in chat bufffer.") (defun khoj--search-markdown () "Set content-type to `markdown'." (interactive) (setq khoj--content-type "markdown")) (defun khoj--search-org () "Set content-type to `org-mode'." (interactive) (setq khoj--content-type "org")) (defun khoj--search-ledger () "Set content-type to `ledger'." (interactive) (setq khoj--content-type "ledger")) @@ -335,15 +338,22 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE." (defun khoj--chat () "Chat with Khoj." + (interactive) + (when (not (get-buffer khoj--chat-buffer-name)) + (khoj--load-chat-history khoj--chat-buffer-name)) + (switch-to-buffer khoj--chat-buffer-name) (let ((query (read-string "Query: "))) - (khoj--query-chat-api-and-render-messages query khoj--chat-buffer-name) - (switch-to-buffer khoj--chat-buffer-name))) + (when (not (string-empty-p query)) + (khoj--query-chat-api-and-render-messages query khoj--chat-buffer-name)))) (defun khoj--load-chat-history (buffer-name) + "Load Khoj Chat conversation history into BUFFER-NAME." (let ((json-response (cdr (assoc 'response (khoj--query-chat-api ""))))) (with-current-buffer (get-buffer-create buffer-name) (erase-buffer) (insert "#+STARTUP: showall hidestars\n") + ;; allow sub, superscript text within {} for footnotes + (insert "#+OPTIONS: ^:{}\n") (thread-last json-response ;; generate chat messages from Khoj Chat API response @@ -352,23 +362,53 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE." (mapc #'insert)) (progn (org-mode) (visual-line-mode) + (khoj--add-hover-text-to-footnote-refs (point-min)) + (use-local-map (copy-keymap org-mode-map)) + (local-set-key (kbd "m") #'khoj--chat) + (local-set-key (kbd "C-x m") #'khoj--chat) (read-only-mode t))))) +(defun khoj--add-hover-text-to-footnote-refs (start-pos) + "Show footnote defs on mouse hover on footnote refs from START-POS." + (org-with-wide-buffer + (goto-char start-pos) + (while (re-search-forward org-footnote-re nil t) + (backward-char) + (let* ((context (org-element-context)) + (label (org-element-property :label context)) + (footnote-def (nth 3 (org-footnote-get-definition label))) + (footnote-width (if (< (length footnote-def) 70) nil 70)) + (begin-pos (org-element-property :begin context)) + (end-pos (org-element-property :end context)) + (overlay (make-overlay begin-pos end-pos))) + (when (memq (org-element-type context) + '(footnote-reference)) + (--> + footnote-def + ;; truncate footnote definition if required + (substring it 0 footnote-width) + ;; append continuation suffix if truncated + (concat it (if footnote-width "..." "")) + ;; show definition on hover on footnote reference + (overlay-put overlay 'help-echo it))))))) + (defun khoj--query-chat-api-and-render-messages (query buffer-name) "Send QUERY to Khoj Chat. Render the chat messages from exchange in BUFFER-NAME." ;; render json response into formatted chat messages - (if (not (get-buffer buffer-name)) - (khoj--load-chat-history buffer-name) - (with-current-buffer (get-buffer buffer-name) - (let ((inhibit-read-only t) - (json-response (khoj--query-chat-api query))) - (goto-char (point-max)) - (insert - (khoj--render-chat-message query "you") - (khoj--render-chat-response json-response))) - (progn (org-mode) - (visual-line-mode)) - (read-only-mode t)))) + (with-current-buffer (get-buffer buffer-name) + (let ((inhibit-read-only t) + (new-content-start-pos (point-max)) + (query-time (format-time-string "%F %T")) + (json-response (khoj--query-chat-api query))) + (goto-char new-content-start-pos) + (insert + (khoj--render-chat-message query "you" query-time) + (khoj--render-chat-response json-response)) + (khoj--add-hover-text-to-footnote-refs new-content-start-pos)) + (progn + (org-set-startup-visibility) + (visual-line-mode) + (re-search-backward "^\*+ 🦅" nil t)))) (defun khoj--query-chat-api (query) "Send QUERY to Khoj Chat API." @@ -386,39 +426,48 @@ MESSAGE is the text of the chat message. SENDER is the message sender. RECEIVE-DATE is the message receive date." (let ((first-message-line (car (split-string message "\n" t))) - (rest-message-lines (string-join (cdr (split-string message "\n" t)) "\n ")) + (rest-message-lines (string-join (cdr (split-string message "\n" t)) "\n")) (heading-level (if (equal sender "you") "**" "***")) - (emojified-by (if (equal sender "you") "🤔 *You*" "🦅 *Khoj*")) + (emojified-sender (if (equal sender "you") "🤔 *You*" "🦅 *Khoj*")) + (suffix-newlines (if (equal sender "khoj") "\n\n" "")) (received (or receive-date (format-time-string "%F %T")))) - (format "%s %s: %s\n :PROPERTIES:\n :RECEIVED: [%s]\n :END:\n %s\n" + (format "%s %s: %s\n :PROPERTIES:\n :RECEIVED: [%s]\n :END:\n%s\n%s" heading-level - emojified-by + emojified-sender first-message-line received - rest-message-lines))) + rest-message-lines + suffix-newlines))) -(defun khoj--generate-reference (index reference) - "Create `org-mode' links with REFERENCE as link and INDEX as link description." - (with-temp-buffer - (org-insert-link - nil - (format "%s" (replace-regexp-in-string "\n" " " reference)) - (format "%s" index)) - (format "[%s]" (buffer-substring-no-properties (point-min) (point-max))))) +(defun khoj--generate-reference (reference) + "Create `org-mode' footnotes with REFERENCE." + (setq khoj--reference-count (1+ khoj--reference-count)) + (cons + (propertize (format "^{ [fn:%x]}" khoj--reference-count) 'help-echo reference) + (thread-last + reference + (replace-regexp-in-string "\n\n" "\n") + (format "\n[fn:%x] %s" khoj--reference-count)))) (defun khoj--render-chat-response (json-response) "Render chat message using JSON-RESPONSE from Khoj Chat API." (let* ((message (cdr (or (assoc 'response json-response) (assoc 'message json-response)))) (sender (cdr (assoc 'by json-response))) (receive-date (cdr (assoc 'created json-response))) - (context (or (cdr (assoc 'context json-response)) "")) - (reference-texts (split-string context "\n\n# " t)) - (reference-links (-map-indexed #'khoj--generate-reference reference-texts))) + (references (or (cdr (assoc 'context json-response)) '())) + (footnotes (mapcar #'khoj--generate-reference references)) + (footnote-links (mapcar #'car footnotes)) + (footnote-defs (mapcar #'cdr footnotes))) (thread-first - ;; extract khoj message from API response and make it bold - (format "%s" message) - ;; append references to khoj message - (concat " " (string-join reference-links " ")) + ;; concatenate khoj message and references from API + (concat + message + ;; append reference links to khoj message + (string-join footnote-links "") + ;; append reference sub-section to khoj message and fold it + (if footnote-defs "\n**** References\n:PROPERTIES:\n:VISIBILITY: folded\n:END:" "") + ;; append reference definitions to references subsection + (string-join footnote-defs " ")) ;; Render chat message using data obtained from API (khoj--render-chat-message sender receive-date)))) diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html index 00056a44..ae038fde 100644 --- a/src/khoj/interface/web/chat.html +++ b/src/khoj/interface/web/chat.html @@ -39,7 +39,6 @@ let references = ''; if (context) { references = context - .split("\n\n# ") .map((reference, index) => generateReference(reference, index)) .join(","); } diff --git a/src/khoj/processor/conversation/gpt.py b/src/khoj/processor/conversation/gpt.py index bcc37db8..cfb236e5 100644 --- a/src/khoj/processor/conversation/gpt.py +++ b/src/khoj/processor/conversation/gpt.py @@ -223,13 +223,14 @@ A:{ "search-type": "notes" }""" return json.loads(story.strip(empty_escape_sequences)) -def converse(text, user_query, conversation_log={}, api_key=None, temperature=0.2): +def converse(references, user_query, conversation_log={}, api_key=None, temperature=0.2): """ Converse with user using OpenAI's ChatGPT """ # Initialize Variables model = "gpt-3.5-turbo" openai.api_key = api_key or os.getenv("OPENAI_API_KEY") + compiled_references = "\n\n".join({f"# {item}" for item in references}) personality_primer = "You are Khoj, a friendly, smart and helpful personal assistant." conversation_primer = f""" @@ -237,7 +238,7 @@ Using the notes and our past conversations as context, answer the following ques Current Date: {datetime.now().strftime("%Y-%m-%d")} Notes: -{text} +{compiled_references} Question: {user_query}""" @@ -246,6 +247,7 @@ Question: {user_query}""" conversation_primer, personality_primer, conversation_log, + model, ) # Get Response from GPT diff --git a/src/khoj/processor/conversation/utils.py b/src/khoj/processor/conversation/utils.py index 42853edf..b8728b6a 100644 --- a/src/khoj/processor/conversation/utils.py +++ b/src/khoj/processor/conversation/utils.py @@ -1,22 +1,44 @@ # Standard Packages from datetime import datetime +# External Packages +import tiktoken + # Internal Packages from khoj.utils.helpers import merge_dicts -def generate_chatml_messages_with_context(user_message, system_message, conversation_log={}): +max_prompt_size = {"gpt-3.5-turbo": 4096, "gpt-4": 8192} + + +def generate_chatml_messages_with_context( + user_message, system_message, conversation_log={}, model_name="gpt-3.5-turbo", lookback_turns=2 +): """Generate messages for ChatGPT with context from previous conversation""" # Extract Chat History for Context chat_logs = [f'{chat["message"]}\n\nNotes:\n{chat.get("context","")}' for chat in conversation_log.get("chat", [])] - last_backnforth = reciprocal_conversation_to_chatml(chat_logs[-2:]) - rest_backnforth = reciprocal_conversation_to_chatml(chat_logs[-4:-2]) + rest_backnforths = [] + # Extract in reverse chronological order + for user_msg, assistant_msg in zip(chat_logs[-2::-2], chat_logs[::-2]): + if len(rest_backnforths) >= 2 * lookback_turns: + break + rest_backnforths += reciprocal_conversation_to_chatml([user_msg, assistant_msg])[::-1] # Format user and system messages to chatml format system_chatml_message = [message_to_chatml(system_message, "system")] user_chatml_message = [message_to_chatml(user_message, "user")] - return rest_backnforth + system_chatml_message + last_backnforth + user_chatml_message + messages = user_chatml_message + rest_backnforths[:2] + system_chatml_message + rest_backnforths[2:] + + # Truncate oldest messages from conversation history until under max supported prompt size by model + encoder = tiktoken.encoding_for_model(model_name) + tokens = sum([len(encoder.encode(value)) for message in messages for value in message.values()]) + while tokens > max_prompt_size[model_name]: + messages.pop() + tokens = sum([len(encoder.encode(value)) for message in messages for value in message.values()]) + + # Return message in chronological order + return messages[::-1] def reciprocal_conversation_to_chatml(message_pair): @@ -38,20 +60,20 @@ def message_to_prompt( return f"{conversation_history}{restart_sequence} {user_message}{start_sequence}{gpt_message}" -def message_to_log(user_message, gpt_message, khoj_message_metadata={}, conversation_log=[]): +def message_to_log(user_message, gpt_message, user_message_metadata={}, khoj_message_metadata={}, conversation_log=[]): """Create json logs from messages, metadata for conversation log""" default_khoj_message_metadata = { "intent": {"type": "remember", "memory-type": "notes", "query": user_message}, "trigger-emotion": "calm", } - current_dt = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + khoj_response_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") # Create json log from Human's message - human_log = {"message": user_message, "by": "you", "created": current_dt} + human_log = merge_dicts({"message": user_message, "by": "you"}, user_message_metadata) # Create json log from GPT's response khoj_log = merge_dicts(khoj_message_metadata, default_khoj_message_metadata) - khoj_log = merge_dicts({"message": gpt_message, "by": "khoj", "created": current_dt}, khoj_log) + khoj_log = merge_dicts({"message": gpt_message, "by": "khoj", "created": khoj_response_time}, khoj_log) conversation_log.extend([human_log, khoj_log]) return conversation_log diff --git a/src/khoj/routers/api.py b/src/khoj/routers/api.py index dd39f0d3..312c1ffc 100644 --- a/src/khoj/routers/api.py +++ b/src/khoj/routers/api.py @@ -2,6 +2,7 @@ import math import yaml import logging +from datetime import datetime from typing import List, Optional, Union # External Packages @@ -192,6 +193,7 @@ def chat(q: Optional[str] = None): # Initialize Variables api_key = state.processor_config.conversation.openai_api_key model = state.processor_config.conversation.model + user_message_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") # Load Conversation History chat_session = state.processor_config.conversation.chat_session @@ -205,16 +207,19 @@ def chat(q: Optional[str] = None): return {"status": "ok", "response": []} # Infer search queries from user message - inferred_queries = extract_questions(q, model=model, api_key=api_key, conversation_log=meta_log) + with timer("Extracting search queries took", logger): + inferred_queries = extract_questions(q, model=model, api_key=api_key, conversation_log=meta_log) # Collate search results as context for GPT - result_list = [] - for query in inferred_queries: - result_list.extend(search(query, n=5, r=True, score_threshold=-5.0, dedupe=False)) - collated_result = "\n\n".join({f"# {item.additional['compiled']}" for item in result_list}) + with timer("Searching knowledge base took", logger): + result_list = [] + for query in inferred_queries: + result_list.extend(search(query, n=5, r=True, score_threshold=-5.0, dedupe=False)) + compiled_references = [item.additional["compiled"] for item in result_list] try: - gpt_response = converse(collated_result, q, meta_log, api_key=api_key) + with timer("Generating chat response took", logger): + gpt_response = converse(compiled_references, q, meta_log, api_key=api_key) status = "ok" except Exception as e: gpt_response = str(e) @@ -225,8 +230,9 @@ def chat(q: Optional[str] = None): state.processor_config.conversation.meta_log["chat"] = message_to_log( q, gpt_response, - khoj_message_metadata={"context": collated_result, "intent": {"inferred-queries": inferred_queries}}, + user_message_metadata={"created": user_message_time}, + khoj_message_metadata={"context": compiled_references, "intent": {"inferred-queries": inferred_queries}}, conversation_log=meta_log.get("chat", []), ) - return {"status": status, "response": gpt_response, "context": collated_result} + return {"status": status, "response": gpt_response, "context": compiled_references} diff --git a/tests/test_chat_actors.py b/tests/test_chat_actors.py index 009ff54e..cd1f9d4d 100644 --- a/tests/test_chat_actors.py +++ b/tests/test_chat_actors.py @@ -111,7 +111,7 @@ def test_extract_multiple_implicit_questions_from_message(): def test_generate_search_query_using_question_from_chat_history(): # Arrange message_list = [ - ("What is the name of Mr. Vaders daughter?", "Princess Leia", ""), + ("What is the name of Mr. Vader's daughter?", "Princess Leia", []), ] # Act @@ -127,7 +127,7 @@ def test_generate_search_query_using_question_from_chat_history(): def test_generate_search_query_using_answer_from_chat_history(): # Arrange message_list = [ - ("What is the name of Mr. Vaders daughter?", "Princess Leia", ""), + ("What is the name of Mr. Vader's daughter?", "Princess Leia", []), ] # Act @@ -143,7 +143,7 @@ def test_generate_search_query_using_answer_from_chat_history(): def test_generate_search_query_using_question_and_answer_from_chat_history(): # Arrange message_list = [ - ("Does Luke Skywalker have any Siblings?", "Yes, Princess Leia", ""), + ("Does Luke Skywalker have any Siblings?", "Yes, Princess Leia", []), ] # Act @@ -159,7 +159,7 @@ def test_generate_search_query_using_question_and_answer_from_chat_history(): def test_generate_search_query_with_date_and_context_from_chat_history(): # Arrange message_list = [ - ("When did I visit Masai Mara?", "You visited Masai Mara in April 2000", ""), + ("When did I visit Masai Mara?", "You visited Masai Mara in April 2000", []), ] # Act @@ -184,7 +184,7 @@ def test_generate_search_query_with_date_and_context_from_chat_history(): def test_chat_with_no_chat_history_or_retrieved_content(): # Act response = converse( - text="", # Assume no context retrieved from notes for the user_query + references=[], # Assume no context retrieved from notes for the user_query user_query="Hello, my name is Testatron. Who are you?", api_key=api_key, ) @@ -202,13 +202,13 @@ def test_chat_with_no_chat_history_or_retrieved_content(): def test_answer_from_chat_history_and_no_content(): # Arrange message_list = [ - ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", ""), - ("When was I born?", "You were born on 1st April 1984.", ""), + ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", []), + ("When was I born?", "You were born on 1st April 1984.", []), ] # Act response = converse( - text="", # Assume no context retrieved from notes for the user_query + references=[], # Assume no context retrieved from notes for the user_query user_query="What is my name?", conversation_log=populate_chat_history(message_list), api_key=api_key, @@ -228,13 +228,17 @@ def test_answer_from_chat_history_and_previously_retrieved_content(): "Chat actor needs to use context in previous notes and chat history to answer question" # Arrange message_list = [ - ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", ""), - ("When was I born?", "You were born on 1st April 1984.", "Testatron was born on 1st April 1984 in Testville."), + ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", []), + ( + "When was I born?", + "You were born on 1st April 1984.", + ["Testatron was born on 1st April 1984 in Testville."], + ), ] # Act response = converse( - text="", # Assume no context retrieved from notes for the user_query + references=[], # Assume no context retrieved from notes for the user_query user_query="Where was I born?", conversation_log=populate_chat_history(message_list), api_key=api_key, @@ -252,13 +256,15 @@ def test_answer_from_chat_history_and_currently_retrieved_content(): "Chat actor needs to use context across currently retrieved notes and chat history to answer question" # Arrange message_list = [ - ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", ""), - ("When was I born?", "You were born on 1st April 1984.", ""), + ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", []), + ("When was I born?", "You were born on 1st April 1984.", []), ] # Act response = converse( - text="Testatron was born on 1st April 1984 in Testville.", # Assume context retrieved from notes for the user_query + references=[ + "Testatron was born on 1st April 1984 in Testville." + ], # Assume context retrieved from notes for the user_query user_query="Where was I born?", conversation_log=populate_chat_history(message_list), api_key=api_key, @@ -275,13 +281,13 @@ def test_no_answer_in_chat_history_or_retrieved_content(): "Chat actor should say don't know as not enough contexts in chat history or retrieved to answer question" # Arrange message_list = [ - ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", ""), - ("When was I born?", "You were born on 1st April 1984.", ""), + ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", []), + ("When was I born?", "You were born on 1st April 1984.", []), ] # Act response = converse( - text="", # Assume no context retrieved from notes for the user_query + references=[], # Assume no context retrieved from notes for the user_query user_query="Where was I born?", conversation_log=populate_chat_history(message_list), api_key=api_key, @@ -300,23 +306,20 @@ def test_no_answer_in_chat_history_or_retrieved_content(): def test_answer_requires_current_date_awareness(): "Chat actor should be able to answer questions relative to current date using provided notes" # Arrange - context = f""" - # {datetime.now().strftime("%Y-%m-%d")} "Naco Taco" "Tacos for Dinner" - Expenses:Food:Dining 10.00 USD - - # {datetime.now().strftime("%Y-%m-%d")} "Sagar Ratna" "Dosa for Lunch" - Expenses:Food:Dining 10.00 USD - - # 2020-04-01 "SuperMercado" "Bananas" - Expenses:Food:Groceries 10.00 USD - - # 2020-01-01 "Naco Taco" "Burittos for Dinner" - Expenses:Food:Dining 10.00 USD - """ + context = [ + f"""{datetime.now().strftime("%Y-%m-%d")} "Naco Taco" "Tacos for Dinner" +Expenses:Food:Dining 10.00 USD""", + f"""{datetime.now().strftime("%Y-%m-%d")} "Sagar Ratna" "Dosa for Lunch" +Expenses:Food:Dining 10.00 USD""", + f"""2020-04-01 "SuperMercado" "Bananas" +Expenses:Food:Groceries 10.00 USD""", + f"""2020-01-01 "Naco Taco" "Burittos for Dinner" +Expenses:Food:Dining 10.00 USD""", + ] # Act response = converse( - text=context, # Assume context retrieved from notes for the user_query + references=context, # Assume context retrieved from notes for the user_query user_query="What did I have for Dinner today?", api_key=api_key, ) @@ -334,23 +337,20 @@ def test_answer_requires_current_date_awareness(): def test_answer_requires_date_aware_aggregation_across_provided_notes(): "Chat actor should be able to answer questions that require date aware aggregation across multiple notes" # Arrange - context = f""" - # {datetime.now().strftime("%Y-%m-%d")} "Naco Taco" "Tacos for Dinner" - Expenses:Food:Dining 10.00 USD - - # {datetime.now().strftime("%Y-%m-%d")} "Sagar Ratna" "Dosa for Lunch" - Expenses:Food:Dining 10.00 USD - - # 2020-04-01 "SuperMercado" "Bananas" - Expenses:Food:Groceries 10.00 USD - - # 2020-01-01 "Naco Taco" "Burittos for Dinner" - Expenses:Food:Dining 10.00 USD - """ + context = [ + f"""# {datetime.now().strftime("%Y-%m-%d")} "Naco Taco" "Tacos for Dinner" +Expenses:Food:Dining 10.00 USD""", + f"""{datetime.now().strftime("%Y-%m-%d")} "Sagar Ratna" "Dosa for Lunch" +Expenses:Food:Dining 10.00 USD""", + f"""2020-04-01 "SuperMercado" "Bananas" +Expenses:Food:Groceries 10.00 USD""", + f"""2020-01-01 "Naco Taco" "Burittos for Dinner" +Expenses:Food:Dining 10.00 USD""", + ] # Act response = converse( - text=context, # Assume context retrieved from notes for the user_query + references=context, # Assume context retrieved from notes for the user_query user_query="How much did I spend on dining this year?", api_key=api_key, ) @@ -366,14 +366,14 @@ def test_answer_general_question_not_in_chat_history_or_retrieved_content(): "Chat actor should be able to answer general questions not requiring looking at chat history or notes" # Arrange message_list = [ - ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", ""), - ("When was I born?", "You were born on 1st April 1984.", ""), - ("Where was I born?", "You were born Testville.", ""), + ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", []), + ("When was I born?", "You were born on 1st April 1984.", []), + ("Where was I born?", "You were born Testville.", []), ] # Act response = converse( - text="", # Assume no context retrieved from notes for the user_query + references=[], # Assume no context retrieved from notes for the user_query user_query="Write a haiku about unit testing in 3 lines", conversation_log=populate_chat_history(message_list), api_key=api_key, @@ -393,20 +393,18 @@ def test_answer_general_question_not_in_chat_history_or_retrieved_content(): def test_ask_for_clarification_if_not_enough_context_in_question(): "Chat actor should ask for clarification if question cannot be answered unambiguously with the provided context" # Arrange - context = f""" - # Ramya - My sister, Ramya, is married to Kali Devi. They have 2 kids, Ravi and Rani. - - # Fang - My sister, Fang Liu is married to Xi Li. They have 1 kid, Xiao Li. - - # Aiyla - My sister, Aiyla is married to Tolga. They have 3 kids, Yildiz, Ali and Ahmet. - """ + context = [ + f"""# Ramya +My sister, Ramya, is married to Kali Devi. They have 2 kids, Ravi and Rani.""", + f"""# Fang +My sister, Fang Liu is married to Xi Li. They have 1 kid, Xiao Li.""", + f"""# Aiyla +My sister, Aiyla is married to Tolga. They have 3 kids, Yildiz, Ali and Ahmet.""", + ] # Act response = converse( - text=context, # Assume context retrieved from notes for the user_query + references=context, # Assume context retrieved from notes for the user_query user_query="How many kids does my older sister have?", api_key=api_key, ) diff --git a/tests/test_chat_director.py b/tests/test_chat_director.py index 99261a72..5feaa3f3 100644 --- a/tests/test_chat_director.py +++ b/tests/test_chat_director.py @@ -47,7 +47,7 @@ def test_chat_with_no_chat_history_or_retrieved_content(chat_client): expected_responses = ["Khoj", "khoj"] assert response.status_code == 200 assert any([expected_response in response_message for expected_response in expected_responses]), ( - "Expected assistants name, [K|k]hoj, in response but got" + response_message + "Expected assistants name, [K|k]hoj, in response but got: " + response_message ) @@ -56,8 +56,8 @@ def test_chat_with_no_chat_history_or_retrieved_content(chat_client): def test_answer_from_chat_history(chat_client): # Arrange message_list = [ - ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", ""), - ("When was I born?", "You were born on 1st April 1984.", ""), + ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", []), + ("When was I born?", "You were born on 1st April 1984.", []), ] populate_chat_history(message_list) @@ -69,7 +69,7 @@ def test_answer_from_chat_history(chat_client): expected_responses = ["Testatron", "testatron"] assert response.status_code == 200 assert any([expected_response in response_message for expected_response in expected_responses]), ( - "Expected [T|t]estatron in response but got" + response_message + "Expected [T|t]estatron in response but got: " + response_message ) @@ -78,8 +78,12 @@ def test_answer_from_chat_history(chat_client): def test_answer_from_currently_retrieved_content(chat_client): # Arrange message_list = [ - ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", ""), - ("When was I born?", "You were born on 1st April 1984.", "Testatron was born on 1st April 1984 in Testville."), + ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", []), + ( + "When was I born?", + "You were born on 1st April 1984.", + ["Testatron was born on 1st April 1984 in Testville."], + ), ] populate_chat_history(message_list) @@ -97,8 +101,12 @@ def test_answer_from_currently_retrieved_content(chat_client): def test_answer_from_chat_history_and_previously_retrieved_content(chat_client): # Arrange message_list = [ - ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", ""), - ("When was I born?", "You were born on 1st April 1984.", "Testatron was born on 1st April 1984 in Testville."), + ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", []), + ( + "When was I born?", + "You were born on 1st April 1984.", + ["Testatron was born on 1st April 1984 in Testville."], + ), ] populate_chat_history(message_list) @@ -119,8 +127,8 @@ def test_answer_from_chat_history_and_previously_retrieved_content(chat_client): def test_answer_from_chat_history_and_currently_retrieved_content(chat_client): # Arrange message_list = [ - ("Hello, my name is Xi Li. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", ""), - ("When was I born?", "You were born on 1st April 1984.", ""), + ("Hello, my name is Xi Li. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", []), + ("When was I born?", "You were born on 1st April 1984.", []), ] populate_chat_history(message_list) @@ -143,8 +151,8 @@ def test_no_answer_in_chat_history_or_retrieved_content(chat_client): "Chat director should say don't know as not enough contexts in chat history or retrieved to answer question" # Arrange message_list = [ - ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", ""), - ("When was I born?", "You were born on 1st April 1984.", ""), + ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", []), + ("When was I born?", "You were born on 1st April 1984.", []), ] populate_chat_history(message_list) @@ -197,9 +205,9 @@ def test_answer_requires_date_aware_aggregation_across_provided_notes(chat_clien def test_answer_general_question_not_in_chat_history_or_retrieved_content(chat_client): # Arrange message_list = [ - ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", ""), - ("When was I born?", "You were born on 1st April 1984.", ""), - ("Where was I born?", "You were born Testville.", ""), + ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", []), + ("When was I born?", "You were born on 1st April 1984.", []), + ("Where was I born?", "You were born Testville.", []), ] populate_chat_history(message_list) @@ -243,9 +251,9 @@ def test_ask_for_clarification_if_not_enough_context_in_question(chat_client): def test_answer_in_chat_history_beyond_lookback_window(chat_client): # Arrange message_list = [ - ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", ""), - ("When was I born?", "You were born on 1st April 1984.", ""), - ("Where was I born?", "You were born Testville.", ""), + ("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", []), + ("When was I born?", "You were born on 1st April 1984.", []), + ("Where was I born?", "You were born Testville.", []), ] populate_chat_history(message_list)