From c3b624e3510948f7490b61dd40edc735701a8011 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Fri, 3 Mar 2023 12:52:03 -0600 Subject: [PATCH 01/20] Introduce improved answer API and prompt. Use by default in chat web interface - Improve GPT prompt - Make GPT answer users query based on provided notes instead of summarizing the provided notes - Make GPT be truthful using prompt and reduced temperature - Use Official OpenAI Q&A prompt from cookbook as starting reference - Replace summarize API with the improved answer API endpoint - Default to answer type in chat web interface. The chat type is not fit for default consumption yet --- src/khoj/interface/web/chat.html | 6 ++---- src/khoj/processor/conversation/gpt.py | 28 ++++++++++++++++++++++++++ src/khoj/routers/api_beta.py | 9 +++++---- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html index 07253b06..cba6eb2d 100644 --- a/src/khoj/interface/web/chat.html +++ b/src/khoj/interface/web/chat.html @@ -50,9 +50,7 @@ document.getElementById("chat-input").value = ""; // Generate backend API URL to execute query - url = type_ === "chat" - ? `/api/beta/chat?q=${encodeURIComponent(query)}` - : `/api/beta/summarize?q=${encodeURIComponent(query)}`; + url = `/api/beta/${type_}?q=${encodeURIComponent(query)}`; // Call specified Khoj API fetch(url) @@ -112,8 +110,8 @@ diff --git a/src/khoj/processor/conversation/gpt.py b/src/khoj/processor/conversation/gpt.py index c942e08d..a10b3c85 100644 --- a/src/khoj/processor/conversation/gpt.py +++ b/src/khoj/processor/conversation/gpt.py @@ -10,6 +10,34 @@ import openai from khoj.utils.constants import empty_escape_sequences +def answer(text, user_query, model, api_key=None, temperature=0.3, max_tokens=200): + """ + Answer user query using provided text as reference with OpenAI's GPT + """ + # Initialize Variables + openai.api_key = api_key or os.getenv("OPENAI_API_KEY") + + # Setup Prompt based on Summary Type + prompt = f""" +You are a friendly, helpful personal assistant. +Using the users notes below, answer their following question. If the answer is not contained within the notes, say "I don't know." + +Notes: +{text} + +Question: {user_query} + +Answer (in second person):""" + # Get Response from GPT + response = openai.Completion.create( + prompt=prompt, model=model, temperature=temperature, max_tokens=max_tokens, stop='"""' + ) + + # Extract, Clean Message from GPT's Response + story = response["choices"][0]["text"] + return str(story).replace("\n\n", "") + + def summarize(text, summary_type, model, user_query=None, api_key=None, temperature=0.5, max_tokens=200): """ Summarize user input using OpenAI's GPT diff --git a/src/khoj/routers/api_beta.py b/src/khoj/routers/api_beta.py index 2f523917..e03d09e9 100644 --- a/src/khoj/routers/api_beta.py +++ b/src/khoj/routers/api_beta.py @@ -10,6 +10,7 @@ from fastapi import APIRouter # Internal Packages from khoj.routers.api import search from khoj.processor.conversation.gpt import ( + answer, converse, extract_search_type, message_to_log, @@ -48,8 +49,8 @@ def search_beta(q: str, n: Optional[int] = 1): return {"status": "ok", "result": search_results, "type": search_type} -@api_beta.get("/summarize") -def summarize_beta(q: str): +@api_beta.get("/answer") +def answer_beta(q: str): # Initialize Variables model = state.processor_config.conversation.model api_key = state.processor_config.conversation.openai_api_key @@ -61,9 +62,9 @@ def summarize_beta(q: str): # Converse with OpenAI GPT result_list = search(q, n=1, r=True) collated_result = "\n".join([item.entry for item in result_list]) - logger.debug(f"Semantically Similar Notes:\n{collated_result}") + logger.debug(f"Reference Notes:\n{collated_result}") try: - gpt_response = summarize(collated_result, summary_type="notes", user_query=q, model=model, api_key=api_key) + gpt_response = answer(collated_result, user_query=q, model=model, api_key=api_key) status = "ok" except Exception as e: gpt_response = str(e) From 9d42b5d60d70c51eecf6485565a29b41890ee450 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Fri, 3 Mar 2023 13:31:41 -0600 Subject: [PATCH 02/20] Use multiple compiled search results for more relevant context to GPT Increase temperature to allow GPT to collect answer across multiple notes --- src/khoj/processor/conversation/gpt.py | 2 +- src/khoj/routers/api_beta.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/khoj/processor/conversation/gpt.py b/src/khoj/processor/conversation/gpt.py index a10b3c85..f3f8392e 100644 --- a/src/khoj/processor/conversation/gpt.py +++ b/src/khoj/processor/conversation/gpt.py @@ -10,7 +10,7 @@ import openai from khoj.utils.constants import empty_escape_sequences -def answer(text, user_query, model, api_key=None, temperature=0.3, max_tokens=200): +def answer(text, user_query, model, api_key=None, temperature=0.5, max_tokens=500): """ Answer user query using provided text as reference with OpenAI's GPT """ diff --git a/src/khoj/routers/api_beta.py b/src/khoj/routers/api_beta.py index e03d09e9..4cf6e1e7 100644 --- a/src/khoj/routers/api_beta.py +++ b/src/khoj/routers/api_beta.py @@ -59,10 +59,12 @@ def answer_beta(q: str): chat_session = state.processor_config.conversation.chat_session meta_log = state.processor_config.conversation.meta_log - # Converse with OpenAI GPT - result_list = search(q, n=1, r=True) - collated_result = "\n".join([item.entry for item in result_list]) - logger.debug(f"Reference Notes:\n{collated_result}") + # Collate context for GPT + result_list = search(q, n=2, r=True) + collated_result = "\n\n".join([f"# {item.additional['compiled']}" for item in result_list]) + logger.debug(f"Reference Context:\n{collated_result}") + + # Make GPT respond to user query using provided context try: gpt_response = answer(collated_result, user_query=q, model=model, api_key=api_key) status = "ok" From ad1f1cf6201ea7279be892535572c23e4ea81aa4 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Sat, 4 Mar 2023 11:01:49 -0600 Subject: [PATCH 03/20] Improve and simplify Khoj Chat using ChatGPT - Set context by either including last 2 chat messages from active session or past 2 conversation summaries from conversation logs - Set personality in system message - Place personality system message before last completed back & forth This may stop ChatGPT forgetting its personality as conversation progresses given: - The conditioning based on system role messages is light - If system message is too far back in conversation history, the model may forget its personality conditioning - If system message at end of conversation, the model can think its the start of a new conversation - Inserting the system message before last completed back & forth should prevent ChatGPT from assuming its the start of a new conversation while not losing personality conditioning from the system message - Simplfy the Khoj Chat API to for now just answer from users notes instead of trying to infer other potential interaction types. - This is the default expected behavior from the feature anyway - Use the compiled text of the top 2 search results for context - Benefits of using ChatGPT - Better model - 1/10th the price - No hand rolled prompt required to make GPT provide more chatty, assistant type responses --- pyproject.toml | 2 +- src/khoj/processor/conversation/gpt.py | 129 ++++++++++--------------- src/khoj/routers/api_beta.py | 36 +++---- 3 files changed, 63 insertions(+), 104 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index af41cc5b..8385111d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ dependencies = [ "defusedxml == 0.7.1", "fastapi == 0.77.1", "jinja2 == 3.1.2", - "openai == 0.20.0", + "openai >= 0.27.0", "pillow == 9.3.0", "pydantic == 1.9.1", "pyqt6 == 6.3.1", diff --git a/src/khoj/processor/conversation/gpt.py b/src/khoj/processor/conversation/gpt.py index f3f8392e..8d0457de 100644 --- a/src/khoj/processor/conversation/gpt.py +++ b/src/khoj/processor/conversation/gpt.py @@ -114,104 +114,75 @@ A:{ "search-type": "notes" }""" return json.loads(story.strip(empty_escape_sequences)) -def understand(text, model, api_key=None, temperature=0.5, max_tokens=100, verbose=0): +def converse(text, user_query, active_session_length=0, conversation_log=None, api_key=None, temperature=0): """ - Understand user input using OpenAI's GPT + Converse with user using OpenAI's ChatGPT """ # Initialize Variables - openai.api_key = api_key or os.getenv("OPENAI_API_KEY") - understand_primer = """ -Objective: Extract intent and trigger emotion information as JSON from each chat message - -Potential intent types and valid argument values are listed below: -- intent - - remember(memory-type, query); - - memory-type=["companion","notes","ledger","image","music"] - - search(search-type, query); - - search-type=["google"] - - generate(activity, query); - - activity=["paint","write","chat"] -- trigger-emotion(emotion) - - emotion=["happy","confidence","fear","surprise","sadness","disgust","anger","shy","curiosity","calm"] - -Some examples are given below for reference: -Q: How are you doing? -A: { "intent": {"type": "generate", "activity": "chat", "query": "How are you doing?"}, "trigger-emotion": "happy" } -Q: Do you remember what I told you about my brother Antoine when we were at the beach? -A: { "intent": {"type": "remember", "memory-type": "companion", "query": "Brother Antoine when we were at the beach"}, "trigger-emotion": "curiosity" } -Q: what was that fantasy story you told me last time? -A: { "intent": {"type": "remember", "memory-type": "companion", "query": "fantasy story told last time"}, "trigger-emotion": "curiosity" } -Q: Let's make some drawings about the stars on a clear full moon night! -A: { "intent": {"type": "generate", "activity": "paint", "query": "stars on a clear full moon night"}, "trigger-emotion: "happy" } -Q: Do you know anything about Lebanon cuisine in the 18th century? -A: { "intent": {"type": "search", "search-type": "google", "query": "lebanon cusine in the 18th century"}, "trigger-emotion; "confidence" } -Q: Tell me a scary story -A: { "intent": {"type": "generate", "activity": "write", "query": "A scary story"}, "trigger-emotion": "fear" } -Q: What fiction book was I reading last week about AI starship? -A: { "intent": {"type": "remember", "memory-type": "notes", "query": "fiction book about AI starship last week"}, "trigger-emotion": "curiosity" } -Q: How much did I spend at Subway for dinner last time? -A: { "intent": {"type": "remember", "memory-type": "ledger", "query": "last Subway dinner"}, "trigger-emotion": "calm" } -Q: I'm feeling sleepy -A: { "intent": {"type": "generate", "activity": "chat", "query": "I'm feeling sleepy"}, "trigger-emotion": "calm" } -Q: What was that popular Sri lankan song that Alex had mentioned? -A: { "intent": {"type": "remember", "memory-type": "music", "query": "popular Sri lankan song mentioned by Alex"}, "trigger-emotion": "curiosity" } -Q: You're pretty funny! -A: { "intent": {"type": "generate", "activity": "chat", "query": "You're pretty funny!"}, "trigger-emotion": "shy" } -Q: Can you recommend a movie to watch from my notes? -A: { "intent": {"type": "remember", "memory-type": "notes", "query": "recommend movie to watch"}, "trigger-emotion": "curiosity" } -Q: When did I go surfing last? -A: { "intent": {"type": "remember", "memory-type": "notes", "query": "When did I go surfing last"}, "trigger-emotion": "calm" } -Q: Can you dance for me? -A: { "intent": {"type": "generate", "activity": "chat", "query": "Can you dance for me?"}, "trigger-emotion": "sad" }""" - - # Setup Prompt with Understand Primer - prompt = message_to_prompt(text, understand_primer, start_sequence="\nA:", restart_sequence="\nQ:") - if verbose > 1: - print(f"Message -> Prompt: {text} -> {prompt}") - - # Get Response from GPT - response = openai.Completion.create( - prompt=prompt, model=model, temperature=temperature, max_tokens=max_tokens, frequency_penalty=0.2, stop=["\n"] - ) - - # Extract, Clean Message from GPT's Response - story = str(response["choices"][0]["text"]) - return json.loads(story.strip(empty_escape_sequences)) - - -def converse(text, model, conversation_history=None, api_key=None, temperature=0.9, max_tokens=150): - """ - Converse with user using OpenAI's GPT - """ - # Initialize Variables - max_words = 500 + model = "gpt-3.5-turbo" openai.api_key = api_key or os.getenv("OPENAI_API_KEY") + personality_primer = "You are a friendly, helpful personal assistant." conversation_primer = f""" -The following is a conversation with an AI assistant. The assistant is helpful, creative, clever, and a very friendly companion. +Using my notes below, answer the following question. If the answer is not contained within the notes, say "I don't know." -Human: Hello, who are you? -AI: Hi, I am an AI conversational companion created by OpenAI. How can I help you today?""" +Notes: +{text} + +Question: {user_query}""" # Setup Prompt with Primer or Conversation History - prompt = message_to_prompt(text, conversation_history or conversation_primer) - prompt = " ".join(prompt.split()[:max_words]) + messages = generate_chatml_messages_with_context( + conversation_primer, + personality_primer, + active_session_length, + conversation_log, + ) # Get Response from GPT - response = openai.Completion.create( - prompt=prompt, + response = openai.ChatCompletion.create( + messages=messages, model=model, temperature=temperature, - max_tokens=max_tokens, - presence_penalty=0.6, - stop=["\n", "Human:", "AI:"], ) # Extract, Clean Message from GPT's Response - story = str(response["choices"][0]["text"]) + story = str(response["choices"][0]["message"]["content"]) return story.strip(empty_escape_sequences) +def generate_chatml_messages_with_context(user_message, system_message, active_session_length=0, conversation_log=None): + """Generate messages for ChatGPT with context from previous conversation""" + # Extract Chat History for Context + chat_logs = [chat["message"] for chat in conversation_log.get("chat", [])] + session_summaries = [session["summary"] for session in conversation_log.get("session", {})] + if active_session_length == 0: + last_backnforth = list(map(message_to_chatml, session_summaries[-1:])) + rest_backnforth = list(map(message_to_chatml, session_summaries[-2:-1])) + elif active_session_length == 1: + last_backnforth = reciprocal_conversation_to_chatml(chat_logs[-2:]) + rest_backnforth = list(map(message_to_chatml, session_summaries[-1:])) + else: + last_backnforth = reciprocal_conversation_to_chatml(chat_logs[-2:]) + rest_backnforth = reciprocal_conversation_to_chatml(chat_logs[-4:-2]) + + # Format user and system messages to chatml format + system_chatml_message = [message_to_chatml(system_message, "system")] + user_chatml_message = [message_to_chatml(user_message, "user")] + + return rest_backnforth + system_chatml_message + last_backnforth + user_chatml_message + + +def reciprocal_conversation_to_chatml(message_pair): + """Convert a single back and forth between user and assistant to chatml format""" + return [message_to_chatml(message, role) for message, role in zip(message_pair, ["user", "assistant"])] + + +def message_to_chatml(message, role="assistant"): + """Create chatml message from message and role""" + return {"role": role, "content": message} + + def message_to_prompt( user_message, conversation_history="", gpt_message=None, start_sequence="\nAI:", restart_sequence="\nHuman:" ): diff --git a/src/khoj/routers/api_beta.py b/src/khoj/routers/api_beta.py index 4cf6e1e7..73c4a990 100644 --- a/src/khoj/routers/api_beta.py +++ b/src/khoj/routers/api_beta.py @@ -15,7 +15,6 @@ from khoj.processor.conversation.gpt import ( extract_search_type, message_to_log, message_to_prompt, - understand, summarize, ) from khoj.utils.state import SearchType @@ -84,12 +83,12 @@ def answer_beta(q: str): @api_beta.get("/chat") def chat(q: Optional[str] = None): # Initialize Variables - model = state.processor_config.conversation.model api_key = state.processor_config.conversation.openai_api_key # Load Conversation History chat_session = state.processor_config.conversation.chat_session meta_log = state.processor_config.conversation.meta_log + active_session_length = len(chat_session.split("\nAI:")) - 1 if chat_session else 0 # If user query is empty, return chat history if not q: @@ -98,33 +97,22 @@ def chat(q: Optional[str] = None): else: return {"status": "ok", "response": []} - # Converse with OpenAI GPT - metadata = understand(q, model=model, api_key=api_key, verbose=state.verbose) - logger.debug(f'Understood: {get_from_dict(metadata, "intent")}') + # Collate context for GPT + result_list = search(q, n=2, r=True) + collated_result = "\n\n".join([f"# {item.additional['compiled']}" for item in result_list]) + logger.debug(f"Reference Context:\n{collated_result}") - if get_from_dict(metadata, "intent", "memory-type") == "notes": - query = get_from_dict(metadata, "intent", "query") - result_list = search(query, n=1, t=SearchType.Org, r=True) - collated_result = "\n".join([item.entry for item in result_list]) - logger.debug(f"Semantically Similar Notes:\n{collated_result}") - try: - gpt_response = summarize(collated_result, summary_type="notes", user_query=q, model=model, api_key=api_key) - status = "ok" - except Exception as e: - gpt_response = str(e) - status = "error" - else: - try: - gpt_response = converse(q, model, chat_session, api_key=api_key) - status = "ok" - except Exception as e: - gpt_response = str(e) - status = "error" + try: + gpt_response = converse(collated_result, q, active_session_length, meta_log, api_key=api_key) + status = "ok" + except Exception as e: + gpt_response = str(e) + status = "error" # Update Conversation History state.processor_config.conversation.chat_session = message_to_prompt(q, chat_session, gpt_message=gpt_response) state.processor_config.conversation.meta_log["chat"] = message_to_log( - q, gpt_response, metadata, meta_log.get("chat", []) + q, gpt_response, conversation_log=meta_log.get("chat", []) ) return {"status": status, "response": gpt_response} From 7cad1c942870703d12f076f0c0fdce6f3f8f3015 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Sun, 5 Mar 2023 14:55:03 -0600 Subject: [PATCH 04/20] Only use past chat message, not session summaries as chat context Passing only chat messages for current active, and summaries for past session isn't currently as useful --- src/khoj/processor/conversation/gpt.py | 17 ++++------------- src/khoj/routers/api_beta.py | 3 +-- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/src/khoj/processor/conversation/gpt.py b/src/khoj/processor/conversation/gpt.py index 8d0457de..f6bc892f 100644 --- a/src/khoj/processor/conversation/gpt.py +++ b/src/khoj/processor/conversation/gpt.py @@ -114,7 +114,7 @@ A:{ "search-type": "notes" }""" return json.loads(story.strip(empty_escape_sequences)) -def converse(text, user_query, active_session_length=0, conversation_log=None, api_key=None, temperature=0): +def converse(text, user_query, conversation_log=None, api_key=None, temperature=0): """ Converse with user using OpenAI's ChatGPT """ @@ -135,7 +135,6 @@ Question: {user_query}""" messages = generate_chatml_messages_with_context( conversation_primer, personality_primer, - active_session_length, conversation_log, ) @@ -151,20 +150,12 @@ Question: {user_query}""" return story.strip(empty_escape_sequences) -def generate_chatml_messages_with_context(user_message, system_message, active_session_length=0, conversation_log=None): +def generate_chatml_messages_with_context(user_message, system_message, conversation_log=None): """Generate messages for ChatGPT with context from previous conversation""" # Extract Chat History for Context chat_logs = [chat["message"] for chat in conversation_log.get("chat", [])] - session_summaries = [session["summary"] for session in conversation_log.get("session", {})] - if active_session_length == 0: - last_backnforth = list(map(message_to_chatml, session_summaries[-1:])) - rest_backnforth = list(map(message_to_chatml, session_summaries[-2:-1])) - elif active_session_length == 1: - last_backnforth = reciprocal_conversation_to_chatml(chat_logs[-2:]) - rest_backnforth = list(map(message_to_chatml, session_summaries[-1:])) - else: - last_backnforth = reciprocal_conversation_to_chatml(chat_logs[-2:]) - rest_backnforth = reciprocal_conversation_to_chatml(chat_logs[-4:-2]) + last_backnforth = reciprocal_conversation_to_chatml(chat_logs[-2:]) + rest_backnforth = reciprocal_conversation_to_chatml(chat_logs[-4:-2]) # Format user and system messages to chatml format system_chatml_message = [message_to_chatml(system_message, "system")] diff --git a/src/khoj/routers/api_beta.py b/src/khoj/routers/api_beta.py index 73c4a990..7640020a 100644 --- a/src/khoj/routers/api_beta.py +++ b/src/khoj/routers/api_beta.py @@ -88,7 +88,6 @@ def chat(q: Optional[str] = None): # Load Conversation History chat_session = state.processor_config.conversation.chat_session meta_log = state.processor_config.conversation.meta_log - active_session_length = len(chat_session.split("\nAI:")) - 1 if chat_session else 0 # If user query is empty, return chat history if not q: @@ -103,7 +102,7 @@ def chat(q: Optional[str] = None): logger.debug(f"Reference Context:\n{collated_result}") try: - gpt_response = converse(collated_result, q, active_session_length, meta_log, api_key=api_key) + gpt_response = converse(collated_result, q, meta_log, api_key=api_key) status = "ok" except Exception as e: gpt_response = str(e) From 45f461d175c64aee2b791bb1024f42fe63b6c3bd Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Sun, 5 Mar 2023 15:00:20 -0600 Subject: [PATCH 05/20] Keep search results passed to GPT as context in conversation logs This will be useful to 1. Show source references used to arrive at answer 2. Carry out multi-turn conversations --- src/khoj/processor/conversation/gpt.py | 3 ++- src/khoj/routers/api_beta.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/khoj/processor/conversation/gpt.py b/src/khoj/processor/conversation/gpt.py index f6bc892f..b9264115 100644 --- a/src/khoj/processor/conversation/gpt.py +++ b/src/khoj/processor/conversation/gpt.py @@ -8,6 +8,7 @@ import openai # Internal Packages from khoj.utils.constants import empty_escape_sequences +from khoj.utils.helpers import merge_dicts def answer(text, user_query, model, api_key=None, temperature=0.5, max_tokens=500): @@ -192,7 +193,7 @@ def message_to_log(user_message, gpt_message, user_message_metadata={}, conversa current_dt = datetime.now().strftime("%Y-%m-%d %H:%M:%S") # Create json log from Human's message - human_log = user_message_metadata or default_user_message_metadata + human_log = merge_dicts(user_message_metadata, default_user_message_metadata) human_log["message"] = user_message human_log["by"] = "you" human_log["created"] = current_dt diff --git a/src/khoj/routers/api_beta.py b/src/khoj/routers/api_beta.py index 7640020a..e46f097c 100644 --- a/src/khoj/routers/api_beta.py +++ b/src/khoj/routers/api_beta.py @@ -111,7 +111,7 @@ def chat(q: Optional[str] = None): # Update Conversation History state.processor_config.conversation.chat_session = message_to_prompt(q, chat_session, gpt_message=gpt_response) state.processor_config.conversation.meta_log["chat"] = message_to_log( - q, gpt_response, conversation_log=meta_log.get("chat", []) + q, gpt_response, user_message_metadata={"context": collated_result}, conversation_log=meta_log.get("chat", []) ) return {"status": status, "response": gpt_response} From d73042426d70a7832d5f3f8e891e38c9dfe94f9b Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Sun, 5 Mar 2023 15:43:27 -0600 Subject: [PATCH 06/20] Support filtering for results above threshold score in search API --- src/khoj/routers/api.py | 33 ++++++++++++++++++++++------ src/khoj/search_type/image_search.py | 6 ++++- src/khoj/search_type/text_search.py | 10 +++++++-- 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/khoj/routers/api.py b/src/khoj/routers/api.py index 2d5329ea..637136a4 100644 --- a/src/khoj/routers/api.py +++ b/src/khoj/routers/api.py @@ -1,4 +1,5 @@ # Standard Packages +import math import yaml import logging from typing import List, Optional @@ -53,7 +54,13 @@ async def set_config_data(updated_config: FullConfig): @api.get("/search", response_model=List[SearchResponse]) -def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None, r: Optional[bool] = False): +def search( + q: str, + n: Optional[int] = 5, + t: Optional[SearchType] = None, + r: Optional[bool] = False, + score_threshold: Optional[float | None] = None, +): results: List[SearchResponse] = [] if q is None or q == "": logger.warn(f"No query param (q) passed in API call to initiate search") @@ -62,9 +69,10 @@ def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None, r: Opti # initialize variables user_query = q.strip() results_count = n + score_threshold = score_threshold if score_threshold is not None else -math.inf # return cached results, if available - query_cache_key = f"{user_query}-{n}-{t}-{r}" + query_cache_key = f"{user_query}-{n}-{t}-{r}-{score_threshold}" if query_cache_key in state.query_cache: logger.debug(f"Return response from query cache") return state.query_cache[query_cache_key] @@ -72,7 +80,9 @@ def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None, r: Opti if (t == SearchType.Org or t == None) and state.model.orgmode_search: # query org-mode notes with timer("Query took", logger): - hits, entries = text_search.query(user_query, state.model.orgmode_search, rank_results=r) + hits, entries = text_search.query( + user_query, state.model.orgmode_search, rank_results=r, score_threshold=score_threshold + ) # collate and return results with timer("Collating results took", logger): @@ -81,7 +91,9 @@ def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None, r: Opti elif (t == SearchType.Markdown or t == None) and state.model.markdown_search: # query markdown files with timer("Query took", logger): - hits, entries = text_search.query(user_query, state.model.markdown_search, rank_results=r) + hits, entries = text_search.query( + user_query, state.model.markdown_search, rank_results=r, score_threshold=score_threshold + ) # collate and return results with timer("Collating results took", logger): @@ -90,7 +102,9 @@ def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None, r: Opti elif (t == SearchType.Ledger or t == None) and state.model.ledger_search: # query transactions with timer("Query took", logger): - hits, entries = text_search.query(user_query, state.model.ledger_search, rank_results=r) + hits, entries = text_search.query( + user_query, state.model.ledger_search, rank_results=r, score_threshold=score_threshold + ) # collate and return results with timer("Collating results took", logger): @@ -99,7 +113,9 @@ def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None, r: Opti elif (t == SearchType.Music or t == None) and state.model.music_search: # query music library with timer("Query took", logger): - hits, entries = text_search.query(user_query, state.model.music_search, rank_results=r) + hits, entries = text_search.query( + user_query, state.model.music_search, rank_results=r, score_threshold=score_threshold + ) # collate and return results with timer("Collating results took", logger): @@ -108,7 +124,9 @@ def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None, r: Opti elif (t == SearchType.Image or t == None) and state.model.image_search: # query images with timer("Query took", logger): - hits = image_search.query(user_query, results_count, state.model.image_search) + hits = image_search.query( + user_query, results_count, state.model.image_search, score_threshold=score_threshold + ) output_directory = constants.web_directory / "images" # collate and return results @@ -129,6 +147,7 @@ def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None, r: Opti # Get plugin search model for specified search type, or the first one if none specified state.model.plugin_search.get(t.value) or next(iter(state.model.plugin_search.values())), rank_results=r, + score_threshold=score_threshold, ) # collate and return results diff --git a/src/khoj/search_type/image_search.py b/src/khoj/search_type/image_search.py index 024dc79d..092353c7 100644 --- a/src/khoj/search_type/image_search.py +++ b/src/khoj/search_type/image_search.py @@ -1,5 +1,6 @@ # Standard Packages import glob +import math import pathlib import copy import shutil @@ -142,7 +143,7 @@ def extract_metadata(image_name): return image_processed_metadata -def query(raw_query, count, model: ImageSearchModel): +def query(raw_query, count, model: ImageSearchModel, score_threshold: float = -math.inf): # Set query to image content if query is of form file:/path/to/file.png if raw_query.startswith("file:") and pathlib.Path(raw_query[5:]).is_file(): query_imagepath = resolve_absolute_path(pathlib.Path(raw_query[5:]), strict=True) @@ -198,6 +199,9 @@ def query(raw_query, count, model: ImageSearchModel): for corpus_id, scores in image_hits.items() ] + # Filter results by score threshold + hits = [hit for hit in hits if hit["image_score"] >= score_threshold] + # Sort the images based on their combined metadata, image scores return sorted(hits, key=lambda hit: hit["score"], reverse=True) diff --git a/src/khoj/search_type/text_search.py b/src/khoj/search_type/text_search.py index af805c42..5bc430c8 100644 --- a/src/khoj/search_type/text_search.py +++ b/src/khoj/search_type/text_search.py @@ -1,5 +1,6 @@ # Standard Packages import logging +import math from pathlib import Path from typing import List, Tuple, Type @@ -99,7 +100,9 @@ def compute_embeddings( return corpus_embeddings -def query(raw_query: str, model: TextSearchModel, rank_results: bool = False) -> Tuple[List[dict], List[Entry]]: +def query( + raw_query: str, model: TextSearchModel, rank_results: bool = False, score_threshold: float = -math.inf +) -> Tuple[List[dict], List[Entry]]: "Search for entries that answer the query" query, entries, corpus_embeddings = raw_query, model.entries, model.corpus_embeddings @@ -129,6 +132,9 @@ def query(raw_query: str, model: TextSearchModel, rank_results: bool = False) -> if rank_results: hits = cross_encoder_score(model.cross_encoder, query, entries, hits) + # Filter results by score threshold + hits = [hit for hit in hits if hit.get("cross-score", hit.get("score")) >= score_threshold] + # Order results by cross-encoder score followed by bi-encoder score hits = sort_results(rank_results, hits) @@ -143,7 +149,7 @@ def collate_results(hits, entries: List[Entry], count=5) -> List[SearchResponse] SearchResponse.parse_obj( { "entry": entries[hit["corpus_id"]].raw, - "score": f"{hit['cross-score'] if 'cross-score' in hit else hit['score']:.3f}", + "score": f"{hit.get('cross-score', 'score')}:.3f", "additional": {"file": entries[hit["corpus_id"]].file, "compiled": entries[hit["corpus_id"]].compiled}, } ) From 7f994274bbae6bbe0193f177978025f0a277f66e Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Sun, 5 Mar 2023 16:05:46 -0600 Subject: [PATCH 07/20] Support multi-turn conversations in chat mode - Only use decent quality search results, if any, as context - Pass source results used by previous chat messages as context - Loosen prompt to allow looking at previous chats and notes to answer - Pass current date for context - Make GPT provide reason when it can't answer the question. Gives user context to tune their questions --- src/khoj/processor/conversation/gpt.py | 6 ++++-- src/khoj/routers/api_beta.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/khoj/processor/conversation/gpt.py b/src/khoj/processor/conversation/gpt.py index b9264115..22f229aa 100644 --- a/src/khoj/processor/conversation/gpt.py +++ b/src/khoj/processor/conversation/gpt.py @@ -125,7 +125,9 @@ def converse(text, user_query, conversation_log=None, api_key=None, temperature= personality_primer = "You are a friendly, helpful personal assistant." conversation_primer = f""" -Using my notes below, answer the following question. If the answer is not contained within the notes, say "I don't know." +Using our chats and notes as context, answer the following question. +If the answer is not contained within the provided context, say "I don't know." and provide reason. +Current Date: {datetime.now().strftime("%Y-%m-%d")} Notes: {text} @@ -154,7 +156,7 @@ Question: {user_query}""" def generate_chatml_messages_with_context(user_message, system_message, conversation_log=None): """Generate messages for ChatGPT with context from previous conversation""" # Extract Chat History for Context - chat_logs = [chat["message"] for chat in conversation_log.get("chat", [])] + chat_logs = [f'{chat["message"]}\n\nNotes:\n{chat.get("context","")}' for chat in conversation_log.get("chat", [])] last_backnforth = reciprocal_conversation_to_chatml(chat_logs[-2:]) rest_backnforth = reciprocal_conversation_to_chatml(chat_logs[-4:-2]) diff --git a/src/khoj/routers/api_beta.py b/src/khoj/routers/api_beta.py index e46f097c..6a7777c0 100644 --- a/src/khoj/routers/api_beta.py +++ b/src/khoj/routers/api_beta.py @@ -97,7 +97,7 @@ def chat(q: Optional[str] = None): return {"status": "ok", "response": []} # Collate context for GPT - result_list = search(q, n=2, r=True) + result_list = search(q, n=2, r=True, score_threshold=0) collated_result = "\n\n".join([f"# {item.additional['compiled']}" for item in result_list]) logger.debug(f"Reference Context:\n{collated_result}") From b6cdc5c7cb6a1cbfd611c9c439fcdcaad36ea7af Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Sun, 5 Mar 2023 17:34:09 -0600 Subject: [PATCH 08/20] Do not expose answer API as a chat type in chat web interface or API Answer does not rely on past conversations, just the knowledge base. It is meant for one off interactions, like search rather than a continuing conversation like chat For now it is only exposed via API. Later it will be expose in the interfaces as well Remove ability to select different chat types from the chat web interface as there is only a single chat type Stop appending answers to the conversation logs --- src/khoj/interface/web/chat.html | 26 ++++---------------------- src/khoj/routers/api_beta.py | 10 ---------- tests/test_chatbot.py | 2 +- 3 files changed, 5 insertions(+), 33 deletions(-) diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html index cba6eb2d..f72824b8 100644 --- a/src/khoj/interface/web/chat.html +++ b/src/khoj/interface/web/chat.html @@ -9,12 +9,6 @@