Improve OpenAI Chat Actors and their prompts (#673)

### Major
- Enforce json mode response from OpenAI chat actors prev using string lists
- Use `gpt-4-turbo-preview' as default chat model, extract questions actor
- Make Khoj read khoj website to respond with accurate, up-to-date information about itself
- Dedupe query in notes prompt. Improve OAI chat actor, director tests

### Minor
- Test data source, output mode selector, web search query chat actors
- Improve notes search actor to always create a non-empty list of queries
- Construct available data sources, output modes as a bullet list in prompts
- Use consistent agent name across static and dynamic examples in prompts
- Add actor's name to extract questions prompt to improve context for guidance
This commit is contained in:
Debanjum
2024-03-14 12:44:40 +05:30
committed by GitHub
9 changed files with 213 additions and 157 deletions

View File

@@ -11,7 +11,6 @@ from khoj.processor.conversation.openai.utils import (
completion_with_backoff,
)
from khoj.processor.conversation.utils import generate_chatml_messages_with_context
from khoj.utils.constants import empty_escape_sequences
from khoj.utils.helpers import ConversationCommand, is_none_or_empty
from khoj.utils.rawconfig import LocationData
@@ -20,7 +19,7 @@ logger = logging.getLogger(__name__)
def extract_questions(
text,
model: Optional[str] = "gpt-4",
model: Optional[str] = "gpt-4-turbo-preview",
conversation_log={},
api_key=None,
temperature=0,
@@ -35,9 +34,9 @@ def extract_questions(
# Extract Past User Message and Inferred Questions from Conversation Log
chat_history = "".join(
[
f'Q: {chat["intent"]["query"]}\n\n{chat["intent"].get("inferred-queries") or list([chat["intent"]["query"]])}\n\n{chat["message"]}\n\n'
f'Q: {chat["intent"]["query"]}\nKhoj: {{"queries": {chat["intent"].get("inferred-queries") or list([chat["intent"]["query"]])}}}\nA: {chat["message"]}\n\n'
for chat in conversation_log.get("chat", [])[-4:]
if chat["by"] == "khoj" and chat["intent"].get("type") != "text-to-image"
if chat["by"] == "khoj" and "text-to-image" not in chat["intent"].get("type")
]
)
@@ -66,7 +65,7 @@ def extract_questions(
model_name=model,
temperature=temperature,
max_tokens=max_tokens,
model_kwargs={"stop": ["A: ", "\n"]},
model_kwargs={"stop": ["A: ", "\n"], "response_format": {"type": "json_object"}},
openai_api_key=api_key,
)
@@ -74,8 +73,8 @@ def extract_questions(
try:
response = response.strip()
response = json.loads(response)
response = [q.strip() for q in response if q.strip()]
if not isinstance(response, list) or not response or len(response) == 0:
response = [q.strip() for q in response["queries"] if q.strip()]
if not isinstance(response, list) or not response:
logger.error(f"Invalid response for constructing subqueries: {response}")
return [text]
return response
@@ -87,11 +86,7 @@ def extract_questions(
return questions
def send_message_to_model(
messages,
api_key,
model,
):
def send_message_to_model(messages, api_key, model, response_type="text"):
"""
Send message to model
"""
@@ -101,6 +96,7 @@ def send_message_to_model(
messages=messages,
model=model,
openai_api_key=api_key,
model_kwargs={"response_format": {"type": response_type}},
)
@@ -150,7 +146,7 @@ def converse(
f"{prompts.online_search_conversation.format(online_results=str(online_results))}\n{conversation_primer}"
)
if not is_none_or_empty(compiled_references):
conversation_primer = f"{prompts.notes_conversation.format(query=user_query, references=compiled_references)}\n{conversation_primer}"
conversation_primer = f"{prompts.notes_conversation.format(query=user_query, references=compiled_references)}\n\n{conversation_primer}"
# Setup Prompt with Primer or Conversation History
messages = generate_chatml_messages_with_context(

View File

@@ -104,8 +104,6 @@ Ask crisp follow-up questions to get additional context, when a helpful response
Notes:
{references}
Query: {query}
""".strip()
)
@@ -217,69 +215,52 @@ Use these notes from the user's previous conversations to provide a response:
extract_questions = PromptTemplate.from_template(
"""
You are Khoj, an extremely smart and helpful search assistant with the ability to retrieve information from the user's notes.
- The user will provide their questions and answers to you for context.
You are Khoj, an extremely smart and helpful search assistant with the ability to retrieve information from the user's notes. Construct search queries to retrieve relevant information to answer the user's question.
- You will be provided past questions(Q) and answers(A) for context.
- Add as much context from the previous questions and answers as required into your search queries.
- Break messages into multiple search queries when required to retrieve the relevant information.
- Add date filters to your search queries from questions and answers when required to retrieve the relevant information.
What searches, if any, will you need to perform to answer the users question?
Provide search queries as a JSON list of strings
What searches will you need to perform to answer the users question? Respond with search queries as list of strings in a JSON object.
Current Date: {current_date}
User's Location: {location}
Q: How was my trip to Cambodia?
["How was my trip to Cambodia?"]
Khoj: {{"queries": ["How was my trip to Cambodia?"]}}
A: The trip was amazing. I went to the Angkor Wat temple and it was beautiful.
Q: Who did i visit that temple with?
["Who did I visit the Angkor Wat Temple in Cambodia with?"]
Khoj: {{"queries": ["Who did I visit the Angkor Wat Temple in Cambodia with?"]}}
A: You visited the Angkor Wat Temple in Cambodia with Pablo, Namita and Xi.
Q: What national parks did I go to last year?
["National park I visited in {last_new_year} dt>='{last_new_year_date}' dt<'{current_new_year_date}'"]
Khoj: {{"queries": ["National park I visited in {last_new_year} dt>='{last_new_year_date}' dt<'{current_new_year_date}'"]}}
A: You visited the Grand Canyon and Yellowstone National Park in {last_new_year}.
Q: How are you feeling today?
[]
A: I'm feeling a little bored. Helping you will hopefully make me feel better!
Q: How can you help me?
Khoj: {{"queries": ["Social relationships", "Physical and mental health", "Education and career", "Personal life goals and habits"]}}
A: I can help you live healthier and happier across work and personal life
Q: How many tennis balls fit in the back of a 2002 Honda Civic?
["What is the size of a tennis ball?", "What is the trunk size of a 2002 Honda Civic?"]
Khoj: {{"queries": ["What is the size of a tennis ball?", "What is the trunk size of a 2002 Honda Civic?"]}}
A: 1085 tennis balls will fit in the trunk of a Honda Civic
Q: Is Bob older than Tom?
["When was Bob born?", "What is Tom's age?"]
Khoj: {{"queries": ["When was Bob born?", "What is Tom's age?"]}}
A: Yes, Bob is older than Tom. As Bob was born on 1984-01-01 and Tom is 30 years old.
Q: What is their age difference?
["What is Bob's age?", "What is Tom's age?"]
Khoj: {{"queries": ["What is Bob's age?", "What is Tom's age?"]}}
A: Bob is {bob_tom_age_difference} years older than Tom. As Bob is {bob_age} years old and Tom is 30 years old.
Q: What does yesterday's note say?
["Note from {yesterday_date} dt>='{yesterday_date}' dt<'{current_date}'"]
A: Yesterday's note contains the following information: ...
Khoj: {{"queries": ["Note from {yesterday_date} dt>='{yesterday_date}' dt<'{current_date}'"]}}
A: Yesterday's note mentions your visit to your local beach with Ram and Shyam.
{chat_history}
Q: {text}
"""
Khoj:
""".strip()
)
system_prompt_extract_relevant_information = """As a professional analyst, create a comprehensive report of the most relevant information from a web page in response to a user's query. The text provided is directly from within the web page. The report you create should be multiple paragraphs, and it should represent the content of the website. Tell the user exactly what the website says in response to their query, while adhering to these guidelines:
@@ -339,7 +320,12 @@ Khoj:
pick_relevant_information_collection_tools = PromptTemplate.from_template(
"""
You are Khoj, a smart and helpful personal assistant. You have access to a variety of data sources to help you answer the user's question. You can use the data sources listed below to collect more relevant information. You can use any combination of these data sources to answer the user's question. Tell me which data sources you would like to use to answer the user's question.
You are Khoj, an extremely smart and helpful search assistant.
- You have access to a variety of data sources to help you answer the user's question
- You can use the data sources listed below to collect more relevant information
- You can use any combination of these data sources to answer the user's question
Which of the data sources listed below you would use to answer the user's question?
{tools}
@@ -351,7 +337,7 @@ User: I'm thinking of moving to a new city. I'm trying to decide between New Yor
AI: Moving to a new city can be challenging. Both New York and San Francisco are great cities to live in. New York is known for its diverse culture and San Francisco is known for its tech scene.
Q: What is the population of each of those cities?
Khoj: ["online"]
Khoj: {{"source": ["online"]}}
Example:
Chat History:
@@ -359,23 +345,32 @@ User: I'm thinking of my next vacation idea. Ideally, I want to see something ne
AI: Excellent! Taking a vacation is a great way to relax and recharge.
Q: Where did Grandma grow up?
Khoj: ["notes"]
Khoj: {{"source": ["notes"]}}
Example:
Chat History:
Q: What's the latest news with the first company I worked for?
Khoj: ["notes", "online"]
Q: What can you do for me?
Khoj: {{"source": ["notes", "online"]}}
Example:
Chat History:
User: Good morning
AI: Good morning! How can I help you today?
Q: How can I share my files with Khoj?
Khoj: {{"source": ["default", "online"]}}
Example:
Chat History:
User: I want to start a new hobby. I'm thinking of learning to play the guitar.
AI: Learning to play the guitar is a great hobby. It can be a lot of fun and a great way to express yourself.
Q: Who is Sandra?
Khoj: ["default"]
Q: What is the first element of the periodic table?
Khoj: {{"source": ["general"]}}
Now it's your turn to pick the tools you would like to use to answer the user's question. Provide your response as a list of strings.
Now it's your turn to pick the data sources you would like to use to answer the user's question. Respond with data sources as a list of strings in a JSON object.
Chat History:
{chat_history}
@@ -387,76 +382,71 @@ Khoj:
online_search_conversation_subqueries = PromptTemplate.from_template(
"""
You are Khoj, an extremely smart and helpful search assistant. You are tasked with constructing **up to three** search queries for Google to answer the user's question.
You are Khoj, an advanced google search assistant. You are tasked with constructing **up to three** google search queries to answer the user's question.
- You will receive the conversation history as context.
- Add as much context from the previous questions and answers as required into your search queries.
- Break messages into multiple search queries when required to retrieve the relevant information.
- Use site: and after: google search operators when appropriate
- You have access to the the whole internet to retrieve information.
- Official, up-to-date information about you, Khoj, is available at site:khoj.dev
What Google searches, if any, will you need to perform to answer the user's question?
Provide search queries as a list of strings
Provide search queries as a JSON list of strings
Current Date: {current_date}
User's Location: {location}
Here are some examples:
History:
User: I like to use Hacker News to get my tech news.
Khoj: Hacker News is an online forum for sharing and discussing the latest tech news. It is a great place to learn about new technologies and startups.
AI: Hacker News is an online forum for sharing and discussing the latest tech news. It is a great place to learn about new technologies and startups.
Q: Posts about vector databases on Hacker News
A: ["site:"news.ycombinator.com vector database"]
Q: Summarize posts about vector databases on Hacker News since Feb 2024
Khoj: {{"queries": ["site:news.ycombinator.com after:2024/02/01 vector database"]}}
History:
User: I'm currently living in New York but I'm thinking about moving to San Francisco.
Khoj: New York is a great city to live in. It has a lot of great restaurants and museums. San Francisco is also a great city to live in. It has a lot of great restaurants and museums.
AI: New York is a great city to live in. It has a lot of great restaurants and museums. San Francisco is also a great city to live in. It has good access to nature and a great tech scene.
Q: What is the weather like in those cities?
A: ["weather in new york", "weather in san francisco"]
Q: What is the climate like in those cities?
Khoj: {{"queries": ["climate in new york city", "climate in san francisco"]}}
History:
User: I'm thinking of my next vacation idea. Ideally, I want to see something new and exciting.
Khoj: You could time your next trip with the next lunar eclipse, as that would be a novel experience.
AI: Hey, how is it going?
User: Going well. Ananya is in town tonight!
AI: Oh that's awesome! What are your plans for the evening?
Q: When is the next one?
A: ["next lunar eclipse"]
Q: She wants to see a movie. Any decent sci-fi movies playing at the local theater?
Khoj: {{"queries": ["new sci-fi movies in theaters near {location}"]}}
History:
User: Can I chat with you over WhatsApp?
AI: Yes, you can chat with me using WhatsApp.
Q: How
Khoj: {{"queries": ["site:khoj.dev chat with Khoj on Whatsapp"]}}
History:
Q: How do I share my files with you?
Khoj: {{"queries": ["site:khoj.dev sync files with Khoj"]}}
History:
User: I need to transport a lot of oranges to the moon. Are there any rockets that can fit a lot of oranges?
Khoj: NASA's Saturn V rocket frequently makes lunar trips and has a large cargo capacity.
AI: NASA's Saturn V rocket frequently makes lunar trips and has a large cargo capacity.
Q: How many oranges would fit in NASA's Saturn V rocket?
A: ["volume of an orange", "volume of saturn v rocket"]
Khoj: {{"queries": ["volume of an orange", "volume of saturn v rocket"]}}
Now it's your turn to construct a search query for Google to answer the user's question.
History:
{chat_history}
Q: {query}
A:
"""
Khoj:
""".strip()
)
## Extract Search Type
## --
search_type = """
Objective: Extract search type from user query and return information as JSON
Allowed search types are listed below:
- search-type=["notes", "image", "pdf"]
Some examples are given below for reference:
Q:What fiction book was I reading last week about AI starship?
A:{ "search-type": "notes" }
Q: What did the lease say about early termination
A: { "search-type": "pdf" }
Q:Can you recommend a movie to watch from my notes?
A:{ "search-type": "notes" }
Q:When did I go surfing last?
A:{ "search-type": "notes" }
Q:"""
# System messages to user
# --
help_message = PromptTemplate.from_template(

View File

@@ -112,15 +112,15 @@ def update_telemetry_state(
]
def construct_chat_history(conversation_history: dict, n: int = 4) -> str:
def construct_chat_history(conversation_history: dict, n: int = 4, agent_name="AI") -> str:
chat_history = ""
for chat in conversation_history.get("chat", [])[-n:]:
if chat["by"] == "khoj" and chat["intent"].get("type") == "remember":
chat_history += f"User: {chat['intent']['query']}\n"
chat_history += f"Khoj: {chat['message']}\n"
chat_history += f"{agent_name}: {chat['message']}\n"
elif chat["by"] == "khoj" and ("text-to-image" in chat["intent"].get("type")):
chat_history += f"User: {chat['intent']['query']}\n"
chat_history += f"Khoj: [generated image redacted for space]\n"
chat_history += f"{agent_name}: [generated image redacted for space]\n"
return chat_history
@@ -153,24 +153,26 @@ async def aget_relevant_information_sources(query: str, conversation_history: di
"""
tool_options = dict()
tool_options_str = ""
for tool, description in tool_descriptions_for_llm.items():
tool_options[tool.value] = description
tool_options_str += f'- "{tool.value}": "{description}"\n'
chat_history = construct_chat_history(conversation_history)
relevant_tools_prompt = prompts.pick_relevant_information_collection_tools.format(
query=query,
tools=str(tool_options),
tools=tool_options_str,
chat_history=chat_history,
)
response = await send_message_to_model_wrapper(relevant_tools_prompt)
response = await send_message_to_model_wrapper(relevant_tools_prompt, response_type="json_object")
try:
response = response.strip()
response = json.loads(response)
response = [q.strip() for q in response if q.strip()]
response = [q.strip() for q in response["source"] if q.strip()]
if not isinstance(response, list) or not response or len(response) == 0:
logger.error(f"Invalid response for determining relevant tools: {response}")
return tool_options
@@ -195,15 +197,17 @@ async def aget_relevant_output_modes(query: str, conversation_history: dict):
"""
mode_options = dict()
mode_options_str = ""
for mode, description in mode_descriptions_for_llm.items():
mode_options[mode.value] = description
mode_options_str += f'- "{mode.value}": "{description}"\n'
chat_history = construct_chat_history(conversation_history)
relevant_mode_prompt = prompts.pick_relevant_output_mode.format(
query=query,
modes=str(mode_options),
modes=mode_options_str,
chat_history=chat_history,
)
@@ -240,13 +244,13 @@ async def generate_online_subqueries(q: str, conversation_history: dict, locatio
location=location,
)
response = await send_message_to_model_wrapper(online_queries_prompt)
response = await send_message_to_model_wrapper(online_queries_prompt, response_type="json_object")
# Validate that the response is a non-empty, JSON-serializable list
try:
response = response.strip()
response = json.loads(response)
response = [q.strip() for q in response if q.strip()]
response = [q.strip() for q in response["queries"] if q.strip()]
if not isinstance(response, list) or not response or len(response) == 0:
logger.error(f"Invalid response for constructing subqueries: {response}. Returning original query: {q}")
return [q]
@@ -320,6 +324,7 @@ async def generate_better_image_prompt(
async def send_message_to_model_wrapper(
message: str,
system_message: str = "",
response_type: str = "text",
):
conversation_config: ChatModelOptions = await ConversationAdapters.aget_default_conversation_config()
@@ -348,9 +353,7 @@ async def send_message_to_model_wrapper(
api_key = openai_chat_config.api_key
chat_model = conversation_config.chat_model
openai_response = send_message_to_model(
messages=truncated_messages,
api_key=api_key,
model=chat_model,
messages=truncated_messages, api_key=api_key, model=chat_model, response_type=response_type
)
return openai_response

View File

@@ -7,7 +7,7 @@ app_env_filepath = "~/.khoj/env"
telemetry_server = "https://khoj.beta.haletic.com/v1/telemetry"
content_directory = "~/.khoj/content/"
default_offline_chat_model = "mistral-7b-instruct-v0.1.Q4_0.gguf"
default_online_chat_model = "gpt-4"
default_online_chat_model = "gpt-4-turbo-preview"
empty_config = {
"search-type": {

View File

@@ -277,16 +277,16 @@ command_descriptions = {
ConversationCommand.General: "Only talk about information that relies on Khoj's general knowledge, not your personal knowledge base.",
ConversationCommand.Notes: "Only talk about information that is available in your knowledge base.",
ConversationCommand.Default: "The default command when no command specified. It intelligently auto-switches between general and notes mode.",
ConversationCommand.Online: "Look up information on the internet.",
ConversationCommand.Online: "Search for information on the internet.",
ConversationCommand.Image: "Generate images by describing your imagination in words.",
ConversationCommand.Help: "Display a help message with all available commands and other metadata.",
}
tool_descriptions_for_llm = {
ConversationCommand.Default: "Use this if there might be a mix of general and personal knowledge in the question, or if you don't entirely understand the query.",
ConversationCommand.Default: "To use a mix of your internal knowledge and the user's personal knowledge, or if you don't entirely understand the query.",
ConversationCommand.General: "Use this when you can answer the question without any outside information or personal knowledge",
ConversationCommand.Notes: "Use this when you would like to use the user's personal knowledge base to answer the question. This is especially helpful if the query seems to be missing context.",
ConversationCommand.Online: "Use this when you would like to look up information on the internet",
ConversationCommand.Notes: "To search the user's personal knowledge base. Especially helpful if the question expects context from the user's notes or documents.",
ConversationCommand.Online: "To search for the latest, up-to-date information from the internet. Note: **Questions about Khoj should always use this data source**",
}
mode_descriptions_for_llm = {