Miscellaneous bugs and fixes for chat sessions (#646)

* Display given_name field only if it is not None

* Add default slugs in the migration script

* Ensure that updated_at is saved appropriately, make sure most recent chat is returned for default history

* Remove the bin button from the chat interface, given deletion is handled in the drop-down menus

* Refresh the side panel when a new chat is created

* Improveme tool retrieval prompt, don't let /online fail, and improve parsing of extract questions

* Fix ending chat response by offline chat on hitting a stop phrase

Previously the whole phrase wouldn't be in the same response chunk, so
chat response wouldn't stop on hitting a stop phrase

Now use a queue to keep track of last 3 chunks, and to stop responding
when hit a stop phrase

* Make chat on Obsidian backward compatible post chat session API updates

- Make chat on Obsidian get chat history from
  `responseJson.response.chat' when available (i.e when using new api)
- Else fallback to loading chat history from
  responseJson.response (i.e when using old api)

* Fix detecting success of indexing update in khoj.el

When khoj.el attempts to index on a Khoj server served behind an https
endpoint, the success reponse status contains plist with certs. This
doesn't mean the update failed.

Look for :errors key in status instead to determine if indexing API
call failed. This fixes detecting indexing API call success on the
Khoj Emacs client, even for Khoj servers running behind SSL/HTTPS

* Fix the mechanism for populating notes references in the conversation primer for both offline and online chat

* Return conversation.default when empty list for dynamic prompt selection, send all cmds in telemetry

* Fix making chat on Obsidian backward compatible post chat session API updates

New API always has conversation_id set, not `chat' which can be unset
when chat session is empty.

So use conversation_id to decide whether to get chat logs from
`responseJson.response.chat' or `responseJson.response' instead

---------

Co-authored-by: Debanjum Singh Solanky <debanjum@gmail.com>

tests/test_gpt4all_chat_director.py

@@ -22,15 +22,21 @@ fake = Faker()
 
 # Helpers
 # ----------------------------------------------------------------------------------------------------
-def populate_chat_history(message_list, user):
+def generate_history(message_list):
     # Generate conversation logs
     conversation_log = {"chat": []}
-    for user_message, llm_message, context in message_list:
+    for user_message, gpt_message, context in message_list:
         conversation_log["chat"] += message_to_log(
             user_message,
-            llm_message,
+            gpt_message,
             {"context": context, "intent": {"query": user_message, "inferred-queries": f'["{user_message}"]'}},
         )
+    return conversation_log
+
+
+def populate_chat_history(message_list, user):
+    # Generate conversation logs
+    conversation_log = generate_history(message_list)
 
     # Update Conversation Metadata Logs in Database
     ConversationFactory(user=user, conversation_log=conversation_log)
@@ -515,8 +521,9 @@ async def test_get_correct_tools_with_chat_history(client_offline_chat):
         (
             "Let's talk about the current events around the world.",
             "Sure, let's discuss the current events. What would you like to know?",
+            [],
         ),
-        ("What's up in New York City?", "A Pride parade has recently been held in New York City, on July 31st."),
+        ("What's up in New York City?", "A Pride parade has recently been held in New York City, on July 31st.", []),
     ]
     chat_history = populate_chat_history(chat_log)
 
@@ -526,15 +533,3 @@ async def test_get_correct_tools_with_chat_history(client_offline_chat):
     # Assert
     tools = [tool.value for tool in tools]
     assert tools == ["online"]
-
-
-def populate_chat_history(message_list):
-    # Generate conversation logs
-    conversation_log = {"chat": []}
-    for user_message, gpt_message in message_list:
-        conversation_log["chat"] += message_to_log(
-            user_message,
-            gpt_message,
-            {"context": [], "intent": {"query": user_message, "inferred-queries": f'["{user_message}"]'}},
-        )
-    return conversation_log

tests/test_openai_chat_director.py

@@ -22,7 +22,7 @@ if api_key is None:
 
 # Helpers
 # ----------------------------------------------------------------------------------------------------
-def populate_chat_history(message_list, user=None):
+def generate_history(message_list):
     # Generate conversation logs
     conversation_log = {"chat": []}
     for user_message, gpt_message, context in message_list:
@@ -31,6 +31,12 @@ def populate_chat_history(message_list, user=None):
             gpt_message,
             {"context": context, "intent": {"query": user_message, "inferred-queries": f'["{user_message}"]'}},
         )
+    return conversation_log
+
+
+def populate_chat_history(message_list, user):
+    # Generate conversation logs
+    conversation_log = generate_history(message_list)
 
     # Update Conversation Metadata Logs in Database
     ConversationFactory(user=user, conversation_log=conversation_log)
@@ -491,10 +497,11 @@ async def test_get_correct_tools_with_chat_history(chat_client):
         (
             "Let's talk about the current events around the world.",
             "Sure, let's discuss the current events. What would you like to know?",
+            [],
         ),
-        ("What's up in New York City?", "A Pride parade has recently been held in New York City, on July 31st."),
+        ("What's up in New York City?", "A Pride parade has recently been held in New York City, on July 31st.", []),
     ]
-    chat_history = populate_chat_history(chat_log)
+    chat_history = generate_history(chat_log)
 
     # Act
     tools = await aget_relevant_information_sources(user_query, chat_history)
@@ -502,15 +509,3 @@ async def test_get_correct_tools_with_chat_history(chat_client):
     # Assert
     tools = [tool.value for tool in tools]
     assert tools == ["online"]
-
-
-def populate_chat_history(message_list):
-    # Generate conversation logs
-    conversation_log = {"chat": []}
-    for user_message, gpt_message in message_list:
-        conversation_log["chat"] += message_to_log(
-            user_message,
-            gpt_message,
-            {"context": [], "intent": {"query": user_message, "inferred-queries": f'["{user_message}"]'}},
-        )
-    return conversation_log