Use Slash Commands and Add Notes Slash Command (#463)

* Store conversation command options in an Enum

* Move to slash commands instead of using @ to specify general commands

* Calculate conversation command once & pass it as arg to child funcs

* Add /notes command to respond using only knowledge base as context

This prevents the chat model to try respond using it's general world
knowledge only without any references pulled from the indexed
knowledge base

* Test general and notes slash commands in openai chat director tests

* Update gpt4all tests to use md configuration

* Add a /help tooltip

* Add dynamic support for describing slash commands. Remove default and treat notes as the default type

---------

Co-authored-by: sabaimran <narmiabas@gmail.com>

tests/conftest.py

@@ -233,22 +233,25 @@ def client(content_config: ContentConfig, search_config: SearchConfig, processor
 
 @pytest.fixture(scope="function")
 def client_offline_chat(
-    content_config: ContentConfig, search_config: SearchConfig, processor_config_offline_chat: ProcessorConfig
+    md_content_config: ContentConfig, search_config: SearchConfig, processor_config_offline_chat: ProcessorConfig
 ):
-    state.config.content_type = content_config
+    # Initialize app state
+    state.config.content_type = md_content_config
     state.config.search_type = search_config
     state.SearchType = configure_search_types(state.config)
 
-    # These lines help us Mock the Search models for these search types
+    # Index Markdown Content for Search
+    filters = [DateFilter(), WordFilter(), FileFilter()]
     state.search_models.text_search = text_search.initialize_model(search_config.asymmetric)
-    state.search_models.image_search = image_search.initialize_model(search_config.image)
-    state.content_index.org = text_search.setup(
-        OrgToJsonl, content_config.org, state.search_models.text_search.bi_encoder, regenerate=False
-    )
-    state.content_index.image = image_search.setup(
-        content_config.image, state.search_models.image_search, regenerate=False
+    state.content_index.markdown = text_search.setup(
+        MarkdownToJsonl,
+        md_content_config.markdown,
+        state.search_models.text_search.bi_encoder,
+        regenerate=False,
+        filters=filters,
     )
 
+    # Initialize Processor from Config
     state.processor_config = configure_processor(processor_config_offline_chat)
 
     configure_routes(app)

tests/test_client.py

@@ -209,7 +209,7 @@ def test_notes_search(client, content_config: ContentConfig, search_config: Sear
     assert response.status_code == 200
     # assert actual_data contains "Khoj via Emacs" entry
     search_result = response.json()[0]["entry"]
-    assert "git clone" in search_result
+    assert "git clone https://github.com/khoj-ai/khoj" in search_result
 
 
 # ----------------------------------------------------------------------------------------------------
@@ -267,6 +267,6 @@ def test_notes_search_with_exclude_filter(client, content_config: ContentConfig,
 
     # Assert
     assert response.status_code == 200
-    # assert actual_data does not contains word "Emacs"
+    # assert actual_data does not contains word "clone"
     search_result = response.json()[0]["entry"]
     assert "clone" not in search_result

tests/test_gpt4all_chat_director.py

@@ -74,7 +74,6 @@ def test_answer_from_chat_history(client_offline_chat):
 
 
 # ----------------------------------------------------------------------------------------------------
-@pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering this question yet")
 @pytest.mark.chatquality
 def test_answer_from_currently_retrieved_content(client_offline_chat):
     # Arrange
@@ -123,7 +122,10 @@ def test_answer_from_chat_history_and_previously_retrieved_content(client_offlin
 
 
 # ----------------------------------------------------------------------------------------------------
-@pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering this question yet")
+@pytest.mark.xfail(
+    AssertionError,
+    reason="Chat director not capable of answering this question yet because it requires extract_questions",
+)
 @pytest.mark.chatquality
 def test_answer_from_chat_history_and_currently_retrieved_content(client_offline_chat):
     # Arrange

tests/test_openai_chat_director.py

@@ -1,9 +1,11 @@
 # Standard Packages
 import os
+import urllib.parse
 
 # External Packages
 import pytest
 from freezegun import freeze_time
+from khoj.processor.conversation import prompts
 
 # Internal Packages
 from khoj.processor.conversation.utils import message_to_log
@@ -168,6 +170,57 @@ def test_no_answer_in_chat_history_or_retrieved_content(chat_client):
     )
 
 
+# ----------------------------------------------------------------------------------------------------
+@pytest.mark.chatquality
+def test_answer_using_general_command(chat_client):
+    # Arrange
+    query = urllib.parse.quote("/general Where was Xi Li born?")
+    message_list = []
+    populate_chat_history(message_list)
+
+    # Act
+    response = chat_client.get(f"/api/chat?q={query}&stream=true")
+    response_message = response.content.decode("utf-8")
+
+    # Assert
+    assert response.status_code == 200
+    assert "Fujiang" not in response_message
+
+
+# ----------------------------------------------------------------------------------------------------
+@pytest.mark.chatquality
+def test_answer_from_retrieved_content_using_notes_command(chat_client):
+    # Arrange
+    query = urllib.parse.quote("/notes Where was Xi Li born?")
+    message_list = []
+    populate_chat_history(message_list)
+
+    # Act
+    response = chat_client.get(f"/api/chat?q={query}&stream=true")
+    response_message = response.content.decode("utf-8")
+
+    # Assert
+    assert response.status_code == 200
+    assert "Fujiang" in response_message
+
+
+# ----------------------------------------------------------------------------------------------------
+@pytest.mark.chatquality
+def test_answer_not_known_using_notes_command(chat_client):
+    # Arrange
+    query = urllib.parse.quote("/notes Where was Testatron born?")
+    message_list = []
+    populate_chat_history(message_list)
+
+    # Act
+    response = chat_client.get(f"/api/chat?q={query}&stream=true")
+    response_message = response.content.decode("utf-8")
+
+    # Assert
+    assert response.status_code == 200
+    assert response_message == prompts.no_notes_found.format()
+
+
 # ----------------------------------------------------------------------------------------------------
 @pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering time aware questions yet")
 @pytest.mark.chatquality