Short-circuit API rate limiter for unauthenticated users (#607)

### Major
- Short-circuit API rate limiter for unauthenticated user
  Calls by unauthenticated users were failing at API rate limiter as it
  failed to access user info object. This is a bug.
  
  API rate limiter should short-circuit for unauthenicated users so a
  proper Forbidden response can be returned by API
  
  Add regression test to verify that unauthenticated users get 403
  response when calling the /chat API endpoint
  
### Minor
- Remove trailing slash to normalize khoj url in obsidian plugin settings
- Move used /api/config API controllers into separate module
- Delete unused /api/beta API endpoint
- Fix error message rendering in khoj.el, khoj obsidian chat
- Handle deprecation warnings for subscribe renew date, langchain, pydantic & logger.warn

tests/conftest.py

@@ -254,51 +254,45 @@ def md_content_config():
 
 @pytest.fixture(scope="function")
 def chat_client(search_config: SearchConfig, default_user2: KhojUser):
-    # Initialize app state
-    state.config.search_type = search_config
-    state.SearchType = configure_search_types()
+    return chat_client_builder(search_config, default_user2, require_auth=False)
 
-    LocalMarkdownConfig.objects.create(
-        input_files=None,
-        input_filter=["tests/data/markdown/*.markdown"],
-        user=default_user2,
-    )
 
-    # Index Markdown Content for Search
-    all_files = fs_syncer.collect_files(user=default_user2)
-    state.content_index, _ = configure_content(
-        state.content_index, state.config.content_type, all_files, state.search_models, user=default_user2
-    )
-
-    # Initialize Processor from Config
-    if os.getenv("OPENAI_API_KEY"):
-        chat_model = ChatModelOptionsFactory(chat_model="gpt-3.5-turbo", model_type="openai")
-        OpenAIProcessorConversationConfigFactory()
-        UserConversationProcessorConfigFactory(user=default_user2, setting=chat_model)
-
-    state.anonymous_mode = True
-
-    app = FastAPI()
-
-    configure_routes(app)
-    configure_middleware(app)
-    app.mount("/static", StaticFiles(directory=web_directory), name="static")
-    return TestClient(app)
+@pytest.fixture(scope="function")
+def chat_client_with_auth(search_config: SearchConfig, default_user2: KhojUser):
+    return chat_client_builder(search_config, default_user2, require_auth=True)
 
 
 @pytest.fixture(scope="function")
 def chat_client_no_background(search_config: SearchConfig, default_user2: KhojUser):
+    return chat_client_builder(search_config, default_user2, index_content=False, require_auth=False)
+
+
+@pytest.mark.django_db
+def chat_client_builder(search_config, user, index_content=True, require_auth=False):
     # Initialize app state
     state.config.search_type = search_config
     state.SearchType = configure_search_types()
 
+    if index_content:
+        LocalMarkdownConfig.objects.create(
+            input_files=None,
+            input_filter=["tests/data/markdown/*.markdown"],
+            user=user,
+        )
+
+        # Index Markdown Content for Search
+        all_files = fs_syncer.collect_files(user=user)
+        state.content_index, _ = configure_content(
+            state.content_index, state.config.content_type, all_files, state.search_models, user=user
+        )
+
     # Initialize Processor from Config
     if os.getenv("OPENAI_API_KEY"):
         chat_model = ChatModelOptionsFactory(chat_model="gpt-3.5-turbo", model_type="openai")
         OpenAIProcessorConversationConfigFactory()
-        UserConversationProcessorConfigFactory(user=default_user2, setting=chat_model)
+        UserConversationProcessorConfigFactory(user=user, setting=chat_model)
 
-    state.anonymous_mode = True
+    state.anonymous_mode = not require_auth
 
     app = FastAPI()
 

tests/helpers.py

@@ -1,6 +1,8 @@
 import os
+from datetime import datetime
 
 import factory
+from django.utils.timezone import make_aware
 
 from khoj.database.models import (
     ChatModelOptions,
@@ -90,4 +92,4 @@ class SubscriptionFactory(factory.django.DjangoModelFactory):
     user = factory.SubFactory(UserFactory)
     type = "standard"
     is_recurring = False
-    renewal_date = "2100-04-01"
+    renewal_date = make_aware(datetime.strptime("2100-04-01", "%Y-%m-%d"))

tests/test_client.py

@@ -1,4 +1,5 @@
 # Standard Modules
+import os
 from io import BytesIO
 from urllib.parse import quote
 
@@ -482,6 +483,21 @@ def test_user_no_data_returns_empty(client, sample_org_data, api_user3: KhojApiU
     assert response.json() == []
 
 
+@pytest.mark.skipif(os.getenv("OPENAI_API_KEY") is None, reason="requires OPENAI_API_KEY")
+@pytest.mark.django_db(transaction=True)
+def test_chat_with_unauthenticated_user(chat_client_with_auth, api_user2: KhojApiUser):
+    # Arrange
+    headers = {"Authorization": f"Bearer {api_user2.token}"}
+
+    # Act
+    auth_response = chat_client_with_auth.get(f'/api/chat?q="Hello!"&stream=true', headers=headers)
+    no_auth_response = chat_client_with_auth.get(f'/api/chat?q="Hello!"&stream=true')
+
+    # Assert
+    assert auth_response.status_code == 200
+    assert no_auth_response.status_code == 403
+
+
 def get_sample_files_data():
     return [
         ("files", ("path/to/filename.org", "* practicing piano", "text/org")),

tests/test_openai_chat_director.py

@@ -53,6 +53,7 @@ def test_chat_with_no_chat_history_or_retrieved_content(chat_client):
 
 
 # ----------------------------------------------------------------------------------------------------
+@pytest.mark.skipif(os.getenv("SERPER_DEV_API_KEY") is None, reason="requires SERPER_DEV_API_KEY")
 @pytest.mark.chatquality
 @pytest.mark.django_db(transaction=True)
 def test_chat_with_online_content(chat_client):
@@ -65,7 +66,7 @@ def test_chat_with_online_content(chat_client):
     response_message = response_message.split("### compiled references")[0]
 
     # Assert
-    expected_responses = ["http://www.paulgraham.com/greatwork.html", "Please set your SERPER_DEV_API_KEY"]
+    expected_responses = ["http://www.paulgraham.com/greatwork.html"]
     assert response.status_code == 200
     assert any([expected_response in response_message for expected_response in expected_responses]), (
         "Expected links or serper not setup in response but got: " + response_message