Bubble up content indexing errors to notify user on client apps

src/khoj/configure.py

@@ -1,5 +1,4 @@
 # Standard Packages
-import sys
 import logging
 import json
 from enum import Enum
@@ -109,7 +108,6 @@ def configure_server(
         state.search_models = configure_search(state.search_models, state.config.search_type)
         initialize_content(regenerate, search_type, init, user)
     except Exception as e:
-        logger.error(f"🚨 Failed to configure search models", exc_info=True)
         raise e
     finally:
         state.config_lock.release()
@@ -125,7 +123,7 @@ def initialize_content(regenerate: bool, search_type: Optional[SearchType] = Non
             else:
                 logger.info("📬 Updating content index...")
                 all_files = collect_files(user=user)
-                state.content_index = configure_content(
+                state.content_index, status = configure_content(
                     state.content_index,
                     state.config.content_type,
                     all_files,
@@ -134,8 +132,9 @@ def initialize_content(regenerate: bool, search_type: Optional[SearchType] = Non
                     search_type,
                     user=user,
                 )
+                if not status:
+                    raise RuntimeError("Failed to update content index")
         except Exception as e:
-            logger.error(f"🚨 Failed to index content", exc_info=True)
             raise e
 
 
@@ -165,13 +164,15 @@ def update_search_index():
         logger.info("📬 Updating content index via Scheduler")
         for user in get_all_users():
             all_files = collect_files(user=user)
-            state.content_index = configure_content(
+            state.content_index, success = configure_content(
                 state.content_index, state.config.content_type, all_files, state.search_models, user=user
             )
         all_files = collect_files(user=None)
-        state.content_index = configure_content(
+        state.content_index, success = configure_content(
             state.content_index, state.config.content_type, all_files, state.search_models, user=None
         )
+        if not success:
+            raise RuntimeError("Failed to update content index")
         logger.info("📪 Content index updated via Scheduler")
     except Exception as e:
         logger.error(f"🚨 Error updating content index via Scheduler: {e}", exc_info=True)

src/khoj/routers/indexer.py

@@ -126,7 +126,7 @@ async def update(
 
         # Extract required fields from config
         loop = asyncio.get_event_loop()
-        state.content_index = await loop.run_in_executor(
+        state.content_index, success = await loop.run_in_executor(
             None,
             configure_content,
             state.content_index,
@@ -138,6 +138,8 @@ async def update(
             False,
             user,
         )
+        if not success:
+            raise RuntimeError("Failed to update content index")
         logger.info(f"Finished processing batch indexing request")
     except Exception as e:
         logger.error(f"Failed to process batch indexing request: {e}", exc_info=True)
@@ -145,6 +147,7 @@ async def update(
             f"🚨 Failed to {force} update {t} content index triggered via API call by {client} client: {e}",
             exc_info=True,
         )
+        return Response(content="Failed", status_code=500)
 
     update_telemetry_state(
         request=request,
@@ -182,18 +185,19 @@ def configure_content(
     t: Optional[state.SearchType] = None,
     full_corpus: bool = True,
     user: KhojUser = None,
-) -> Optional[ContentIndex]:
+) -> tuple[Optional[ContentIndex], bool]:
     content_index = ContentIndex()
 
+    success = True
     if t is not None and not t.value in [type.value for type in state.SearchType]:
         logger.warning(f"🚨 Invalid search type: {t}")
-        return None
+        return None, False
 
     search_type = t.value if t else None
 
     if files is None:
         logger.warning(f"🚨 No files to process for {search_type} search.")
-        return None
+        return None, True
 
     try:
         # Initialize Org Notes Search
@@ -209,6 +213,7 @@ def configure_content(
             )
     except Exception as e:
         logger.error(f"🚨 Failed to setup org: {e}", exc_info=True)
+        success = False
 
     try:
         # Initialize Markdown Search
@@ -225,6 +230,7 @@ def configure_content(
 
     except Exception as e:
         logger.error(f"🚨 Failed to setup markdown: {e}", exc_info=True)
+        success = False
 
     try:
         # Initialize PDF Search
@@ -241,6 +247,7 @@ def configure_content(
 
     except Exception as e:
         logger.error(f"🚨 Failed to setup PDF: {e}", exc_info=True)
+        success = False
 
     try:
         # Initialize Plaintext Search
@@ -257,6 +264,7 @@ def configure_content(
 
     except Exception as e:
         logger.error(f"🚨 Failed to setup plaintext: {e}", exc_info=True)
+        success = False
 
     try:
         # Initialize Image Search
@@ -274,6 +282,7 @@ def configure_content(
 
     except Exception as e:
         logger.error(f"🚨 Failed to setup images: {e}", exc_info=True)
+        success = False
 
     try:
         github_config = GithubConfig.objects.filter(user=user).prefetch_related("githubrepoconfig").first()
@@ -291,6 +300,7 @@ def configure_content(
 
     except Exception as e:
         logger.error(f"🚨 Failed to setup GitHub: {e}", exc_info=True)
+        success = False
 
     try:
         # Initialize Notion Search
@@ -308,12 +318,13 @@ def configure_content(
 
     except Exception as e:
         logger.error(f"🚨 Failed to setup GitHub: {e}", exc_info=True)
+        success = False
 
     # Invalidate Query Cache
     if user:
         state.query_cache[user.uuid] = LRU()
 
-    return content_index
+    return content_index, success
 
 
 def load_content(

tests/conftest.py

@@ -196,7 +196,7 @@ def chat_client(search_config: SearchConfig, default_user2: KhojUser):
 
     # Index Markdown Content for Search
     all_files = fs_syncer.collect_files(user=default_user2)
-    state.content_index = configure_content(
+    state.content_index, _ = configure_content(
         state.content_index, state.config.content_type, all_files, state.search_models, user=default_user2
     )