From 404d47f1a1fbf1ea37c539f393695f92b6f94aec Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Tue, 7 Nov 2023 02:20:11 -0800 Subject: [PATCH] Bubble up content indexing errors to notify user on client apps --- src/khoj/configure.py | 13 +++++++------ src/khoj/routers/indexer.py | 21 ++++++++++++++++----- tests/conftest.py | 2 +- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/khoj/configure.py b/src/khoj/configure.py index bc9e9bf8..ecd35cf9 100644 --- a/src/khoj/configure.py +++ b/src/khoj/configure.py @@ -1,5 +1,4 @@ # Standard Packages -import sys import logging import json from enum import Enum @@ -109,7 +108,6 @@ def configure_server( state.search_models = configure_search(state.search_models, state.config.search_type) initialize_content(regenerate, search_type, init, user) except Exception as e: - logger.error(f"🚨 Failed to configure search models", exc_info=True) raise e finally: state.config_lock.release() @@ -125,7 +123,7 @@ def initialize_content(regenerate: bool, search_type: Optional[SearchType] = Non else: logger.info("📬 Updating content index...") all_files = collect_files(user=user) - state.content_index = configure_content( + state.content_index, status = configure_content( state.content_index, state.config.content_type, all_files, @@ -134,8 +132,9 @@ def initialize_content(regenerate: bool, search_type: Optional[SearchType] = Non search_type, user=user, ) + if not status: + raise RuntimeError("Failed to update content index") except Exception as e: - logger.error(f"🚨 Failed to index content", exc_info=True) raise e @@ -165,13 +164,15 @@ def update_search_index(): logger.info("📬 Updating content index via Scheduler") for user in get_all_users(): all_files = collect_files(user=user) - state.content_index = configure_content( + state.content_index, success = configure_content( state.content_index, state.config.content_type, all_files, state.search_models, user=user ) all_files = collect_files(user=None) - state.content_index = configure_content( + state.content_index, success = configure_content( state.content_index, state.config.content_type, all_files, state.search_models, user=None ) + if not success: + raise RuntimeError("Failed to update content index") logger.info("📪 Content index updated via Scheduler") except Exception as e: logger.error(f"🚨 Error updating content index via Scheduler: {e}", exc_info=True) diff --git a/src/khoj/routers/indexer.py b/src/khoj/routers/indexer.py index 1bbf53c2..a7a1249d 100644 --- a/src/khoj/routers/indexer.py +++ b/src/khoj/routers/indexer.py @@ -126,7 +126,7 @@ async def update( # Extract required fields from config loop = asyncio.get_event_loop() - state.content_index = await loop.run_in_executor( + state.content_index, success = await loop.run_in_executor( None, configure_content, state.content_index, @@ -138,6 +138,8 @@ async def update( False, user, ) + if not success: + raise RuntimeError("Failed to update content index") logger.info(f"Finished processing batch indexing request") except Exception as e: logger.error(f"Failed to process batch indexing request: {e}", exc_info=True) @@ -145,6 +147,7 @@ async def update( f"🚨 Failed to {force} update {t} content index triggered via API call by {client} client: {e}", exc_info=True, ) + return Response(content="Failed", status_code=500) update_telemetry_state( request=request, @@ -182,18 +185,19 @@ def configure_content( t: Optional[state.SearchType] = None, full_corpus: bool = True, user: KhojUser = None, -) -> Optional[ContentIndex]: +) -> tuple[Optional[ContentIndex], bool]: content_index = ContentIndex() + success = True if t is not None and not t.value in [type.value for type in state.SearchType]: logger.warning(f"🚨 Invalid search type: {t}") - return None + return None, False search_type = t.value if t else None if files is None: logger.warning(f"🚨 No files to process for {search_type} search.") - return None + return None, True try: # Initialize Org Notes Search @@ -209,6 +213,7 @@ def configure_content( ) except Exception as e: logger.error(f"🚨 Failed to setup org: {e}", exc_info=True) + success = False try: # Initialize Markdown Search @@ -225,6 +230,7 @@ def configure_content( except Exception as e: logger.error(f"🚨 Failed to setup markdown: {e}", exc_info=True) + success = False try: # Initialize PDF Search @@ -241,6 +247,7 @@ def configure_content( except Exception as e: logger.error(f"🚨 Failed to setup PDF: {e}", exc_info=True) + success = False try: # Initialize Plaintext Search @@ -257,6 +264,7 @@ def configure_content( except Exception as e: logger.error(f"🚨 Failed to setup plaintext: {e}", exc_info=True) + success = False try: # Initialize Image Search @@ -274,6 +282,7 @@ def configure_content( except Exception as e: logger.error(f"🚨 Failed to setup images: {e}", exc_info=True) + success = False try: github_config = GithubConfig.objects.filter(user=user).prefetch_related("githubrepoconfig").first() @@ -291,6 +300,7 @@ def configure_content( except Exception as e: logger.error(f"🚨 Failed to setup GitHub: {e}", exc_info=True) + success = False try: # Initialize Notion Search @@ -308,12 +318,13 @@ def configure_content( except Exception as e: logger.error(f"🚨 Failed to setup GitHub: {e}", exc_info=True) + success = False # Invalidate Query Cache if user: state.query_cache[user.uuid] = LRU() - return content_index + return content_index, success def load_content( diff --git a/tests/conftest.py b/tests/conftest.py index fbb98476..59104123 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -196,7 +196,7 @@ def chat_client(search_config: SearchConfig, default_user2: KhojUser): # Index Markdown Content for Search all_files = fs_syncer.collect_files(user=default_user2) - state.content_index = configure_content( + state.content_index, _ = configure_content( state.content_index, state.config.content_type, all_files, state.search_models, user=default_user2 )