Automate updating embeddings, search index on a hourly schedule

- Use the schedule pypi package
- Use QTimer to poll schedule.run_pending() regularly for jobs to run

setup.py

@@ -41,6 +41,7 @@ setup(
         "dateparser == 1.1.1",
         "pyqt6 == 6.3.1",
         "defusedxml == 0.7.1",
+        'schedule == 1.1.0',
     ],
     include_package_data=True,
     entry_points={"console_scripts": ["khoj = src.main:run"]},

src/configure.py

@@ -3,6 +3,9 @@ import sys
 import logging
 import json
 
+# External Packages
+import schedule
+
 # Internal Packages
 from src.processor.ledger.beancount_to_jsonl import BeancountToJsonl
 from src.processor.markdown.markdown_to_jsonl import MarkdownToJsonl
@@ -37,6 +40,12 @@ def configure_server(args, required=False):
     state.processor_config = configure_processor(args.config.processor)
 
 
+@schedule.repeat(schedule.every(1).hour)
+def update_search_index():
+    state.model = configure_search(state.model, state.config, regenerate=False)
+    logger.info("Search Index updated via Scheduler")
+
+
 def configure_search(model: SearchModels, config: FullConfig, regenerate: bool, t: SearchType = None):
     # Initialize Org Notes Search
     if (t == SearchType.Org or t == None) and config.content_type.org:

src/main.py

@@ -16,6 +16,7 @@ from fastapi import FastAPI
 from fastapi.staticfiles import StaticFiles
 from PyQt6 import QtWidgets
 from PyQt6.QtCore import QThread, QTimer
+import schedule
 
 # Internal Packages
 from src.configure import configure_server
@@ -99,10 +100,10 @@ def run():
 
         # Setup Signal Handlers
         signal.signal(signal.SIGINT, sigint_handler)
-        # Invoke python Interpreter every 500ms to handle signals
+        # Invoke Python interpreter every 500ms to handle signals, run scheduled tasks
         timer = QTimer()
         timer.start(500)
-        timer.timeout.connect(lambda: None)
+        timer.timeout.connect(schedule.run_pending)
 
         # Start Application
         server.start()

src/routers/api.py

@@ -126,4 +126,6 @@ def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None, r: Opti
 @api.get('/update')
 def update(t: Optional[SearchType] = None, force: Optional[bool] = False):
     state.model = configure_search(state.model, state.config, regenerate=force, t=t)
+    logger.info("Search Index updated via API call")
+
     return {'status': 'ok', 'message': 'index updated'}