From 3b0783aab9a9486c9d202b84d2ec02eb99c7f880 Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Sun, 1 Jan 2023 16:22:35 -0300
Subject: [PATCH] Automate updating embeddings, search index on a hourly
 schedule

- Use the schedule pypi package
- Use QTimer to poll schedule.run_pending() regularly for jobs to run
---
 setup.py           | 1 +
 src/configure.py   | 9 +++++++++
 src/main.py        | 5 +++--
 src/routers/api.py | 2 ++
 4 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 558a23ab..add6e6e9 100644
--- a/setup.py
+++ b/setup.py
@@ -41,6 +41,7 @@ setup(
         "dateparser == 1.1.1",
         "pyqt6 == 6.3.1",
         "defusedxml == 0.7.1",
+        'schedule == 1.1.0',
     ],
     include_package_data=True,
     entry_points={"console_scripts": ["khoj = src.main:run"]},
diff --git a/src/configure.py b/src/configure.py
index f38b7d11..81952280 100644
--- a/src/configure.py
+++ b/src/configure.py
@@ -3,6 +3,9 @@ import sys
 import logging
 import json
 
+# External Packages
+import schedule
+
 # Internal Packages
 from src.processor.ledger.beancount_to_jsonl import BeancountToJsonl
 from src.processor.markdown.markdown_to_jsonl import MarkdownToJsonl
@@ -37,6 +40,12 @@ def configure_server(args, required=False):
     state.processor_config = configure_processor(args.config.processor)
 
 
+@schedule.repeat(schedule.every(1).hour)
+def update_search_index():
+    state.model = configure_search(state.model, state.config, regenerate=False)
+    logger.info("Search Index updated via Scheduler")
+
+
 def configure_search(model: SearchModels, config: FullConfig, regenerate: bool, t: SearchType = None):
     # Initialize Org Notes Search
     if (t == SearchType.Org or t == None) and config.content_type.org:
diff --git a/src/main.py b/src/main.py
index 6b81bfba..a44ef3ca 100644
--- a/src/main.py
+++ b/src/main.py
@@ -16,6 +16,7 @@ from fastapi import FastAPI
 from fastapi.staticfiles import StaticFiles
 from PyQt6 import QtWidgets
 from PyQt6.QtCore import QThread, QTimer
+import schedule
 
 # Internal Packages
 from src.configure import configure_server
@@ -99,10 +100,10 @@ def run():
 
         # Setup Signal Handlers
         signal.signal(signal.SIGINT, sigint_handler)
-        # Invoke python Interpreter every 500ms to handle signals
+        # Invoke Python interpreter every 500ms to handle signals, run scheduled tasks
         timer = QTimer()
         timer.start(500)
-        timer.timeout.connect(lambda: None)
+        timer.timeout.connect(schedule.run_pending)
 
         # Start Application
         server.start()
diff --git a/src/routers/api.py b/src/routers/api.py
index c8347f03..313e48c9 100644
--- a/src/routers/api.py
+++ b/src/routers/api.py
@@ -126,4 +126,6 @@ def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None, r: Opti
 @api.get('/update')
 def update(t: Optional[SearchType] = None, force: Optional[bool] = False):
     state.model = configure_search(state.model, state.config, regenerate=force, t=t)
+    logger.info("Search Index updated via API call")
+
     return {'status': 'ok', 'message': 'index updated'}