Merge branch 'master' of github.com:khoj-ai/khoj into features/add-chat-controls

.github/workflows/run_evals.yml

@@ -77,7 +77,7 @@ jobs:
         run: |
           # install postgres and other dependencies
           sudo apt update && sudo apt install -y git python3-pip libegl1 sqlite3 libsqlite3-dev libsqlite3-0 ffmpeg libsm6 libxext6
-          sudo apt install -y postgresql postgresql-client && sudo apt install -y postgresql-server-dev-14
+          sudo apt install -y postgresql postgresql-client && sudo apt install -y postgresql-server-dev-16
           # upgrade pip
           python -m ensurepip --upgrade && python -m pip install --upgrade pip
           # install terrarium for code sandbox

manifest.json

@@ -1,7 +1,7 @@
 {
 	"id": "khoj",
 	"name": "Khoj",
-	"version": "1.34.0",
+	"version": "1.35.2",
 	"minAppVersion": "0.15.0",
 	"description": "Your Second Brain",
 	"author": "Khoj Inc.",

src/interface/desktop/package.json

@@ -1,6 +1,6 @@
 {
   "name": "Khoj",
-  "version": "1.34.0",
+  "version": "1.35.2",
   "description": "Your Second Brain",
   "author": "Khoj Inc. <team@khoj.dev>",
   "license": "GPL-3.0-or-later",

src/interface/emacs/khoj.el

@@ -6,7 +6,7 @@
 ;;         Saba Imran <saba@khoj.dev>
 ;; Description: Your Second Brain
 ;; Keywords: search, chat, ai, org-mode, outlines, markdown, pdf, image
-;; Version: 1.34.0
+;; Version: 1.35.2
 ;; Package-Requires: ((emacs "27.1") (transient "0.3.0") (dash "2.19.1"))
 ;; URL: https://github.com/khoj-ai/khoj/tree/master/src/interface/emacs
 

src/interface/obsidian/manifest.json

@@ -1,7 +1,7 @@
 {
 	"id": "khoj",
 	"name": "Khoj",
-	"version": "1.34.0",
+	"version": "1.35.2",
 	"minAppVersion": "0.15.0",
 	"description": "Your Second Brain",
 	"author": "Khoj Inc.",

src/interface/obsidian/package.json

@@ -1,6 +1,6 @@
 {
     "name": "Khoj",
-    "version": "1.34.0",
+    "version": "1.35.2",
     "description": "Your Second Brain",
     "author": "Debanjum Singh Solanky, Saba Imran <team@khoj.dev>",
     "license": "GPL-3.0-or-later",

src/interface/obsidian/versions.json

@@ -109,5 +109,8 @@
 	"1.33.0": "0.15.0",
 	"1.33.1": "0.15.0",
 	"1.33.2": "0.15.0",
-	"1.34.0": "0.15.0"
+	"1.34.0": "0.15.0",
+	"1.35.0": "0.15.0",
+	"1.35.1": "0.15.0",
+	"1.35.2": "0.15.0"
 }

src/interface/web/package.json

@@ -1,6 +1,6 @@
 {
     "name": "khoj-ai",
-    "version": "1.34.0",
+    "version": "1.35.2",
     "private": true,
     "scripts": {
         "dev": "next dev",

src/khoj/database/management/commands/delete_orphaned_fileobjects.py

@@ -1,4 +1,5 @@
 from django.core.management.base import BaseCommand
+from django.db import transaction
 from django.db.models import Exists, OuterRef
 
 from khoj.database.models import Entry, FileObject

src/khoj/database/migrations/0079_entry_file_object.py

@@ -9,45 +9,40 @@ def migrate_entry_objects(apps, schema_editor):
     FileObject = apps.get_model("database", "FileObject")
     db_alias = schema_editor.connection.alias
 
-    # Create lookup dictionary of all file objects
+    # Process file objects in chunks
-    file_objects_map = {(fo.user_id, fo.file_name): fo for fo in FileObject.objects.using(db_alias).all()}
-
-    # Process entries in chunks of 1000
     chunk_size = 1000
     processed = 0
-
+    processed_file_ids = set()
-    processed_entry_ids = set()
 
     while True:
-        entries = list(
+        file_objects = list(
-            Entry.objects.using(db_alias)
+            FileObject.objects.using(db_alias)
+            .exclude(id__in=processed_file_ids)
             .select_related("user")
-            .filter(file_object__isnull=True)
+            .only("id", "user", "file_name")[:chunk_size]
-            .exclude(id__in=processed_entry_ids)
-            .only("id", "user", "file_path")[:chunk_size]
         )
 
-        if not entries:
+        if not file_objects:
             break
 
-        processed_entry_ids.update([entry.id for entry in entries])
+        processed_file_ids.update([fo.id for fo in file_objects])
 
-        entries_to_update = []
+        for file_object in file_objects:
-        for entry in entries:
             try:
-                file_object = file_objects_map.get((entry.user_id, entry.file_path))
+                # Find all entries matching this file object
-                if file_object:
+                matching_entries = Entry.objects.using(db_alias).filter(
-                    entry.file_object = file_object
+                    user_id=file_object.user_id, file_path=file_object.file_name, file_object__isnull=True
-                    entries_to_update.append(entry)
+                )
+
+                if matching_entries.exists():
+                    # Update all matching entries in bulk
+                    matching_entries.update(file_object=file_object)
             except Exception as e:
-                print(f"Error processing entry {entry.id}: {str(e)}")
+                print(f"Error processing file object {file_object.id}: {str(e)}")
                 continue
 
-        if entries_to_update:
+        processed += len(file_objects)
-            Entry.objects.using(db_alias).bulk_update(entries_to_update, ["file_object"], batch_size=chunk_size)
+        print(f"Processed {processed} file objects")
-
-        processed += len(entries)
-        print(f"Processed {processed} entries")
 
 
 def reverse_migration(apps, schema_editor):

versions.json

@@ -109,5 +109,8 @@
 	"1.33.0": "0.15.0",
 	"1.33.1": "0.15.0",
 	"1.33.2": "0.15.0",
-	"1.34.0": "0.15.0"
+	"1.34.0": "0.15.0",
+	"1.35.0": "0.15.0",
+	"1.35.1": "0.15.0",
+	"1.35.2": "0.15.0"
 }