Add front-end Electron application for Khoj local file syncing (#473)

* Initial version - setup a file-push architecture for generating embeddings with Khoj
* Use state.host and state.port for configuring the URL for the indexer
* Fix parsing of PDF files
* Read markdown files from streamed data and update unit tests
* On application startup, load in embeddings from configurations files, rather than regenerating the corpus based on file system
* Init: refactor indexer/batch endpoint to support a generic file ingestion format
* Add features to better support indexing from files sent by the desktop client
* Initial commit with Electron application
- Adds electron app
* Add import for pymupdf, remove import for pypdf
* Allow user to configure khoj host URL
* Remove search type configuration from index.html
* Use v1 path for current indexer routes

tests/test_client.py

@@ -66,7 +66,7 @@ def test_index_batch(client):
     headers = {"x-api-key": "secret"}
 
     # Act
-    response = client.post("/indexer/batch", json=request_body, headers=headers)
+    response = client.post("/v1/indexer/batch", json=request_body, headers=headers)
 
     # Assert
     assert response.status_code == 200
@@ -81,7 +81,7 @@ def test_regenerate_with_valid_content_type(client):
         headers = {"x-api-key": "secret"}
 
         # Act
-        response = client.post(f"/indexer/batch?search_type={content_type}", json=request_body, headers=headers)
+        response = client.post(f"/v1/indexer/batch?search_type={content_type}", json=request_body, headers=headers)
         # Assert
         assert response.status_code == 200, f"Returned status: {response.status_code} for content type: {content_type}"
 
@@ -97,7 +97,7 @@ def test_regenerate_with_github_fails_without_pat(client):
     headers = {"x-api-key": "secret"}
 
     # Act
-    response = client.post(f"/indexer/batch?search_type=github", json=request_body, headers=headers)
+    response = client.post(f"/v1/indexer/batch?search_type=github", json=request_body, headers=headers)
     # Assert
     assert response.status_code == 200, f"Returned status: {response.status_code} for content type: github"
 

tests/test_pdf_to_jsonl.py

@@ -1,6 +1,7 @@
 # Standard Packages
 import json
 import os
+import base64
 
 # Internal Packages
 from khoj.processor.pdf.pdf_to_jsonl import PdfToJsonl
@@ -15,7 +16,7 @@ def test_single_page_pdf_to_jsonl():
     # Extract Entries from specified Pdf files
     # Read singlepage.pdf into memory as bytes
     with open("tests/data/pdf/singlepage.pdf", "rb") as f:
-        pdf_bytes = f.read()
+        pdf_bytes = base64.b64encode(f.read()).decode("utf-8")
 
     data = {"tests/data/pdf/singlepage.pdf": pdf_bytes}
     entries, entry_to_file_map = PdfToJsonl.extract_pdf_entries(pdf_files=data)
@@ -35,7 +36,7 @@ def test_multi_page_pdf_to_jsonl():
     # Act
     # Extract Entries from specified Pdf files
     with open("tests/data/pdf/multipage.pdf", "rb") as f:
-        pdf_bytes = f.read()
+        pdf_bytes = base64.b64encode(f.read()).decode("utf-8")
 
     data = {"tests/data/pdf/multipage.pdf": pdf_bytes}
     entries, entry_to_file_map = PdfToJsonl.extract_pdf_entries(pdf_files=data)

tests/test_rawconfig.py

@@ -6,18 +6,6 @@ from khoj.utils.rawconfig import TextContentConfig, ImageContentConfig
 
 
 # Test
-# ----------------------------------------------------------------------------------------------------
-def test_input_file_or_filter_required_in_text_content_config():
-    # Act
-    with pytest.raises(ValueError):
-        TextContentConfig(
-            input_files=None,
-            input_filter=None,
-            compressed_jsonl="notes.jsonl",
-            embeddings_file="note_embeddings.pt",
-        )
-
-
 # ----------------------------------------------------------------------------------------------------
 def test_input_filter_or_directories_required_in_image_content_config():
     # Act

tests/test_text_search.py

@@ -131,6 +131,65 @@ def test_entry_chunking_by_max_tokens(org_config_with_only_new_file: TextContent
     assert len(initial_notes_model.corpus_embeddings) == 2
 
 
+# ----------------------------------------------------------------------------------------------------
+# @pytest.mark.skip(reason="Flaky due to compressed_jsonl file being rewritten by other tests")
+def test_entry_chunking_by_max_tokens_not_full_corpus(
+    org_config_with_only_new_file: TextContentConfig, search_models: SearchModels
+):
+    # Arrange
+    # Insert org-mode entry with size exceeding max token limit to new org file
+    data = {
+        "readme.org": """
+* Khoj
+  /Allow natural language search on user content like notes, images using transformer based models/
+
+  All data is processed locally. User can interface with khoj app via [[./interface/emacs/khoj.el][Emacs]], API or Commandline
+
+** Dependencies
+   - Python3
+   - [[https://docs.conda.io/en/latest/miniconda.html#latest-miniconda-installer-links][Miniconda]]
+
+** Install
+   #+begin_src shell
+   git clone https://github.com/khoj-ai/khoj && cd khoj
+   conda env create -f environment.yml
+   conda activate khoj
+   #+end_src"""
+    }
+    text_search.setup(
+        OrgToJsonl,
+        data,
+        org_config_with_only_new_file,
+        search_models.text_search.bi_encoder,
+        regenerate=False,
+    )
+
+    max_tokens = 256
+    new_file_to_index = Path(org_config_with_only_new_file.input_files[0])
+    with open(new_file_to_index, "w") as f:
+        f.write(f"* Entry more than {max_tokens} words\n")
+        for index in range(max_tokens + 1):
+            f.write(f"{index} ")
+
+    data = get_org_files(org_config_with_only_new_file)
+
+    # Act
+    # reload embeddings, entries, notes model after adding new org-mode file
+    initial_notes_model = text_search.setup(
+        OrgToJsonl,
+        data,
+        org_config_with_only_new_file,
+        search_models.text_search.bi_encoder,
+        regenerate=False,
+        full_corpus=False,
+    )
+
+    # Assert
+    # verify newly added org-mode entry is split by max tokens
+    assert len(initial_notes_model.entries) == 5
+    assert len(initial_notes_model.corpus_embeddings) == 5
+
+
 # ----------------------------------------------------------------------------------------------------
 def test_regenerate_index_with_new_entry(
     content_config: ContentConfig, search_models: SearchModels, new_org_file: Path