From e281a498b4d814f51b8f53063f7b04972bdb4005 Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Sun, 26 Mar 2023 06:34:18 +0700
Subject: [PATCH 01/18] Style Khoj search org buffer via elisp instead of
 in-buffer settings

---
 src/interface/emacs/khoj.el | 40 ++++++++++++++++++++-----------------
 1 file changed, 22 insertions(+), 18 deletions(-)

diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el
index b0b999f0..b7fa444d 100644
--- a/src/interface/emacs/khoj.el
+++ b/src/interface/emacs/khoj.el
@@ -196,22 +196,21 @@ Use `which-key` if available, else display simple message in echo area"
 
 (defun khoj--extract-entries-as-org (json-response query)
   "Convert JSON-RESPONSE, QUERY from API to `org-mode' entries."
-  (let ((org-results-buffer-format-str "* %s\n%s\n#+STARTUP: showall hidestars inlineimages"))
-    (thread-last
-      json-response
-      ;; Extract and render each org-mode entry from response
-      (mapcar (lambda (json-response-item)
-                (thread-last
-                  ;; Extract org entry from each item in json response
-                  (cdr (assoc 'entry json-response-item))
-                  ;; Format org entry as a string
-                  (format "%s")
-                  ;; Standardize results to 2nd level heading for consistent rendering
-                  (replace-regexp-in-string "^\*+" "**"))))
-      ;; Render entries into org formatted string with query set as as top level heading
-      (format org-results-buffer-format-str query)
-      ;; remove leading (, ) or SPC from extracted entries string
-      (replace-regexp-in-string "^[\(\) ]" ""))))
+  (thread-last
+    json-response
+    ;; Extract and render each org-mode entry from response
+    (mapcar (lambda (json-response-item)
+              (thread-last
+                ;; Extract org entry from each item in json response
+                (cdr (assoc 'entry json-response-item))
+                ;; Format org entry as a string
+                (format "%s")
+                ;; Standardize results to 2nd level heading for consistent rendering
+                (replace-regexp-in-string "^\*+" "**"))))
+    ;; Render entries into org formatted string with query set as as top level heading
+    (format "* %s\n%s\n" query)
+    ;; remove leading (, ) or SPC from extracted entries string
+    (replace-regexp-in-string "^[\(\) ]" "")))
 
 (defun khoj--extract-entries-as-ledger (json-response query)
   "Convert JSON-RESPONSE, QUERY from API to ledger entries."
@@ -319,8 +318,13 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
              ((equal content-type "ledger") (khoj--extract-entries-as-ledger json-response query))
              ((equal content-type "image") (khoj--extract-entries-as-images json-response query))
              (t (khoj--extract-entries json-response query))))
-      (cond ((equal content-type "org") (progn (org-mode)
-                                               (visual-line-mode)))
+      (cond ((equal content-type "org") (progn (visual-line-mode)
+                                               (org-mode)
+                                               (setq-local
+                                                org-startup-folded "showall"
+                                                org-hide-leading-stars t
+                                                org-startup-with-inline-images t)
+                                               (org-set-startup-visibility)))
             ((equal content-type "markdown") (progn (markdown-mode)
                                                     (visual-line-mode)))
             ((equal content-type "ledger") (beancount-mode))

From c92d79118a64cf22c18a119f3d6622d64374fd3c Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Sun, 26 Mar 2023 08:41:51 +0700
Subject: [PATCH 02/18] Install Khoj server from Emacs using khoj.el

---
 src/interface/emacs/khoj.el | 56 +++++++++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el
index b7fa444d..d884dc1c 100644
--- a/src/interface/emacs/khoj.el
+++ b/src/interface/emacs/khoj.el
@@ -171,6 +171,62 @@ Use `which-key` if available, else display simple message in echo area"
                                 nil t t))
     (message "%s" (khoj--keybindings-info-message))))
 
+
+;; ----------------
+;; Khoj Setup
+;; ----------------
+(defcustom khoj-server-command
+  (or (executable-find "khoj")
+      (executable-find "khoj.exe")
+      "khoj")
+  "Command to interact with Khoj server."
+  :group 'khoj
+  :type 'string)
+
+(defcustom khoj-server-python-command
+  (if (equal system-type 'windows-nt)
+      (or (executable-find "py")
+          (executable-find "pythonw")
+          "python")
+    (if (executable-find "python")
+        "python"
+      ;; Fallback on systems where python is not
+      ;; symlinked to python3.
+      "python3"))
+  "The Python interpreter used for the Khoj server.
+
+Khoj will try to use the system interpreter if it exists. If you wish
+to use a specific python interpreter (from a virtual environment
+for example), set this to the full interpreter path."
+  :type '(choice (const :tag "python" "python")
+                 (const :tag "python3" "python3")
+                 (const :tag "pythonw (Python on Windows)" "pythonw")
+                 (const :tag "py (other Python on Windows)" "py")
+                 (string :tag "Other"))
+  :safe (lambda (val)
+          (member val '("python" "python3" "pythonw" "py")))
+  :group 'khoj)
+
+(defun khoj--server-get-version ()
+  "Return the khoj server version."
+  (with-temp-buffer
+    (call-process khoj-server-command nil t nil "--version")
+    (goto-char (point-min))
+    (re-search-forward "\\([a-z0-9.]+\\)")
+    (match-string 1)))
+
+(defun khoj--server-install-upgrade ()
+  "Install or upgrade the khoj server."
+  (with-temp-buffer
+    (message "khoj.el: Installing server...")
+    (if (/= (apply 'call-process khoj-server-python-command
+                     nil t nil
+                     "-m" "pip" "install" "--upgrade"
+                     '("khoj-assistant"))
+            0)
+        (message "khoj.el: Failed to install Khoj server. Please install it manually using pip install `khoj-assistant'.\n%s" (buffer-string))
+      (message "khoj.el: Installed and upgraded Khoj server version: %s" (khoj--server-get-version)))))
+
 
 ;; -----------------------------------------------
 ;; Extract and Render Entries of each Content Type

From 99d19dcf43b0421b9535e064c92eb29034e82cb9 Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Sun, 26 Mar 2023 09:38:46 +0700
Subject: [PATCH 03/18] Start Khoj server from Emacs using khoj.el

---
 src/interface/emacs/khoj.el | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el
index d884dc1c..6ca5f853 100644
--- a/src/interface/emacs/khoj.el
+++ b/src/interface/emacs/khoj.el
@@ -183,6 +183,11 @@ Use `which-key` if available, else display simple message in echo area"
   :group 'khoj
   :type 'string)
 
+(defcustom khoj-server-args '("--no-gui")
+  "Arguments to pass to Khoj server on startup."
+  :group 'khoj
+  :type '(repeat string))
+
 (defcustom khoj-server-python-command
   (if (equal system-type 'windows-nt)
       (or (executable-find "py")
@@ -207,6 +212,9 @@ for example), set this to the full interpreter path."
           (member val '("python" "python3" "pythonw" "py")))
   :group 'khoj)
 
+(defvar khoj--server-process nil "Track Khoj server process.")
+(defvar khoj--server-name "khoj-server" "Track Khoj server buffer.")
+
 (defun khoj--server-get-version ()
   "Return the khoj server version."
   (with-temp-buffer
@@ -227,6 +235,25 @@ for example), set this to the full interpreter path."
         (message "khoj.el: Failed to install Khoj server. Please install it manually using pip install `khoj-assistant'.\n%s" (buffer-string))
       (message "khoj.el: Installed and upgraded Khoj server version: %s" (khoj--server-get-version)))))
 
+(defun khoj--server-start ()
+  "Start the khoj server."
+  (let* ((url-parts (split-string (cadr (split-string khoj-server-url "://")) ":"))
+         (server-host (nth 0 url-parts))
+         (server-port (or (nth 1 url-parts) "80"))
+         (server-args (append khoj-server-args
+                              (list (format "--host=%s" server-host)
+                                    (format "--port=%s" server-port)))))
+    (message "khoj.el: Starting server at %s %s..." server-host server-port)
+    (setq khoj--server-process
+          (apply 'start-process
+                 khoj--server-name
+                 khoj--server-name
+                 khoj-server-command
+                 server-args))
+    (if (not khoj--server-process)
+        (message "khoj.el: Failed to start Khoj server. Please start it manually by running `khoj' on terminal.\n%s" (buffer-string))
+      (message "khoj.el: Khoj server running at: %s" khoj-server-url))))
+
 
 ;; -----------------------------------------------
 ;; Extract and Render Entries of each Content Type

From 82eb4bfd0d6b7277bd76f4cfc026af97e68820b6 Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Sun, 26 Mar 2023 10:12:06 +0700
Subject: [PATCH 04/18] Setup Khoj server on opening khoj from with Emacs

- Create helper methods to check, stop, restart, setup khoj server
- (Ask to) setup khoj server on calling khoj main entrypoint function
---
 src/interface/emacs/khoj.el | 42 +++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el
index 6ca5f853..fca6be40 100644
--- a/src/interface/emacs/khoj.el
+++ b/src/interface/emacs/khoj.el
@@ -65,6 +65,11 @@
   :group 'khoj
   :type 'string)
 
+(defcustom is-khoj-server-local t
+  "Is Khoj server on local machine?."
+  :group 'khoj
+  :type 'boolean)
+
 (defcustom khoj-image-width 156
   "Width of rendered images returned by Khoj."
   :group 'khoj
@@ -254,6 +259,39 @@ for example), set this to the full interpreter path."
         (message "khoj.el: Failed to start Khoj server. Please start it manually by running `khoj' on terminal.\n%s" (buffer-string))
       (message "khoj.el: Khoj server running at: %s" khoj-server-url))))
 
+(defun khoj--server-running? ()
+  "Check if the khoj server is running."
+  (or
+   ;; check for when server process handled from within emacs
+   (and khoj--server-process
+        (not (null (process-live-p khoj--server-process))))
+   ;; else general check via ping to khoj-server-url
+   (ignore-errors
+     (not (null (khoj--get-enabled-content-types) t)))))
+
+(defun khoj--server-stop ()
+  "Stop the khoj server."
+  (when (khoj--server-running?)
+    (message "khoj.el: Stopping server...")
+    (kill-process khoj--server-process)
+    (message "khoj.el: Stopped server.")))
+
+(defun khoj--server-restart ()
+  "Restart the khoj server."
+  (khoj--server-stop)
+  (khoj--server-start))
+
+(defun khoj--server-setup ()
+  "Install and start the khoj server, if required."
+  ;; Install khoj server, if not available but expected on local machine
+  (when (and is-khoj-server-local
+             (or (not (executable-find khoj-server-command))
+                 (not (khoj--server-get-version))))
+      (khoj--server-install-upgrade))
+  ;; Start khoj server if not running at expected URL
+  (when (not (khoj--server-running?))
+    (khoj--server-start)))
+
 
 ;; -----------------------------------------------
 ;; Extract and Render Entries of each Content Type
@@ -781,6 +819,10 @@ Paragraph only starts at first text after blank line."
 (defun khoj ()
   "Natural, Incremental Search for your personal notes, transactions and images."
   (interactive)
+  ;; Setup khoj server if not running before executing user commands
+  (when (and (not (khoj--server-running?))
+             (y-or-n-p "Could not connect to Khoj server. Should I install and start it?"))
+    (khoj--server-setup))
   (khoj-menu))
 
 (provide 'khoj)

From 50760acc3706f6eb9f6eb19d02176543793fae44 Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Sun, 26 Mar 2023 11:56:04 +0700
Subject: [PATCH 05/18] Wait for Khoj server to get ready before opening
 khoj.el transient menu

- Use process filter, sentinel to mark when khoj server is ready or not
- Display server messages for visibility into server boot-up process
- Wait until server ready to open khoj transient menu in Emacs
  Until then khoj features wouldn't work anyway, so avoids confusion
---
 src/interface/emacs/khoj.el | 32 +++++++++++++++++++++++---------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el
index fca6be40..528256b5 100644
--- a/src/interface/emacs/khoj.el
+++ b/src/interface/emacs/khoj.el
@@ -218,7 +218,8 @@ for example), set this to the full interpreter path."
   :group 'khoj)
 
 (defvar khoj--server-process nil "Track Khoj server process.")
-(defvar khoj--server-name "khoj-server" "Track Khoj server buffer.")
+(defvar khoj--server-name "*khoj-server*" "Track Khoj server buffer.")
+(defvar khoj--server-ready? nil "Track if khoj server is ready to receive API calls.")
 
 (defun khoj--server-get-version ()
   "Return the khoj server version."
@@ -250,14 +251,25 @@ for example), set this to the full interpreter path."
                                     (format "--port=%s" server-port)))))
     (message "khoj.el: Starting server at %s %s..." server-host server-port)
     (setq khoj--server-process
-          (apply 'start-process
-                 khoj--server-name
-                 khoj--server-name
-                 khoj-server-command
-                 server-args))
-    (if (not khoj--server-process)
-        (message "khoj.el: Failed to start Khoj server. Please start it manually by running `khoj' on terminal.\n%s" (buffer-string))
-      (message "khoj.el: Khoj server running at: %s" khoj-server-url))))
+          (make-process
+           :name khoj--server-name
+           :buffer khoj--server-name
+           :command (append (list khoj-server-command) server-args)
+           :sentinel (lambda (process event)
+                       (message "khoj.el: khoj server stopped with: %s" event)
+                       (setq khoj--server-ready? nil))
+           :filter (lambda (process msg)
+                     (cond ((string-match (format "Uvicorn running on %s" khoj-server-url) msg)
+                            (setq khoj--server-ready? t))
+                           ((not khoj--server-ready?)
+                            (dolist (line (split-string msg "\n"))
+                              (message "khoj.el: %s" (nth 1 (split-string msg "  " t " *"))))))
+                     ;; call default process filter to write output to process buffer
+                     (internal-default-process-filter process msg))
+           ))
+    (set-process-query-on-exit-flag khoj--server-process nil)
+    (when (not khoj--server-process)
+        (message "khoj.el: Failed to start Khoj server. Please start it manually by running `khoj' on terminal.\n%s" (buffer-string)))))
 
 (defun khoj--server-running? ()
   "Check if the khoj server is running."
@@ -823,6 +835,8 @@ Paragraph only starts at first text after blank line."
   (when (and (not (khoj--server-running?))
              (y-or-n-p "Could not connect to Khoj server. Should I install and start it?"))
     (khoj--server-setup))
+  (while (not khoj--server-ready?)
+    (sleep-for 0.5))
   (khoj-menu))
 
 (provide 'khoj)

From cb40a96c851e4ae266833b438638edc8710efbad Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Sun, 26 Mar 2023 18:44:10 +0700
Subject: [PATCH 06/18] Index configured org files from khoj.el

- Set `khoj-org-files-index' to list of files to index
- Defaults to indexing org-agenda-files
- Uses khoj server api to configure org files to index
---
 src/interface/emacs/khoj.el | 84 +++++++++++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)

diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el
index 528256b5..8e15c36d 100644
--- a/src/interface/emacs/khoj.el
+++ b/src/interface/emacs/khoj.el
@@ -217,6 +217,11 @@ for example), set this to the full interpreter path."
           (member val '("python" "python3" "pythonw" "py")))
   :group 'khoj)
 
+(defcustom khoj-org-files-index (org-agenda-files t t)
+  "List of org-files to index on khoj server"
+  :group 'khoj
+  :type '(repeat string))
+
 (defvar khoj--server-process nil "Track Khoj server process.")
 (defvar khoj--server-name "*khoj-server*" "Track Khoj server buffer.")
 (defvar khoj--server-ready? nil "Track if khoj server is ready to receive API calls.")
@@ -304,6 +309,72 @@ for example), set this to the full interpreter path."
   (when (not (khoj--server-running?))
     (khoj--server-start)))
 
+(defun khoj--get-content-index-dir (config type)
+  "Extract directory containing index files of specified content TYPE from CONFIG.
+CONFIG is json obtained from Khoj config API."
+  (--> config
+       (cdr (assoc 'content-type it))
+       (cdr (assoc type it))
+       (cdr (assoc 'embeddings-file it))
+       (split-string it "/")
+       (butlast it)
+       (string-join it "/")))
+
+(defun khoj--server-configure ()
+  "Configure the khoj server to index specified files."
+  (interactive)
+  (let* ((current-config
+          (with-temp-buffer
+            (url-insert-file-contents (format "%s/api/config/data" khoj-server-url))
+            (ignore-error json-end-of-file (json-parse-buffer :object-type 'alist :array-type 'list :null-object json-null :false-object json-false))))
+         (default-config
+           (with-temp-buffer
+             (url-insert-file-contents (format "%s/api/config/data/default" khoj-server-url))
+             (ignore-error json-end-of-file (json-parse-buffer :object-type 'alist :array-type 'list :null-object json-null :false-object json-false))))
+         (default-index-dir (khoj--get-content-index-dir default-config 'org))
+         (config (or current-config default-config)))
+    (cond
+     ;; If khoj backend is not configured yet
+     ((not current-config)
+      (setq config (delq (assoc 'content-type config) config))
+      (setq config (delq (assoc 'processor config) config))
+      (add-to-list 'config `(content-type . ((org . ((input-files . ,khoj-org-files-index)
+                                                     (input-filter . ,json-null)
+                                                     (compressed-jsonl . ,(format "%s/org.jsonl.gz" default-index-dir))
+                                                     (embeddings-file . ,(format "%s/org.pt" default-index-dir))
+                                                     (index-heading-entries . ,json-false))))))
+      (khoj--post-new-config config)
+      (message "khoj.el: ⚙️ Generated new khoj server configuration."))
+
+     ;; Else if khoj config has no org content config
+     ((not (alist-get 'org (alist-get 'content-type config)))
+      (let ((new-content-type (alist-get 'content-type config)))
+        (setq new-content-type (delq (assoc 'org new-content-type) new-content-type))
+        (add-to-list 'new-content-type `(org . ((input-files . ,khoj-org-files-index)
+                                                (input-filter . ,json-null)
+                                                (compressed-jsonl . ,(format "%s/org.jsonl.gz" default-index-dir))
+                                                (embeddings-file . ,(format "%s/org.pt" default-index-dir))
+                                                (index-heading-entries . ,json-false))))
+        (setq config (delq (assoc 'content-type config) config))
+        (add-to-list 'config `(content-type . ,new-content-type)))
+      (khoj--post-new-config config)
+      (message "Khoj: ⚙️ Added org content to index on khoj server."))
+
+     ;; Else if khoj is not configured to index org files
+     ((not (equal (alist-get 'input-files (alist-get 'org (alist-get 'content-type config))) khoj-org-files-index))
+      (let* ((index-directory (khoj--get-content-index-dir config 'org))
+             (new-content-type (alist-get 'content-type config)))
+        (setq new-content-type (delq (assoc 'org new-content-type) new-content-type))
+        (add-to-list 'new-content-type `(org . ((input-files . ,khoj-org-files-index)
+                                                (input-filter . ,json-null)
+                                                (compressed-jsonl . ,(format "%s/org.jsonl.gz" index-directory))
+                                                (embeddings-file . ,(format "%s/org.pt" index-directory))
+                                                (index-heading-entries . ,json-false))))
+        (setq config (delq (assoc 'content-type config) config))
+        (add-to-list 'config `(content-type . ,new-content-type)))
+      (khoj--post-new-config config)
+      (message "Khoj: ⚙️ Updated org content in index on khoj server")))))
+
 
 ;; -----------------------------------------------
 ;; Extract and Render Entries of each Content Type
@@ -413,6 +484,19 @@ for example), set this to the full interpreter path."
 ;; Query Khoj API
 ;; --------------
 
+(defun khoj--post-new-config (config)
+  "Configure khoj server with provided CONFIG."
+  ;; POST provided config to khoj server
+  (let ((url-request-method "POST")
+        (url-request-extra-headers '(("Content-Type" . "application/json")))
+        (url-request-data (json-encode-alist config))
+        (config-url (format "%s/api/config/data" khoj-server-url)))
+    (with-current-buffer (url-retrieve-synchronously config-url)
+      (buffer-string)))
+  ;; Update index on khoj server after configuration update
+  (let ((khoj--server-ready? nil))
+    (url-retrieve (format "%s/api/update?t=org" khoj-server-url) #'identity)))
+
 (defun khoj--get-enabled-content-types ()
   "Get content types enabled for search from API."
   (let ((config-url (format "%s/api/config/types" khoj-server-url))

From 8a21aff4380a6b5f7b388542cdec2568f20d396d Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Mon, 27 Mar 2023 01:11:43 +0700
Subject: [PATCH 07/18] Make khoj.el server start, stop, restart, setup methods
 interactive

No need to erase temporary buffers before working on them
---
 src/interface/emacs/khoj.el | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el
index 8e15c36d..ba9fc344 100644
--- a/src/interface/emacs/khoj.el
+++ b/src/interface/emacs/khoj.el
@@ -248,6 +248,7 @@ for example), set this to the full interpreter path."
 
 (defun khoj--server-start ()
   "Start the khoj server."
+  (interactive)
   (let* ((url-parts (split-string (cadr (split-string khoj-server-url "://")) ":"))
          (server-host (nth 0 url-parts))
          (server-port (or (nth 1 url-parts) "80"))
@@ -288,6 +289,7 @@ for example), set this to the full interpreter path."
 
 (defun khoj--server-stop ()
   "Stop the khoj server."
+  (interactive)
   (when (khoj--server-running?)
     (message "khoj.el: Stopping server...")
     (kill-process khoj--server-process)
@@ -295,11 +297,13 @@ for example), set this to the full interpreter path."
 
 (defun khoj--server-restart ()
   "Restart the khoj server."
+  (interactive)
   (khoj--server-stop)
   (khoj--server-start))
 
 (defun khoj--server-setup ()
   "Install and start the khoj server, if required."
+  (interactive)
   ;; Install khoj server, if not available but expected on local machine
   (when (and is-khoj-server-local
              (or (not (executable-find khoj-server-command))
@@ -502,7 +506,6 @@ CONFIG is json obtained from Khoj config API."
   (let ((config-url (format "%s/api/config/types" khoj-server-url))
         (url-request-method "GET"))
     (with-temp-buffer
-      (erase-buffer)
       (url-insert-file-contents config-url)
       (thread-last
         (json-parse-buffer :object-type 'alist)
@@ -649,7 +652,6 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
          (encoded-query (url-hexify-string query))
          (query-url (format "%s/api/chat?q=%s" khoj-server-url encoded-query)))
     (with-temp-buffer
-      (erase-buffer)
       (url-insert-file-contents query-url)
       (json-parse-buffer :object-type 'alist))))
 
@@ -892,7 +894,7 @@ Paragraph only starts at first text after blank line."
   (interactive (list (transient-args transient-current-command)))
   (khoj--chat))
 
-(transient-define-prefix khoj-menu ()
+(transient-define-prefix khoj--menu ()
   "Create Khoj Menu to Configure and Execute Commands."
   [["Configure Search"
     ("n" "Results Count" "--results-count=" :init-value (lambda (obj) (oset obj value (format "%s" khoj-results-count))))
@@ -921,7 +923,7 @@ Paragraph only starts at first text after blank line."
     (khoj--server-setup))
   (while (not khoj--server-ready?)
     (sleep-for 0.5))
-  (khoj-menu))
+  (khoj--menu))
 
 (provide 'khoj)
 

From d7fb9a596e62cb747d40810eb45ed70178a98a27 Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Mon, 27 Mar 2023 01:16:05 +0700
Subject: [PATCH 08/18] Auto configure server before loading khoj-menu

If the config hasn't changed there'll be no update. If config has
changed indexing will get triggered asynchronously. But user cannot
make query till indexing done

As easier to know when server ready to configure
---
 src/interface/emacs/khoj.el | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el
index ba9fc344..c2aa013d 100644
--- a/src/interface/emacs/khoj.el
+++ b/src/interface/emacs/khoj.el
@@ -266,7 +266,9 @@ for example), set this to the full interpreter path."
                        (setq khoj--server-ready? nil))
            :filter (lambda (process msg)
                      (cond ((string-match (format "Uvicorn running on %s" khoj-server-url) msg)
-                            (setq khoj--server-ready? t))
+                            (progn
+                              (setq khoj--server-ready? t)
+                              (khoj--server-configure)))
                            ((not khoj--server-ready?)
                             (dolist (line (split-string msg "\n"))
                               (message "khoj.el: %s" (nth 1 (split-string msg "  " t " *"))))))
@@ -923,6 +925,7 @@ Paragraph only starts at first text after blank line."
     (khoj--server-setup))
   (while (not khoj--server-ready?)
     (sleep-for 0.5))
+  (khoj--server-configure)
   (khoj--menu))
 
 (provide 'khoj)

From 359a2cacefdd14f57341a7449d022a8f0ec53342 Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Mon, 27 Mar 2023 01:29:58 +0700
Subject: [PATCH 09/18] Fix khoj--server-running to work with unconfigured or
 external server

- If khoj server started outside emacs, khoj--server-ready should be set
to true by khoj--server-running method (instead of waiting for proc msg)

- If khoj server is unconfigured the /config/types endpoint wouldn't
return anything. Using config/data/default allows checking khoj server
running status without requiring it to be configured as well
---
 src/interface/emacs/khoj.el | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el
index c2aa013d..8bfaf782 100644
--- a/src/interface/emacs/khoj.el
+++ b/src/interface/emacs/khoj.el
@@ -281,13 +281,14 @@ for example), set this to the full interpreter path."
 
 (defun khoj--server-running? ()
   "Check if the khoj server is running."
-  (or
-   ;; check for when server process handled from within emacs
-   (and khoj--server-process
-        (not (null (process-live-p khoj--server-process))))
-   ;; else general check via ping to khoj-server-url
-   (ignore-errors
-     (not (null (khoj--get-enabled-content-types) t)))))
+  (when (or
+       ;; check for when server process handled from within emacs
+       (and khoj--server-process
+            (not (null (process-live-p khoj--server-process))))
+       ;; else general check via ping to khoj-server-url
+       (ignore-errors
+         (not (null (url-retrieve-synchronously (format "%s/api/config/data/default" khoj-server-url))))))
+      (setq khoj--server-ready? t)))
 
 (defun khoj--server-stop ()
   "Stop the khoj server."

From 924424c7541a6a5209e8c41935792cd45885f1f4 Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Mon, 27 Mar 2023 02:08:21 +0700
Subject: [PATCH 10/18] Throw actionable exceptions when content types or chat
 not configured

---
 src/interface/emacs/khoj.el | 10 ++++++++--
 src/khoj/routers/api.py     | 15 +++++++++++++++
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el
index 8bfaf782..604bdc0a 100644
--- a/src/interface/emacs/khoj.el
+++ b/src/interface/emacs/khoj.el
@@ -655,8 +655,14 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
          (encoded-query (url-hexify-string query))
          (query-url (format "%s/api/chat?q=%s" khoj-server-url encoded-query)))
     (with-temp-buffer
-      (url-insert-file-contents query-url)
-      (json-parse-buffer :object-type 'alist))))
+      (condition-case ex
+          (progn
+            (url-insert-file-contents query-url)
+            (json-parse-buffer :object-type 'alist))
+        ('file-error (cond ((string-match "Internal server error" (nth 2 ex))
+                      (message "Chat processor not configured. Configure OpenAI API key and restart it. Exception: [%s]" ex))
+                     (t (message "Chat exception: [%s]" ex))))))))
+
 
 (defun khoj--render-chat-message (message sender &optional receive-date)
   "Render chat messages as `org-mode' list item.
diff --git a/src/khoj/routers/api.py b/src/khoj/routers/api.py
index 312c1ffc..8ee6de13 100644
--- a/src/khoj/routers/api.py
+++ b/src/khoj/routers/api.py
@@ -33,6 +33,12 @@ def get_default_config_data():
 @api.get("/config/types", response_model=List[str])
 def get_config_types():
     """Get configured content types"""
+    if state.config is None or state.config.content_type is None:
+        raise HTTPException(
+            status_code=500,
+            detail="Content types not configured. Configure at least one content type on server and restart it.",
+        )
+
     configured_content_types = state.config.content_type.dict(exclude_none=True)
     return [
         search_type.value
@@ -190,6 +196,15 @@ def update(t: Optional[SearchType] = None, force: Optional[bool] = False):
 
 @api.get("/chat")
 def chat(q: Optional[str] = None):
+    if (
+        state.processor_config is None
+        or state.processor_config.conversation is None
+        or state.processor_config.conversation.openai_api_key is None
+    ):
+        raise HTTPException(
+            status_code=500, detail="Chat processor not configured. Configure OpenAI API key on server and restart it."
+        )
+
     # Initialize Variables
     api_key = state.processor_config.conversation.openai_api_key
     model = state.processor_config.conversation.model

From 36b17d4ae07d69e737ea08fea6ff3fa77b5ee04f Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Mon, 27 Mar 2023 03:44:03 +0700
Subject: [PATCH 11/18] Generalize the directory from config extraction elisp
 method

---
 src/interface/emacs/khoj.el | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el
index 604bdc0a..8b170ea8 100644
--- a/src/interface/emacs/khoj.el
+++ b/src/interface/emacs/khoj.el
@@ -316,16 +316,16 @@ for example), set this to the full interpreter path."
   (when (not (khoj--server-running?))
     (khoj--server-start)))
 
-(defun khoj--get-content-index-dir (config type)
-  "Extract directory containing index files of specified content TYPE from CONFIG.
+(defun khoj--get-directory-from-config (config keys &optional level)
+  "Extract directory under specified KEYS in CONFIG and trim it to LEVEL.
 CONFIG is json obtained from Khoj config API."
-  (--> config
-       (cdr (assoc 'content-type it))
-       (cdr (assoc type it))
-       (cdr (assoc 'embeddings-file it))
-       (split-string it "/")
-       (butlast it)
-       (string-join it "/")))
+  (let ((item config))
+    (dolist (key keys)
+      (setq item (cdr (assoc key item))))
+      (-> item
+          (split-string "/")
+          (butlast (or level nil))
+          (string-join "/"))))
 
 (defun khoj--server-configure ()
   "Configure the khoj server to index specified files."
@@ -338,7 +338,7 @@ CONFIG is json obtained from Khoj config API."
            (with-temp-buffer
              (url-insert-file-contents (format "%s/api/config/data/default" khoj-server-url))
              (ignore-error json-end-of-file (json-parse-buffer :object-type 'alist :array-type 'list :null-object json-null :false-object json-false))))
-         (default-index-dir (khoj--get-content-index-dir default-config 'org))
+         (default-index-dir (khoj--get-directory-from-config default-config '(content-type org embeddings-file)))
          (config (or current-config default-config)))
     (cond
      ;; If khoj backend is not configured yet
@@ -369,7 +369,7 @@ CONFIG is json obtained from Khoj config API."
 
      ;; Else if khoj is not configured to index org files
      ((not (equal (alist-get 'input-files (alist-get 'org (alist-get 'content-type config))) khoj-org-files-index))
-      (let* ((index-directory (khoj--get-content-index-dir config 'org))
+      (let* ((index-directory (khoj--get-directory-from-config config '(content-type org embeddings-file)))
              (new-content-type (alist-get 'content-type config)))
         (setq new-content-type (delq (assoc 'org new-content-type) new-content-type))
         (add-to-list 'new-content-type `(org . ((input-files . ,khoj-org-files-index)

From ae535a06eb76efcb0b7cffbee66d4cf54c52925f Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Mon, 27 Mar 2023 04:38:17 +0700
Subject: [PATCH 12/18] Configure Khoj chat using khoj.el by setting OpenAI API
 key in Emacs

---
 src/interface/emacs/khoj.el | 77 +++++++++++++++++++++++++++++--------
 src/khoj/utils/constants.py |  1 +
 2 files changed, 61 insertions(+), 17 deletions(-)

diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el
index 8b170ea8..d2d2bf91 100644
--- a/src/interface/emacs/khoj.el
+++ b/src/interface/emacs/khoj.el
@@ -218,10 +218,15 @@ for example), set this to the full interpreter path."
   :group 'khoj)
 
 (defcustom khoj-org-files-index (org-agenda-files t t)
-  "List of org-files to index on khoj server"
+  "List of org-files to index on khoj server."
   :group 'khoj
   :type '(repeat string))
 
+(defcustom khoj-openai-api-key nil
+  "OpenAI API key used to configure chat on khoj server."
+  :group 'khoj
+  :type 'string)
+
 (defvar khoj--server-process nil "Track Khoj server process.")
 (defvar khoj--server-name "*khoj-server*" "Track Khoj server buffer.")
 (defvar khoj--server-ready? nil "Track if khoj server is ready to receive API calls.")
@@ -328,7 +333,7 @@ CONFIG is json obtained from Khoj config API."
           (string-join "/"))))
 
 (defun khoj--server-configure ()
-  "Configure the khoj server to index specified files."
+  "Configure the the Khoj server for search and chat."
   (interactive)
   (let* ((current-config
           (with-temp-buffer
@@ -339,19 +344,21 @@ CONFIG is json obtained from Khoj config API."
              (url-insert-file-contents (format "%s/api/config/data/default" khoj-server-url))
              (ignore-error json-end-of-file (json-parse-buffer :object-type 'alist :array-type 'list :null-object json-null :false-object json-false))))
          (default-index-dir (khoj--get-directory-from-config default-config '(content-type org embeddings-file)))
+         (default-chat-dir (khoj--get-directory-from-config default-config '(processor conversation conversation-logfile)))
+         (default-model (or (alist-get 'model (alist-get 'conversation (alist-get 'processor default-config))) "text-davinci-003"))
          (config (or current-config default-config)))
+
+    ;; Configure content types
     (cond
      ;; If khoj backend is not configured yet
      ((not current-config)
       (setq config (delq (assoc 'content-type config) config))
-      (setq config (delq (assoc 'processor config) config))
-      (add-to-list 'config `(content-type . ((org . ((input-files . ,khoj-org-files-index)
-                                                     (input-filter . ,json-null)
-                                                     (compressed-jsonl . ,(format "%s/org.jsonl.gz" default-index-dir))
-                                                     (embeddings-file . ,(format "%s/org.pt" default-index-dir))
-                                                     (index-heading-entries . ,json-false))))))
-      (khoj--post-new-config config)
-      (message "khoj.el: ⚙️ Generated new khoj server configuration."))
+      (add-to-list 'config
+                   `(content-type . ((org . ((input-files . ,khoj-org-files-index)
+                                             (input-filter . ,json-null)
+                                             (compressed-jsonl . ,(format "%s/org.jsonl.gz" default-index-dir))
+                                             (embeddings-file . ,(format "%s/org.pt" default-index-dir))
+                                             (index-heading-entries . ,json-false)))))))
 
      ;; Else if khoj config has no org content config
      ((not (alist-get 'org (alist-get 'content-type config)))
@@ -363,11 +370,9 @@ CONFIG is json obtained from Khoj config API."
                                                 (embeddings-file . ,(format "%s/org.pt" default-index-dir))
                                                 (index-heading-entries . ,json-false))))
         (setq config (delq (assoc 'content-type config) config))
-        (add-to-list 'config `(content-type . ,new-content-type)))
-      (khoj--post-new-config config)
-      (message "Khoj: ⚙️ Added org content to index on khoj server."))
+        (add-to-list 'config `(content-type . ,new-content-type))))
 
-     ;; Else if khoj is not configured to index org files
+     ;; Else if khoj is not configured to index specified org files
      ((not (equal (alist-get 'input-files (alist-get 'org (alist-get 'content-type config))) khoj-org-files-index))
       (let* ((index-directory (khoj--get-directory-from-config config '(content-type org embeddings-file)))
              (new-content-type (alist-get 'content-type config)))
@@ -378,9 +383,47 @@ CONFIG is json obtained from Khoj config API."
                                                 (embeddings-file . ,(format "%s/org.pt" index-directory))
                                                 (index-heading-entries . ,json-false))))
         (setq config (delq (assoc 'content-type config) config))
-        (add-to-list 'config `(content-type . ,new-content-type)))
-      (khoj--post-new-config config)
-      (message "Khoj: ⚙️ Updated org content in index on khoj server")))))
+        (add-to-list 'config `(content-type . ,new-content-type)))))
+
+    ;; Configure processors
+    (cond
+     ((not khoj-openai-api-key)
+      (setq config (delq (assoc 'processor config) config)))
+
+     ((not current-config)
+      (setq config (delq (assoc 'processor config) config))
+      (add-to-list 'config
+                   `(processor . ((conversation . ((conversation-logfile . ,(format "%s/conversation.json" default-chat-dir))
+                                                   (model . ,default-model)
+                                                   (openai-api-key . ,khoj-openai-api-key)))))))
+
+     ((not (alist-get 'conversation (alist-get 'processor config)))
+       (let ((new-processor-type (alist-get 'processor config)))
+         (setq new-processor-type (delq (assoc 'conversation new-processor-type) new-processor-type))
+         (add-to-list 'new-processor-type `(conversation . ((conversation-logfile . ,(format "%s/conversation.json" default-chat-dir))
+                                                            (model . ,default-model)
+                                                            (openai-api-key . ,khoj-openai-api-key))))
+        (setq config (delq (assoc 'processor config) config))
+        (add-to-list 'config `(processor . ,new-processor-type))))
+
+     ;; Else if khoj is not configured with specified openai api key
+     ((not (equal (alist-get 'openai-api-key (alist-get 'conversation (alist-get 'processor config))) khoj-openai-api-key))
+      (let* ((chat-directory (khoj--get-directory-from-config config '(processor conversation conversation-logfile)))
+             (model-name (khoj--get-directory-from-config config '(processor conversation model)))
+             (new-processor-type (alist-get 'processor config)))
+        (setq new-processor-type (delq (assoc 'conversation new-processor-type) new-processor-type))
+        (add-to-list 'new-processor-type `(conversation . ((conversation-logfile . ,(format "%s/conversation.json" chat-directory))
+                                                           (model . ,model-name)
+                                                           (openai-api-key . ,khoj-openai-api-key))))
+        (setq config (delq (assoc 'processor config) config))
+        (add-to-list 'config `(processor . ,new-processor-type)))))
+
+     ;; Update server with latest configuration
+     (khoj--post-new-config config)
+     (cond ((not current-config)
+            (message "khoj.el: ⚙️ Generated new khoj server configuration."))
+           ((not (equal config current-config))
+            (message "Khoj: ⚙️ Updated khoj server configuration")))))
 
 
 ;; -----------------------------------------------
diff --git a/src/khoj/utils/constants.py b/src/khoj/utils/constants.py
index 40e01c5d..71722c1a 100644
--- a/src/khoj/utils/constants.py
+++ b/src/khoj/utils/constants.py
@@ -56,6 +56,7 @@ default_config = {
     "processor": {
         "conversation": {
             "openai-api-key": None,
+            "model": "text-davinci-003",
             "conversation-logfile": "~/.khoj/processor/conversation/conversation_logs.json",
         }
     },

From 8a9055f9183bd8a6729643c612c1d177c20b765e Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Mon, 27 Mar 2023 04:46:31 +0700
Subject: [PATCH 13/18] Restrict server messages show in echo area to main
 server files

---
 src/interface/emacs/khoj.el | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el
index d2d2bf91..8c37ecdf 100644
--- a/src/interface/emacs/khoj.el
+++ b/src/interface/emacs/khoj.el
@@ -274,12 +274,14 @@ for example), set this to the full interpreter path."
                             (progn
                               (setq khoj--server-ready? t)
                               (khoj--server-configure)))
-                           ((not khoj--server-ready?)
+                           ((and (not khoj--server-ready?)
+                                 (or (string-match "configure.py" msg)
+                                     (string-match "main.py" msg)
+                                     (string-match "api.py" msg)))
                             (dolist (line (split-string msg "\n"))
                               (message "khoj.el: %s" (nth 1 (split-string msg "  " t " *"))))))
                      ;; call default process filter to write output to process buffer
-                     (internal-default-process-filter process msg))
-           ))
+                     (internal-default-process-filter process msg))))
     (set-process-query-on-exit-flag khoj--server-process nil)
     (when (not khoj--server-process)
         (message "khoj.el: Failed to start Khoj server. Please start it manually by running `khoj' on terminal.\n%s" (buffer-string)))))

From 7243059507c3bbd6dc3563acf83c5e188ced7b84 Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Mon, 27 Mar 2023 05:52:32 +0700
Subject: [PATCH 14/18] Track index update asynchronously via moon phase
 progressbar in khoj.el

---
 src/interface/emacs/khoj.el | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el
index 8c37ecdf..60757c9d 100644
--- a/src/interface/emacs/khoj.el
+++ b/src/interface/emacs/khoj.el
@@ -230,6 +230,8 @@ for example), set this to the full interpreter path."
 (defvar khoj--server-process nil "Track Khoj server process.")
 (defvar khoj--server-name "*khoj-server*" "Track Khoj server buffer.")
 (defvar khoj--server-ready? nil "Track if khoj server is ready to receive API calls.")
+(defvar khoj--server-configured? t "Track if khoj server is configured to receive API calls.")
+(defvar khoj--progressbar '(🌑 🌘 🌗 🌖 🌕 🌔 🌓 🌒) "Track progress via moon phase animations.")
 
 (defun khoj--server-get-version ()
   "Return the khoj server version."
@@ -274,6 +276,15 @@ for example), set this to the full interpreter path."
                             (progn
                               (setq khoj--server-ready? t)
                               (khoj--server-configure)))
+                           ((string-match "Batches:  " msg)
+                            (when (string-match "\\([0-9]+\\.[0-9]+\\|\\([0-9]+\\)\\)%?" msg)
+                              (message "khoj.el: %s updating index %s"
+                                       (nth (% (string-to-number (match-string 1 msg)) (length khoj--progressbar)) khoj--progressbar)
+                                       (match-string 0 msg)))
+                            (setq khoj--server-configured? nil))
+                           ((and (not khoj--server-configured?)
+                                 (string-match "Processor reconfigured via API" msg))
+                            (setq khoj--server-configured? t))
                            ((and (not khoj--server-ready?)
                                  (or (string-match "configure.py" msg)
                                      (string-match "main.py" msg)
@@ -976,7 +987,7 @@ Paragraph only starts at first text after blank line."
              (y-or-n-p "Could not connect to Khoj server. Should I install and start it?"))
     (khoj--server-setup))
   (while (not khoj--server-ready?)
-    (sleep-for 0.5))
+    (sit-for 0.5))
   (khoj--server-configure)
   (khoj--menu))
 

From 526a927bced8b04bc6e67694bad8cd46d9d494ed Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Mon, 27 Mar 2023 16:42:38 +0700
Subject: [PATCH 15/18] Fix org entry extraction test, variable prefixed with
 khoj in khoj.el

Discovered via failing build and test workflows on Github
---
 src/interface/emacs/khoj.el             | 20 ++++++++++----------
 src/interface/emacs/tests/khoj-tests.el |  3 +--
 2 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el
index 60757c9d..7835b294 100644
--- a/src/interface/emacs/khoj.el
+++ b/src/interface/emacs/khoj.el
@@ -65,7 +65,7 @@
   :group 'khoj
   :type 'string)
 
-(defcustom is-khoj-server-local t
+(defcustom khoj-server-is-local t
   "Is Khoj server on local machine?."
   :group 'khoj
   :type 'boolean)
@@ -185,13 +185,13 @@ Use `which-key` if available, else display simple message in echo area"
       (executable-find "khoj.exe")
       "khoj")
   "Command to interact with Khoj server."
-  :group 'khoj
-  :type 'string)
+  :type 'string
+  :group 'khoj)
 
 (defcustom khoj-server-args '("--no-gui")
   "Arguments to pass to Khoj server on startup."
-  :group 'khoj
-  :type '(repeat string))
+  :type '(repeat string)
+  :group 'khoj)
 
 (defcustom khoj-server-python-command
   (if (equal system-type 'windows-nt)
@@ -219,13 +219,13 @@ for example), set this to the full interpreter path."
 
 (defcustom khoj-org-files-index (org-agenda-files t t)
   "List of org-files to index on khoj server."
-  :group 'khoj
-  :type '(repeat string))
+  :type '(repeat string)
+  :group 'khoj)
 
 (defcustom khoj-openai-api-key nil
   "OpenAI API key used to configure chat on khoj server."
-  :group 'khoj
-  :type 'string)
+  :type 'string
+  :group 'khoj)
 
 (defvar khoj--server-process nil "Track Khoj server process.")
 (defvar khoj--server-name "*khoj-server*" "Track Khoj server buffer.")
@@ -326,7 +326,7 @@ for example), set this to the full interpreter path."
   "Install and start the khoj server, if required."
   (interactive)
   ;; Install khoj server, if not available but expected on local machine
-  (when (and is-khoj-server-local
+  (when (and khoj-server-is-local
              (or (not (executable-find khoj-server-command))
                  (not (khoj--server-get-version))))
       (khoj--server-install-upgrade))
diff --git a/src/interface/emacs/tests/khoj-tests.el b/src/interface/emacs/tests/khoj-tests.el
index 577f2c5c..83985b67 100644
--- a/src/interface/emacs/tests/khoj-tests.el
+++ b/src/interface/emacs/tests/khoj-tests.el
@@ -109,8 +109,7 @@ Penance to Immortality\n\
 ** Act\n\
 \n\
 Rule everything\n\
-\n\
-#+STARTUP: showall hidestars inlineimages"))))
+\n"))))
 
 
 (ert-deftest khoj-tests--extract-entries-as-ledger ()

From 6e8a40906df68b05008afbed54ca54b6c7655487 Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Mon, 27 Mar 2023 17:53:08 +0700
Subject: [PATCH 16/18] Allow disabling automatic server setup. Fix server
 start vs ready logic

- khoj-auto-setup controls whether to automatically check for and
  setup khoj server from within Emacs
- extract install, start, configure sequence into public, interactive
  method. Allows calling khoj-setup during package load via init.el

- Fix: Do not attempt to configure or wait for server ready if
  user has said no to auto-setup request
- Fix logic to mark server started vs ready
  - Previously the started/running vs ready variables defs were getting
    intertwined
  - Server started indicates server bootup has been triggered
  - Server ready indicates server API ready to accept requests
---
 src/interface/emacs/khoj.el | 67 +++++++++++++++++++++++++------------
 1 file changed, 46 insertions(+), 21 deletions(-)

diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el
index 7835b294..54aa6c06 100644
--- a/src/interface/emacs/khoj.el
+++ b/src/interface/emacs/khoj.el
@@ -227,6 +227,12 @@ for example), set this to the full interpreter path."
   :type 'string
   :group 'khoj)
 
+(defcustom khoj-auto-setup t
+  "Automate install, configure and starting khoj server.
+Auto invokes setup steps on calling main entrypoint."
+  :type 'string
+  :group 'khoj)
+
 (defvar khoj--server-process nil "Track Khoj server process.")
 (defvar khoj--server-name "*khoj-server*" "Track Khoj server buffer.")
 (defvar khoj--server-ready? nil "Track if khoj server is ready to receive API calls.")
@@ -297,21 +303,24 @@ for example), set this to the full interpreter path."
     (when (not khoj--server-process)
         (message "khoj.el: Failed to start Khoj server. Please start it manually by running `khoj' on terminal.\n%s" (buffer-string)))))
 
-(defun khoj--server-running? ()
-  "Check if the khoj server is running."
-  (when (or
-       ;; check for when server process handled from within emacs
-       (and khoj--server-process
-            (not (null (process-live-p khoj--server-process))))
-       ;; else general check via ping to khoj-server-url
-       (ignore-errors
-         (not (null (url-retrieve-synchronously (format "%s/api/config/data/default" khoj-server-url))))))
-      (setq khoj--server-ready? t)))
+(defun khoj--server-started? ()
+  "Check if the khoj server has been started."
+  ;; check for when server process handled from within emacs
+  (if (and khoj--server-process
+           (not (null (process-live-p khoj--server-process))))
+      t
+    ;; else general check via ping to khoj-server-url
+    (if (ignore-errors
+          (not (null (url-retrieve-synchronously (format "%s/api/config/data/default" khoj-server-url)))))
+        ;; Successful ping to non-emacs khoj server indicates it is started and ready.
+        ;; So update ready state tracker variable (and implicitly return true for started)
+        (setq khoj--server-ready? t)
+      nil)))
 
 (defun khoj--server-stop ()
   "Stop the khoj server."
   (interactive)
-  (when (khoj--server-running?)
+  (when (khoj--server-started?)
     (message "khoj.el: Stopping server...")
     (kill-process khoj--server-process)
     (message "khoj.el: Stopped server.")))
@@ -330,8 +339,8 @@ for example), set this to the full interpreter path."
              (or (not (executable-find khoj-server-command))
                  (not (khoj--server-get-version))))
       (khoj--server-install-upgrade))
-  ;; Start khoj server if not running at expected URL
-  (when (not (khoj--server-running?))
+  ;; Start khoj server if not already started
+  (when (not (khoj--server-started?))
     (khoj--server-start)))
 
 (defun khoj--get-directory-from-config (config keys &optional level)
@@ -438,6 +447,27 @@ CONFIG is json obtained from Khoj config API."
            ((not (equal config current-config))
             (message "Khoj: ⚙️ Updated khoj server configuration")))))
 
+(defun khoj-setup (&optional interact)
+  "Install, start and configure Khoj server."
+  (interactive "p")
+  ;; Setup khoj server if not running
+  (let* ((not-started (not (khoj--server-started?)))
+         (permitted (if (and not-started interact)
+                        (y-or-n-p "Could not connect to Khoj server. Should I install, start and configure it for you?")
+                      t)))
+    ;; Install, start server if user permitted and server not ready
+    (when (and permitted not-started)
+      (khoj--server-setup))
+
+    ;; Server can be started but not ready (to use/configure)
+    ;; Wait until server is ready if setup was permitted
+    (while (and permitted (not khoj--server-ready?))
+      (sit-for 0.5))
+
+    ;; Configure server once server ready if user permitted
+    (when permitted
+      (khoj--server-configure))))
+
 
 ;; -----------------------------------------------
 ;; Extract and Render Entries of each Content Type
@@ -980,15 +1010,10 @@ Paragraph only starts at first text after blank line."
 
 ;;;###autoload
 (defun khoj ()
-  "Natural, Incremental Search for your personal notes, transactions and images."
+  "Provide natural, search assistance for your notes, transactions and images."
   (interactive)
-  ;; Setup khoj server if not running before executing user commands
-  (when (and (not (khoj--server-running?))
-             (y-or-n-p "Could not connect to Khoj server. Should I install and start it?"))
-    (khoj--server-setup))
-  (while (not khoj--server-ready?)
-    (sit-for 0.5))
-  (khoj--server-configure)
+  (when khoj-auto-setup
+    (khoj-setup t))
   (khoj--menu))
 
 (provide 'khoj)

From 5c2327ee4f174bb6d78c83c418221e933fb9c63f Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Mon, 27 Mar 2023 18:30:53 +0700
Subject: [PATCH 17/18] Configure org directories to index from khoj.el

Converts paths to glob style regexes that will index all org files
recursively under the specified list of path

Should help setup for org-roam users from khoj.el
---
 src/interface/emacs/khoj.el | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el
index 54aa6c06..4b26a328 100644
--- a/src/interface/emacs/khoj.el
+++ b/src/interface/emacs/khoj.el
@@ -217,11 +217,16 @@ for example), set this to the full interpreter path."
           (member val '("python" "python3" "pythonw" "py")))
   :group 'khoj)
 
-(defcustom khoj-org-files-index (org-agenda-files t t)
+(defcustom khoj-org-files (org-agenda-files t t)
   "List of org-files to index on khoj server."
   :type '(repeat string)
   :group 'khoj)
 
+(defcustom khoj-org-directories nil
+  "List of directories with org-mode files to index on khoj server."
+  :type '(repeat string)
+  :group 'khoj)
+
 (defcustom khoj-openai-api-key nil
   "OpenAI API key used to configure chat on khoj server."
   :type 'string
@@ -357,7 +362,8 @@ CONFIG is json obtained from Khoj config API."
 (defun khoj--server-configure ()
   "Configure the the Khoj server for search and chat."
   (interactive)
-  (let* ((current-config
+  (let* ((org-directory-regexes (or (mapcar (lambda (dir) (format "%s/**/*.org" dir)) khoj-org-directories) json-null))
+         (current-config
           (with-temp-buffer
             (url-insert-file-contents (format "%s/api/config/data" khoj-server-url))
             (ignore-error json-end-of-file (json-parse-buffer :object-type 'alist :array-type 'list :null-object json-null :false-object json-false))))
@@ -376,8 +382,8 @@ CONFIG is json obtained from Khoj config API."
      ((not current-config)
       (setq config (delq (assoc 'content-type config) config))
       (add-to-list 'config
-                   `(content-type . ((org . ((input-files . ,khoj-org-files-index)
-                                             (input-filter . ,json-null)
+                   `(content-type . ((org . ((input-files . ,khoj-org-files)
+                                             (input-filter . ,org-directory-regexes)
                                              (compressed-jsonl . ,(format "%s/org.jsonl.gz" default-index-dir))
                                              (embeddings-file . ,(format "%s/org.pt" default-index-dir))
                                              (index-heading-entries . ,json-false)))))))
@@ -386,8 +392,8 @@ CONFIG is json obtained from Khoj config API."
      ((not (alist-get 'org (alist-get 'content-type config)))
       (let ((new-content-type (alist-get 'content-type config)))
         (setq new-content-type (delq (assoc 'org new-content-type) new-content-type))
-        (add-to-list 'new-content-type `(org . ((input-files . ,khoj-org-files-index)
-                                                (input-filter . ,json-null)
+        (add-to-list 'new-content-type `(org . ((input-files . ,khoj-org-files)
+                                                (input-filter . ,org-directory-regexes)
                                                 (compressed-jsonl . ,(format "%s/org.jsonl.gz" default-index-dir))
                                                 (embeddings-file . ,(format "%s/org.pt" default-index-dir))
                                                 (index-heading-entries . ,json-false))))
@@ -395,12 +401,13 @@ CONFIG is json obtained from Khoj config API."
         (add-to-list 'config `(content-type . ,new-content-type))))
 
      ;; Else if khoj is not configured to index specified org files
-     ((not (equal (alist-get 'input-files (alist-get 'org (alist-get 'content-type config))) khoj-org-files-index))
+     ((not (and (equal (alist-get 'input-files (alist-get 'org (alist-get 'content-type config))) khoj-org-files)
+                (equal (alist-get 'input-filter (alist-get 'org (alist-get 'content-type config))) org-directory-regexes)))
       (let* ((index-directory (khoj--get-directory-from-config config '(content-type org embeddings-file)))
              (new-content-type (alist-get 'content-type config)))
         (setq new-content-type (delq (assoc 'org new-content-type) new-content-type))
-        (add-to-list 'new-content-type `(org . ((input-files . ,khoj-org-files-index)
-                                                (input-filter . ,json-null)
+        (add-to-list 'new-content-type `(org . ((input-files . ,khoj-org-files)
+                                                (input-filter . ,org-directory-regexes)
                                                 (compressed-jsonl . ,(format "%s/org.jsonl.gz" index-directory))
                                                 (embeddings-file . ,(format "%s/org.pt" index-directory))
                                                 (index-heading-entries . ,json-false))))

From 83a7ccd729d79079eee23da84c1372761ebfb856 Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Mon, 27 Mar 2023 18:33:09 +0700
Subject: [PATCH 18/18] Fix docstrings and method ordering in khoj.el

---
 src/interface/emacs/khoj.el | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el
index 4b26a328..ddcd50c6 100644
--- a/src/interface/emacs/khoj.el
+++ b/src/interface/emacs/khoj.el
@@ -233,13 +233,13 @@ for example), set this to the full interpreter path."
   :group 'khoj)
 
 (defcustom khoj-auto-setup t
-  "Automate install, configure and starting khoj server.
+  "Automate install, configure and start of khoj server.
 Auto invokes setup steps on calling main entrypoint."
   :type 'string
   :group 'khoj)
 
-(defvar khoj--server-process nil "Track Khoj server process.")
-(defvar khoj--server-name "*khoj-server*" "Track Khoj server buffer.")
+(defvar khoj--server-process nil "Track khoj server process.")
+(defvar khoj--server-name "*khoj-server*" "Track khoj server buffer.")
 (defvar khoj--server-ready? nil "Track if khoj server is ready to receive API calls.")
 (defvar khoj--server-configured? t "Track if khoj server is configured to receive API calls.")
 (defvar khoj--progressbar '(🌑 🌘 🌗 🌖 🌕 🌔 🌓 🌒) "Track progress via moon phase animations.")
@@ -322,6 +322,12 @@ Auto invokes setup steps on calling main entrypoint."
         (setq khoj--server-ready? t)
       nil)))
 
+(defun khoj--server-restart ()
+  "Restart the khoj server."
+  (interactive)
+  (khoj--server-stop)
+  (khoj--server-start))
+
 (defun khoj--server-stop ()
   "Stop the khoj server."
   (interactive)
@@ -330,12 +336,6 @@ Auto invokes setup steps on calling main entrypoint."
     (kill-process khoj--server-process)
     (message "khoj.el: Stopped server.")))
 
-(defun khoj--server-restart ()
-  "Restart the khoj server."
-  (interactive)
-  (khoj--server-stop)
-  (khoj--server-start))
-
 (defun khoj--server-setup ()
   "Install and start the khoj server, if required."
   (interactive)