klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-09 05:39:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Clean and fix the content indexing code in the Emacs client

Browse Source 
- Pass payloads as unibyte. This was causing the request to fail for
  files with unicode characters
- Suppress messages with file content in on index updates
- Fix rendering response from server on index update API call
- Extract code to populate body of index update HTTP request with files
This commit is contained in:
Debanjum Singh Solanky
2023-10-13 18:00:37 -07:00
parent bea196aa30
commit b669aa2395
1 changed files with 31 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
src/interface/emacs/khoj.el
View File

@@ -535,17 +535,33 @@ CONFIG is json obtained from Khoj config API."
;; ------------------- ;; -------------------
(defun khoj--server-index-files (&optional file-paths) (defun khoj--server-index-files (&optional file-paths)
"Send files to the Khoj server to index for search and chat." "Send files at `FILE-PATHS' to the Khoj server to index for search and chat."
(interactive) (interactive)
(let ((boundary (format "-------------------------%d" (random (expt 10 10)))) (let ((boundary (format "-------------------------%d" (random (expt 10 10))))
(files-to-index (or file-paths (files-to-index (or file-paths
(append (mapcan (lambda (dir) (directory-files-recursively dir "\\.org$")) khoj-org-directories) khoj-org-files)))) (append (mapcan (lambda (dir) (directory-files-recursively dir "\\.org$")) khoj-org-directories) khoj-org-files)))
(let* ((url-request-method "POST") (inhibit-message t)
(message-log-max nil))
(let ((url-request-method "POST")
(url-request-data (khoj--render-files-as-request-body files-to-index boundary))
(url-request-extra-headers `(("content-type" . ,(format "multipart/form-data; boundary=%s" boundary)) (url-request-extra-headers `(("content-type" . ,(format "multipart/form-data; boundary=%s" boundary))
("x-api-key" . ,khoj-server-api-key))) ("x-api-key" . ,khoj-server-api-key))))
;; add files to index as form data (with-current-buffer
(url-request-data (with-temp-buffer (url-retrieve (format "%s/api/v1/indexer/batch" khoj-server-url)
(set-buffer-multibyte t) ;; render response from indexing API endpoint on server
(lambda (status)
(if (not status)
(message "khoj.el: Updated Content Index")
(with-current-buffer (current-buffer)
(goto-char "\n\n")
(message "khoj.el: Failed to update Content Index. Status: %s. Response: %s" status (string-trim (buffer-substring-no-properties (point) (point-max)))))))
nil t t)))))
(defun khoj--render-files-as-request-body (files-to-index boundary)
"Render `FILES-TO-INDEX' as multi-part form body using `BOUNDARY'.
This is sent to Khoj server as a POST request."
(with-temp-buffer
(set-buffer-multibyte nil)
(insert "\n") (insert "\n")
(dolist (file-to-index files-to-index) (dolist (file-to-index files-to-index)
(insert (format "--%s\r\n" boundary)) (insert (format "--%s\r\n" boundary))
@@ -556,17 +572,9 @@ CONFIG is json obtained from Khoj config API."
(buffer-string))) (buffer-string)))
(insert "\r\n")) (insert "\r\n"))
(insert (format "--%s--\r\n" boundary)) (insert (format "--%s--\r\n" boundary))
(buffer-string)))) (buffer-string)))
(with-current-buffer
(url-retrieve (format "%s/api/v1/indexer/batch" khoj-server-url)
;; render response from indexing API endpoint on server
(lambda (status)
(with-current-buffer (current-buffer)
(goto-char url-http-end-of-headers)
(message "khoj.el: Update Content Index. Status: %s. response: %s" status (string-trim (buffer-substring-no-properties (point) (point-max))))))
nil t t)))))
;; Cancel any running indexing timer ;; Cancel any running indexing timer, first
(when khoj--index-timer (when khoj--index-timer
(cancel-timer khoj--index-timer)) (cancel-timer khoj--index-timer))
;; Send files to index on server every `khoj-index-interval' seconds ;; Send files to index on server every `khoj-index-interval' seconds