From d23f2849d4064ecb6b388e6caa80c246350b2fae Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Sun, 16 Jun 2024 13:13:28 +0530 Subject: [PATCH 1/8] Update help message to only show the smaller set of new keybindings --- src/interface/emacs/khoj.el | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el index 616d88ce..ffd64bda 100644 --- a/src/interface/emacs/khoj.el +++ b/src/interface/emacs/khoj.el @@ -163,15 +163,7 @@ NO-PAGING FILTER)) " Set Content Type -------------------------\n" - ("C-c RET | improve sort \n") - (when (member 'markdown enabled-content-types) - "C-x m | markdown\n") - (when (member 'org enabled-content-types) - "C-x o | org-mode\n") - (when (member 'image enabled-content-types) - "C-x i | image\n") - (when (member 'pdf enabled-content-types) - "C-x p | pdf\n")))) + ("C-c RET | improve sort \n")))) (defvar khoj--rerank nil "Track when re-rank of results triggered.") (defvar khoj--reference-count 0 "Track number of references currently in chat bufffer.") From d042e073ccec846f3340364f6e937889e700493b Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Sun, 16 Jun 2024 16:17:39 +0530 Subject: [PATCH 2/8] Pass absolute path of file to index from khoj.el emacs client --- src/interface/emacs/khoj.el | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el index ffd64bda..f28b7255 100644 --- a/src/interface/emacs/khoj.el +++ b/src/interface/emacs/khoj.el @@ -399,8 +399,10 @@ Auto invokes setup steps on calling main entrypoint." ;; This is a temporary change. `khoj-org-directories', `khoj-org-files' are deprecated. They will be removed in a future release (content-directories (or khoj-index-directories khoj-org-directories)) (content-files (or khoj-index-files khoj-org-files)) - (files-to-index (or file-paths - (append (mapcan (lambda (dir) (directory-files-recursively dir "\\.\\(org\\|md\\|markdown\\|pdf\\|txt\\|rst\\|xml\\|htm\\|html\\)$")) content-directories) content-files))) + (files-to-index (mapcar + #'expand-file-name + (or file-paths + (append (mapcan (lambda (dir) (directory-files-recursively dir "\\.\\(org\\|md\\|markdown\\|pdf\\|txt\\|rst\\|xml\\|htm\\|html\\)$")) content-directories) content-files)))) (type-query (if (or (equal content-type "all") (not content-type)) "" (format "t=%s" content-type))) (delete-files (-difference khoj--indexed-files files-to-index)) (inhibit-message t) From 2b12a5514e72a0fe5607308a2053b20403b70276 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Mon, 17 Jun 2024 19:37:40 +0530 Subject: [PATCH 3/8] Find similar notes to current note at cursor automatically in background - Call find similar on current element if point has moved to new element - Delete the first result from find-similar search results as that'll be the current note (which is trivially most similar to itself) - Determine find-similar based text formating at the rendering layer rather than at the top level find-similar func --- src/interface/emacs/khoj.el | 106 ++++++++++++++++++------ src/interface/emacs/tests/khoj-tests.el | 4 +- 2 files changed, 83 insertions(+), 27 deletions(-) diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el index f28b7255..187d0f24 100644 --- a/src/interface/emacs/khoj.el +++ b/src/interface/emacs/khoj.el @@ -93,6 +93,11 @@ :group 'khoj :type 'number) +(defcustom khoj-auto-find-similar t + "Should try find similar notes automatically." + :group 'khoj + :type 'boolean) + (defcustom khoj-api-key nil "API Key to your Khoj. Default at https://app.khoj.dev/config#clients." :group 'khoj @@ -186,6 +191,9 @@ Use `which-key` if available, else display simple message in echo area" nil t t)) (message "%s" (khoj--keybindings-info-message)))) +(defvar khoj--last-heading-pos nil + "The last heading position point was in.") + ;; ---------------- ;; Khoj Setup @@ -495,11 +503,17 @@ Use `BOUNDARY' to separate files. This is sent to Khoj server as a POST request. ;; ------------------------------------------- ;; Render Response from Khoj server for Emacs ;; ------------------------------------------- +(defun construct-find-similar-title (query) + "Construct title for find-similar query." + (format "Similar to: %s" + (replace-regexp-in-string "^[#\\*]* " "" (car (split-string query "\n"))))) -(defun khoj--extract-entries-as-markdown (json-response query) +(defun khoj--extract-entries-as-markdown (json-response query is-find-similar) "Convert JSON-RESPONSE, QUERY from API to markdown entries." (thread-last json-response + ;; filter our first result if is find similar as it'll be the current entry at point + ((lambda (response) (if is-find-similar (seq-drop response 1) response))) ;; Extract and render each markdown entry from response (mapcar (lambda (json-response-item) (thread-last @@ -510,14 +524,17 @@ Use `BOUNDARY' to separate files. This is sent to Khoj server as a POST request. ;; Standardize results to 2nd level heading for consistent rendering (replace-regexp-in-string "^\#+" "##")))) ;; Render entries into markdown formatted string with query set as as top level heading - (format "# %s\n%s" query) + (format "# %s\n%s" (if is-find-similar (construct-find-similar-title query) query)) ;; remove leading (, ) or SPC from extracted entries string (replace-regexp-in-string "^[\(\) ]" ""))) -(defun khoj--extract-entries-as-org (json-response query) - "Convert JSON-RESPONSE, QUERY from API to `org-mode' entries." +(defun khoj--extract-entries-as-org (json-response query is-find-similar) + "Convert JSON-RESPONSE, QUERY from API to `org-mode' entries. +Use IS-FIND-SIMILAR to determine if results should be ." (thread-last json-response + ;; filter our first result if is find similar as it'll be the current entry at point + ((lambda (response) (if is-find-similar (seq-drop response 1) response))) ;; Extract and render each org-mode entry from response (mapcar (lambda (json-response-item) (thread-last @@ -528,14 +545,16 @@ Use `BOUNDARY' to separate files. This is sent to Khoj server as a POST request. ;; Standardize results to 2nd level heading for consistent rendering (replace-regexp-in-string "^\*+" "**")))) ;; Render entries into org formatted string with query set as as top level heading - (format "* %s\n%s\n" query) + (format "* %s\n%s\n" (if is-find-similar (construct-find-similar-title query) query)) ;; remove leading (, ) or SPC from extracted entries string (replace-regexp-in-string "^[\(\) ]" ""))) -(defun khoj--extract-entries-as-pdf (json-response query) +(defun khoj--extract-entries-as-pdf (json-response query is-find-similar) "Convert QUERY, JSON-RESPONSE from API with PDF results to `org-mode' entries." (thread-last json-response + ;; filter our first result if is find similar as it'll be the current entry at point + ((lambda (response) (if is-find-similar (seq-drop response 1) response))) ;; Extract and render each pdf entry from response (mapcar (lambda (json-response-item) (thread-last @@ -544,7 +563,7 @@ Use `BOUNDARY' to separate files. This is sent to Khoj server as a POST request. ;; Format pdf entry as a org entry string (format "** %s\n\n")))) ;; Render entries into org formatted string with query set as as top level heading - (format "* %s\n%s\n" query) + (format "* %s\n%s\n" (if is-find-similar (construct-find-similar-title query) query)) ;; remove leading (, ) or SPC from extracted entries string (replace-regexp-in-string "^[\(\) ]" ""))) @@ -576,9 +595,11 @@ Use `BOUNDARY' to separate files. This is sent to Khoj server as a POST request. ;; remove trailing (, ) or SPC from extracted entries string (replace-regexp-in-string "[\(\) ]$" "")))) -(defun khoj--extract-entries (json-response query) +(defun khoj--extract-entries (json-response query is-find-similar) "Convert JSON-RESPONSE, QUERY from API to text entries." (thread-last json-response + ;; filter our first result if is find similar as it'll be the current entry at point + ((lambda (response) (if is-find-similar (seq-drop response 1) response))) ;; extract and render entries from API response (mapcar (lambda (json-response-item) (thread-last @@ -592,7 +613,7 @@ Use `BOUNDARY' to separate files. This is sent to Khoj server as a POST request. ;; Format entries as org entry string (format "** %s")))) ;; Set query as heading in rendered results buffer - (format "* %s\n%s\n" query) + (format "* %s\n%s\n" (if is-find-similar (construct-find-similar-title query) query)) ;; remove leading (, ) or SPC from extracted entries string (replace-regexp-in-string "^[\(\) ]" "") ;; remove trailing (, ) or SPC from extracted entries string @@ -656,30 +677,30 @@ Return json parsed response as alist." "Get content types enabled for search from API." (khoj--call-api "/api/config/types" "GET" nil `(lambda (item) (mapcar #'intern item)))) -(defun khoj--query-search-api-and-render-results (query content-type buffer-name &optional rerank title) +(defun khoj--query-search-api-and-render-results (query content-type buffer-name &optional rerank is-find-similar) "Query Khoj Search API with QUERY, CONTENT-TYPE and (optional) RERANK as query params Render results in BUFFER-NAME using search results, CONTENT-TYPE and (optional) TITLE." - (let ((title (or title query)) - (rerank (or rerank "false")) + (let ((rerank (or rerank "false")) (params `((q ,query) (t ,content-type) (r ,rerank) (n ,khoj-results-count))) (path "/api/search")) (khoj--call-api-async path "GET" params 'khoj--render-search-results - content-type title buffer-name))) + content-type query buffer-name is-find-similar))) -(defun khoj--render-search-results (json-response content-type query buffer-name) +(defun khoj--render-search-results (json-response content-type query buffer-name &optional is-find-similar) ;; render json response into formatted entries (with-current-buffer buffer-name - (let ((inhibit-read-only t)) + (let ((is-find-similar (or is-find-similar nil)) + (inhibit-read-only t)) (erase-buffer) (insert - (cond ((equal content-type "org") (khoj--extract-entries-as-org json-response query)) - ((equal content-type "markdown") (khoj--extract-entries-as-markdown json-response query)) - ((equal content-type "pdf") (khoj--extract-entries-as-pdf json-response query)) - ((equal content-type "image") (khoj--extract-entries-as-images json-response query)) - (t (khoj--extract-entries json-response query)))) + (cond ((equal content-type "org") (khoj--extract-entries-as-org json-response query is-find-similar)) + ((equal content-type "markdown") (khoj--extract-entries-as-markdown json-response query is-find-similar)) + ((equal content-type "pdf") (khoj--extract-entries-as-pdf json-response query is-find-similar)) + ((equal content-type "image") (khoj--extract-entries-as-images json-response)) + (t (khoj--extract-entries json-response query is-find-similar)))) (cond ((or (equal content-type "all") (equal content-type "pdf") (equal content-type "org")) @@ -1081,6 +1102,16 @@ RECEIVE-DATE is the message receive date." ;; Similar Search ;; -------------- +(defun khoj--get-current-outline-entry-pos () + "Get heading position of current outline section." + ;; get heading position of current outline entry + (cond + ;; when at heading of entry + ((looking-at outline-regexp) + (point)) + ;; when within entry + (t (save-excursion (outline-previous-heading) (point))))) + (defun khoj--get-current-outline-entry-text () "Get text under current outline section." (string-trim @@ -1124,10 +1155,6 @@ Paragraph only starts at first text after blank line." ;; get paragraph, if in text mode (t (khoj--get-current-paragraph-text)))) - ;; extract heading to show in result buffer from query - (query-title - (format "Similar to: %s" - (replace-regexp-in-string "^[#\\*]* " "" (car (split-string query "\n"))))) (buffer-name (get-buffer-create khoj--search-buffer-name))) (progn (khoj--query-search-api-and-render-results @@ -1135,10 +1162,39 @@ Paragraph only starts at first text after blank line." content-type buffer-name rerank - query-title) + t) (khoj--open-side-pane buffer-name) (goto-char (point-min))))) +(defun khoj--auto-find-similar () + "Call find similar on current element, if point has moved to a new element." + ;; Call find similar + (when (and (derived-mode-p 'org-mode) + (not (null (org-element-at-point))) + (not (string= (buffer-name (current-buffer)) khoj--search-buffer-name)) + (not (null (get-buffer-window khoj--search-buffer-name)))) + (let ((current-heading-pos (khoj--get-current-outline-entry-pos))) + (unless (eq current-heading-pos khoj--last-heading-pos) + (progn + (setq khoj--last-heading-pos current-heading-pos) + (save-excursion + (khoj--find-similar))))))) + +(defun khoj--setup-auto-find-similar () + "Setup automatic call to find similar to current element." + (if khoj-auto-find-similar + (add-hook 'post-command-hook 'khoj--auto-find-similar) + (remove-hook 'post-command-hook 'khoj--auto-find-similar))) + +(defun khoj-toggle-auto-find-similar () + "Toggle automatic call to find similar to current element." + (interactive) + (setq khoj-auto-find-similar (not khoj-auto-find-similar)) + (khoj--setup-auto-find-similar) + (if khoj-auto-find-similar + (message "Auto find similar enabled") + (message "Auto find similar disabled"))) + ;; --------- ;; Khoj Menu diff --git a/src/interface/emacs/tests/khoj-tests.el b/src/interface/emacs/tests/khoj-tests.el index c1dfe90e..aa12a179 100644 --- a/src/interface/emacs/tests/khoj-tests.el +++ b/src/interface/emacs/tests/khoj-tests.el @@ -64,7 +64,7 @@ "))) (should (equal - (khoj--extract-entries-as-markdown json-response-from-khoj-backend user-query) + (khoj--extract-entries-as-markdown json-response-from-khoj-backend user-query nil) "\ # Become God\n\ ## Upgrade\n\ @@ -100,7 +100,7 @@ Rule everything\n\n")))) "))) (should (equal - (khoj--extract-entries-as-org json-response-from-khoj-backend user-query) + (khoj--extract-entries-as-org json-response-from-khoj-backend user-query nil) "\ * Become God\n\ ** Upgrade\n\ From afe91a2633ac412a3707d921f4a1370379717fe2 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Mon, 17 Jun 2024 19:42:13 +0530 Subject: [PATCH 4/8] Only show headings of search result and increase total count returned Previously it would show complete result body this would make the result width variable and hard to track all the returned results Showing just heading makes it easier to track --- src/interface/emacs/khoj.el | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el index 187d0f24..6681cad9 100644 --- a/src/interface/emacs/khoj.el +++ b/src/interface/emacs/khoj.el @@ -83,7 +83,7 @@ :group 'khoj :type 'integer) -(defcustom khoj-results-count 5 +(defcustom khoj-results-count 8 "Number of results to show in search and use for chat responses." :group 'khoj :type 'integer) @@ -707,10 +707,9 @@ Render results in BUFFER-NAME using search results, CONTENT-TYPE and (optional) (progn (visual-line-mode) (org-mode) (setq-local - org-startup-folded "showall" org-hide-leading-stars t org-startup-with-inline-images t) - (org-set-startup-visibility))) + (org-cycle-content 2))) ((equal content-type "markdown") (progn (markdown-mode) (visual-line-mode))) ((equal content-type "image") (progn (shr-render-region (point-min) (point-max)) From 0afe66ac399cb01182ef4ef31878d6d29791b894 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Thu, 20 Jun 2024 01:44:43 +0530 Subject: [PATCH 5/8] Restore cursor to original window after opening Khoj side pane Previously the cursor would move to the Khoj side pane on opening it. This would break user's flow, especially when find similar triggers automatically New behavior maintains smoother update of auto find similar without disrupting user browsing --- src/interface/emacs/khoj.el | 52 ++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el index 6681cad9..b641dde3 100644 --- a/src/interface/emacs/khoj.el +++ b/src/interface/emacs/khoj.el @@ -738,28 +738,30 @@ Render results in BUFFER-NAME using search results, CONTENT-TYPE and (optional) (defun khoj--open-side-pane (buffer-name) "Open Khoj BUFFER-NAME in right side pane." - (if (get-buffer-window-list buffer-name) - ;; if window is already open, switch to it - (progn - (select-window (get-buffer-window buffer-name)) - (switch-to-buffer buffer-name)) - ;; else if window is not open, open it as a right-side window pane - (let ((bottomright-window (some-window (lambda (window) (and (window-at-side-p window 'right) (window-at-side-p window 'bottom)))))) - (progn - ;; Select the right-most window - (select-window bottomright-window) - ;; if bottom-right window is not a vertical pane, split it vertically, else use the existing bottom-right vertical window - (let ((khoj-window (if (window-at-side-p bottomright-window 'left) - (split-window-right) - bottomright-window))) - ;; Set the buffer in the khoj window - (set-window-buffer khoj-window buffer-name) - ;; Switch to the khoj window - (select-window khoj-window) - ;; Resize the window to 1/3 of the frame width - (window-resize khoj-window - (- (truncate (* 0.33 (frame-width))) (window-width)) - t)))))) + (save-selected-window + (if (get-buffer-window-list buffer-name) + ;; if window is already open, switch to it + (progn + (select-window (get-buffer-window buffer-name)) + (switch-to-buffer buffer-name)) + ;; else if window is not open, open it as a right-side window pane + (let ((bottomright-window (some-window (lambda (window) (and (window-at-side-p window 'right) (window-at-side-p window 'bottom)))))) + (progn + ;; Select the right-most window + (select-window bottomright-window) + ;; if bottom-right window is not a vertical pane, split it vertically, else use the existing bottom-right vertical window + (let ((khoj-window (if (window-at-side-p bottomright-window 'left) + (split-window-right) + bottomright-window))) + ;; Set the buffer in the khoj window + (set-window-buffer khoj-window buffer-name) + ;; Switch to the khoj window + (select-window khoj-window) + ;; Resize the window to 1/3 of the frame width + (window-resize khoj-window + (- (truncate (* 0.33 (frame-width))) (window-width)) + t))))) + (goto-char (point-min)))) (defun khoj--load-chat-session (buffer-name &optional session-id) "Load Khoj Chat conversation history into BUFFER-NAME." @@ -1162,8 +1164,7 @@ Paragraph only starts at first text after blank line." buffer-name rerank t) - (khoj--open-side-pane buffer-name) - (goto-char (point-min))))) + (khoj--open-side-pane buffer-name)))) (defun khoj--auto-find-similar () "Call find similar on current element, if point has moved to a new element." @@ -1176,8 +1177,7 @@ Paragraph only starts at first text after blank line." (unless (eq current-heading-pos khoj--last-heading-pos) (progn (setq khoj--last-heading-pos current-heading-pos) - (save-excursion - (khoj--find-similar))))))) + (khoj--find-similar)))))) (defun khoj--setup-auto-find-similar () "Setup automatic call to find similar to current element." From 9262aea7a5b3c4460833a590d3d3959f04d811f5 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Thu, 20 Jun 2024 17:08:42 +0530 Subject: [PATCH 6/8] Fix comments, func calls based on melpazoid, checkdoc, package-lint --- src/interface/emacs/khoj.el | 149 +++++++++++++++++++++--------------- 1 file changed, 86 insertions(+), 63 deletions(-) diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el index b641dde3..2f6ae239 100644 --- a/src/interface/emacs/khoj.el +++ b/src/interface/emacs/khoj.el @@ -163,20 +163,18 @@ NO-PAGING FILTER)) (defun khoj--keybindings-info-message () "Show available khoj keybindings in-context, when khoj invoked." - (let ((enabled-content-types (khoj--get-enabled-content-types))) - (concat - " + (concat + " Set Content Type -------------------------\n" - ("C-c RET | improve sort \n")))) + "C-c RET | improve sort \n")) (defvar khoj--rerank nil "Track when re-rank of results triggered.") (defvar khoj--reference-count 0 "Track number of references currently in chat bufffer.") (defun khoj--improve-sort () "Use cross-encoder to improve sorting of search results." (interactive) (khoj--incremental-search t)) (defun khoj--make-search-keymap (&optional existing-keymap) "Setup keymap to configure Khoj search. Build of EXISTING-KEYMAP when passed." - (let ((enabled-content-types (khoj--get-enabled-content-types)) - (kmap (or existing-keymap (make-sparse-keymap)))) + (let ((kmap (or existing-keymap (make-sparse-keymap)))) (define-key kmap (kbd "C-c RET") #'khoj--improve-sort) kmap)) @@ -249,12 +247,12 @@ for example), set this to the full interpreter path." (make-obsolete-variable 'khoj-org-files 'khoj-index-files "1.2.0" 'set) (defcustom khoj-index-files (org-agenda-files t t) - "List of org, markdown, pdf and other plaintext to index on khoj server." + "List of org, md, text, pdf to index on khoj server." :type '(repeat string) :group 'khoj) (defcustom khoj-index-directories nil - "List of directories with org, markdown, pdf and other plaintext files to index on khoj server." + "List of directories with org, md, text, pdf to index on khoj server." :type '(repeat string) :group 'khoj) @@ -503,13 +501,15 @@ Use `BOUNDARY' to separate files. This is sent to Khoj server as a POST request. ;; ------------------------------------------- ;; Render Response from Khoj server for Emacs ;; ------------------------------------------- -(defun construct-find-similar-title (query) - "Construct title for find-similar query." +(defun khoj--construct-find-similar-title (query) + "Construct title for find-similar QUERY." (format "Similar to: %s" (replace-regexp-in-string "^[#\\*]* " "" (car (split-string query "\n"))))) (defun khoj--extract-entries-as-markdown (json-response query is-find-similar) - "Convert JSON-RESPONSE, QUERY from API to markdown entries." + "Convert JSON-RESPONSE, QUERY from API to markdown entries. +Use IS-FIND-SIMILAR bool to filter out first result. +As first result is the current entry at point." (thread-last json-response ;; filter our first result if is find similar as it'll be the current entry at point @@ -524,13 +524,14 @@ Use `BOUNDARY' to separate files. This is sent to Khoj server as a POST request. ;; Standardize results to 2nd level heading for consistent rendering (replace-regexp-in-string "^\#+" "##")))) ;; Render entries into markdown formatted string with query set as as top level heading - (format "# %s\n%s" (if is-find-similar (construct-find-similar-title query) query)) + (format "# %s\n%s" (if is-find-similar (khoj--construct-find-similar-title query) query)) ;; remove leading (, ) or SPC from extracted entries string (replace-regexp-in-string "^[\(\) ]" ""))) (defun khoj--extract-entries-as-org (json-response query is-find-similar) "Convert JSON-RESPONSE, QUERY from API to `org-mode' entries. -Use IS-FIND-SIMILAR to determine if results should be ." +Use IS-FIND-SIMILAR bool to filter out first result. +As first result is the current entry at point." (thread-last json-response ;; filter our first result if is find similar as it'll be the current entry at point @@ -545,12 +546,14 @@ Use IS-FIND-SIMILAR to determine if results should be ." ;; Standardize results to 2nd level heading for consistent rendering (replace-regexp-in-string "^\*+" "**")))) ;; Render entries into org formatted string with query set as as top level heading - (format "* %s\n%s\n" (if is-find-similar (construct-find-similar-title query) query)) + (format "* %s\n%s\n" (if is-find-similar (khoj--construct-find-similar-title query) query)) ;; remove leading (, ) or SPC from extracted entries string (replace-regexp-in-string "^[\(\) ]" ""))) (defun khoj--extract-entries-as-pdf (json-response query is-find-similar) - "Convert QUERY, JSON-RESPONSE from API with PDF results to `org-mode' entries." + "Convert JSON-RESPONSE, QUERY from API to PDF entries. +Use IS-FIND-SIMILAR bool to filter out first result. +As first result is the current entry at point." (thread-last json-response ;; filter our first result if is find similar as it'll be the current entry at point @@ -563,7 +566,7 @@ Use IS-FIND-SIMILAR to determine if results should be ." ;; Format pdf entry as a org entry string (format "** %s\n\n")))) ;; Render entries into org formatted string with query set as as top level heading - (format "* %s\n%s\n" (if is-find-similar (construct-find-similar-title query) query)) + (format "* %s\n%s\n" (if is-find-similar (khoj--construct-find-similar-title query) query)) ;; remove leading (, ) or SPC from extracted entries string (replace-regexp-in-string "^[\(\) ]" ""))) @@ -596,7 +599,9 @@ Use IS-FIND-SIMILAR to determine if results should be ." (replace-regexp-in-string "[\(\) ]$" "")))) (defun khoj--extract-entries (json-response query is-find-similar) - "Convert JSON-RESPONSE, QUERY from API to text entries." + "Convert JSON-RESPONSE, QUERY from API to text entries. +Use IS-FIND-SIMILAR bool to filter out first result. +As first result is the current entry at point." (thread-last json-response ;; filter our first result if is find similar as it'll be the current entry at point ((lambda (response) (if is-find-similar (seq-drop response 1) response))) @@ -613,7 +618,7 @@ Use IS-FIND-SIMILAR to determine if results should be ." ;; Format entries as org entry string (format "** %s")))) ;; Set query as heading in rendered results buffer - (format "* %s\n%s\n" (if is-find-similar (construct-find-similar-title query) query)) + (format "* %s\n%s\n" (if is-find-similar (khoj--construct-find-similar-title query) query)) ;; remove leading (, ) or SPC from extracted entries string (replace-regexp-in-string "^[\(\) ]" "") ;; remove trailing (, ) or SPC from extracted entries string @@ -629,13 +634,30 @@ Use IS-FIND-SIMILAR to determine if results should be ." ((and (member 'markdown enabled-content-types) (or (equal file-extension "markdown") (equal file-extension "md"))) "markdown") (t khoj-default-content-type)))) + +(defun khoj--org-cycle-content (&optional arg) + "Show all headlines in the buffer, like a table of contents. +With numerical argument ARG, show content up to level ARG. + +Simplified fork of `org-cycle-content' from Emacs 29.1 to work with >=27.1." + (interactive "p") + (save-excursion + (goto-char (point-max)) + (let ((regexp (if (and (wholenump arg) (> arg 0)) + (format "^\\*\\{1,%d\\} " arg) + "^\\*+ ")) + (last (point))) + (while (re-search-backward regexp nil t) + (org-fold-region (line-end-position) last t 'outline) + (setq last (line-end-position 0)))))) + ;; -------------- ;; Query Khoj API ;; -------------- (defun khoj--call-api (path &optional method params callback &rest cbargs) "Sync call API at PATH with METHOD and query PARAMS as kv assoc list. -Return json parsed response as alist." +Optionally apply CALLBACK with JSON parsed response and CBARGS." (let* ((url-request-method (or method "GET")) (url-request-extra-headers `(("Authorization" . ,(format "Bearer %s" khoj-api-key)))) (param-string (if params (url-build-query-string params) "")) @@ -654,7 +676,7 @@ Return json parsed response as alist." (defun khoj--call-api-async (path &optional method params callback &rest cbargs) "Async call to API at PATH with METHOD and query PARAMS as kv assoc list. -Return json parsed response as alist." +Optionally apply CALLBACK with JSON parsed response and CBARGS." (let* ((url-request-method (or method "GET")) (url-request-extra-headers `(("Authorization" . ,(format "Bearer %s" khoj-api-key)))) (param-string (if params (url-build-query-string params) "")) @@ -678,11 +700,12 @@ Return json parsed response as alist." (khoj--call-api "/api/config/types" "GET" nil `(lambda (item) (mapcar #'intern item)))) (defun khoj--query-search-api-and-render-results (query content-type buffer-name &optional rerank is-find-similar) - "Query Khoj Search API with QUERY, CONTENT-TYPE and (optional) RERANK as query params -Render results in BUFFER-NAME using search results, CONTENT-TYPE and (optional) TITLE." - (let ((rerank (or rerank "false")) - (params `((q ,query) (t ,content-type) (r ,rerank) (n ,khoj-results-count))) - (path "/api/search")) + "Query Khoj Search API with QUERY, CONTENT-TYPE and RERANK as query params. +Render search results in BUFFER-NAME using CONTENT-TYPE and QUERY. +Filter out first similar result if IS-FIND-SIMILAR set." + (let* ((rerank (or rerank "false")) + (params `((q ,query) (t ,content-type) (r ,rerank) (n ,khoj-results-count))) + (path "/api/search")) (khoj--call-api-async path "GET" params @@ -690,6 +713,8 @@ Render results in BUFFER-NAME using search results, CONTENT-TYPE and (optional) content-type query buffer-name is-find-similar))) (defun khoj--render-search-results (json-response content-type query buffer-name &optional is-find-similar) + "Render search results in BUFFER-NAME using JSON-RESPONSE, CONTENT-TYPE, QUERY. +Filter out first similar result if IS-FIND-SIMILAR set." ;; render json response into formatted entries (with-current-buffer buffer-name (let ((is-find-similar (or is-find-similar nil)) @@ -699,7 +724,7 @@ Render results in BUFFER-NAME using search results, CONTENT-TYPE and (optional) (cond ((equal content-type "org") (khoj--extract-entries-as-org json-response query is-find-similar)) ((equal content-type "markdown") (khoj--extract-entries-as-markdown json-response query is-find-similar)) ((equal content-type "pdf") (khoj--extract-entries-as-pdf json-response query is-find-similar)) - ((equal content-type "image") (khoj--extract-entries-as-images json-response)) + ((equal content-type "image") (khoj--extract-entries-as-images json-response query)) (t (khoj--extract-entries json-response query is-find-similar)))) (cond ((or (equal content-type "all") (equal content-type "pdf") @@ -709,7 +734,7 @@ Render results in BUFFER-NAME using search results, CONTENT-TYPE and (optional) (setq-local org-hide-leading-stars t org-startup-with-inline-images t) - (org-cycle-content 2))) + (khoj--org-cycle-content 2))) ((equal content-type "markdown") (progn (markdown-mode) (visual-line-mode))) ((equal content-type "image") (progn (shr-render-region (point-min) (point-max)) @@ -730,8 +755,7 @@ Render results in BUFFER-NAME using search results, CONTENT-TYPE and (optional) "Chat with Khoj." (interactive) (when (not (get-buffer khoj--chat-buffer-name)) - (khoj--load-chat-session khoj--chat-buffer-name)) - (khoj--open-side-pane khoj--chat-buffer-name) + (khoj--load-chat-session khoj--chat-buffer-name)) (let ((query (read-string "Query: "))) (when (not (string-empty-p query)) (khoj--query-chat-api-and-render-messages query khoj--chat-buffer-name)))) @@ -761,27 +785,27 @@ Render results in BUFFER-NAME using search results, CONTENT-TYPE and (optional) (window-resize khoj-window (- (truncate (* 0.33 (frame-width))) (window-width)) t))))) - (goto-char (point-min)))) + (goto-char (point-max)))) (defun khoj--load-chat-session (buffer-name &optional session-id) - "Load Khoj Chat conversation history into BUFFER-NAME." + "Load Khoj Chat conversation history from SESSION-ID into BUFFER-NAME." (setq khoj--reference-count 0) (let ((inhibit-read-only t) (json-response (cdr (assoc 'chat (cdr (assoc 'response (khoj--get-chat-session session-id))))))) (with-current-buffer (get-buffer-create buffer-name) - (erase-buffer) - (insert "* Khoj Chat\n") - (when json-response - (thread-last - json-response - ;; generate chat messages from Khoj Chat API response - (mapcar #'khoj--format-chat-response) - ;; insert chat messages into Khoj Chat Buffer - (mapc #'insert))) (progn + (erase-buffer) + (insert "* Khoj Chat\n") + (when json-response + (thread-last + json-response + ;; generate chat messages from Khoj Chat API response + (mapcar #'khoj--format-chat-response) + ;; insert chat messages into Khoj Chat Buffer + (mapc #'insert))) (org-mode) - (khoj--add-hover-text-to-footnote-refs (point-min)) - + ;; commented add-hover-text func due to perf issues with the implementation + ;;(khoj--add-hover-text-to-footnote-refs (point-min)) ;; render reference footnotes as superscript (setq-local org-startup-folded "showall" @@ -799,10 +823,11 @@ Render results in BUFFER-NAME using search results, CONTENT-TYPE and (optional) ;; enable minor modes for khoj chat (visual-line-mode) - (read-only-mode t))))) + (read-only-mode t))) + (khoj--open-side-pane buffer-name))) (defun khoj--close () - "Kill Khoj buffer and window" + "Kill Khoj buffer and window." (interactive) (progn (kill-buffer (current-buffer)) @@ -846,8 +871,7 @@ Render results in BUFFER-NAME using search results, CONTENT-TYPE and (optional) #'khoj--render-chat-response buffer-name)))) (defun khoj--query-chat-api (query callback &rest cbargs) - "Send QUERY to Khoj Chat API and call CALLBACK with the response. -CBARGS are optional additional arguments to pass to CALLBACK." + "Send QUERY to Khoj Chat API and call CALLBACK with the response and CBARGS." (khoj--call-api-async "/api/chat" "GET" `(("q" ,query) ("n" ,khoj-results-count)) @@ -879,8 +903,7 @@ CBARGS are optional additional arguments to pass to CALLBACK." (defun khoj--open-conversation-session () "Menu to select Khoj conversation session to open." (let ((selected-session-id (khoj--select-conversation-session "Open"))) - (khoj--load-chat-session khoj--chat-buffer-name selected-session-id) - (khoj--open-side-pane khoj--chat-buffer-name))) + (khoj--load-chat-session khoj--chat-buffer-name selected-session-id))) (defun khoj--create-chat-session () "Create new chat session." @@ -890,19 +913,17 @@ CBARGS are optional additional arguments to pass to CALLBACK." "Create new Khoj conversation session." (let* ((session (khoj--create-chat-session)) (new-session-id (cdr (assoc 'conversation_id session)))) - (khoj--load-chat-session khoj--chat-buffer-name new-session-id) - (khoj--open-side-pane khoj--chat-buffer-name))) + (khoj--load-chat-session khoj--chat-buffer-name new-session-id))) (defun khoj--delete-chat-session (session-id) - "Delete new chat session." + "Delete chat session with SESSION-ID." (khoj--call-api "/api/chat/history" "DELETE" `(("conversation_id" ,session-id)))) (defun khoj--delete-conversation-session () "Delete new Khoj conversation session." (let* ((selected-session-id (khoj--select-conversation-session "Delete")) (session (khoj--delete-chat-session selected-session-id))) - (khoj--load-chat-session khoj--chat-buffer-name) - (khoj--open-side-pane khoj--chat-buffer-name))) + (khoj--load-chat-session khoj--chat-buffer-name))) (defun khoj--render-chat-message (message sender &optional receive-date) "Render chat messages as `org-mode' list item. @@ -939,6 +960,7 @@ RECEIVE-DATE is the message receive date." (format "\n[fn:%x] %s" khoj--reference-count))))) (defun khoj--generate-online-reference (reference) + "Create `org-mode' footnotes for online REFERENCE." (setq khoj--reference-count (1+ khoj--reference-count)) (let ((link (cdr (assoc 'link reference))) (title (cdr (assoc 'title reference))) @@ -951,8 +973,8 @@ RECEIVE-DATE is the message receive date." (replace-regexp-in-string "\n\n" "\n") (format "\n[fn:%x] [[%s][%s]]\n%s\n" khoj--reference-count link title))))) -(defun khoj--extract-online-references (result-types searches) - "Extract link, title, and description of specified RESULT-TYPES from SEARCHES." +(defun khoj--extract-online-references (result-types query-result-pairs) + "Extract link, title and description from RESULT-TYPES in QUERY-RESULT-PAIRS." (let ((result '())) (-map (lambda (search) @@ -974,10 +996,11 @@ RECEIVE-DATE is the message receive date." (list (cdr search-result)) (cdr search-result)))) search-results))) - searches) + query-result-pairs) result)) (defun khoj--render-chat-response (response buffer-name) + "Insert chat message from RESPONSE into BUFFER-NAME." (with-current-buffer (get-buffer buffer-name) (let ((start-pos (point)) (inhibit-read-only t)) @@ -991,7 +1014,8 @@ RECEIVE-DATE is the message receive date." (re-search-backward "^\*+ 🏮" nil t))))) (defun khoj--format-chat-response (json-response &optional callback &rest cbargs) - "Render chat message using JSON-RESPONSE from Khoj Chat API." + "Format chat message using JSON-RESPONSE from Khoj Chat API. +Run CALLBACK with CBARGS on formatted message." (let* ((message (cdr (or (assoc 'response json-response) (assoc 'message json-response)))) (sender (cdr (assoc 'by json-response))) (receive-date (cdr (assoc 'created json-response))) @@ -1170,20 +1194,19 @@ Paragraph only starts at first text after blank line." "Call find similar on current element, if point has moved to a new element." ;; Call find similar (when (and (derived-mode-p 'org-mode) - (not (null (org-element-at-point))) + (org-element-at-point) (not (string= (buffer-name (current-buffer)) khoj--search-buffer-name)) - (not (null (get-buffer-window khoj--search-buffer-name)))) + (get-buffer-window khoj--search-buffer-name)) (let ((current-heading-pos (khoj--get-current-outline-entry-pos))) (unless (eq current-heading-pos khoj--last-heading-pos) - (progn (setq khoj--last-heading-pos current-heading-pos) - (khoj--find-similar)))))) + (khoj--find-similar))))) (defun khoj--setup-auto-find-similar () "Setup automatic call to find similar to current element." (if khoj-auto-find-similar - (add-hook 'post-command-hook 'khoj--auto-find-similar) - (remove-hook 'post-command-hook 'khoj--auto-find-similar))) + (add-hook 'post-command-hook #'khoj--auto-find-similar) + (remove-hook 'post-command-hook #'khoj--auto-find-similar))) (defun khoj-toggle-auto-find-similar () "Toggle automatic call to find similar to current element." From 59032a06d54b4a07be969724670500e3c182402b Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Thu, 20 Jun 2024 17:19:32 +0530 Subject: [PATCH 7/8] Improve defaults when extracting fields from online reference in khoj.el --- src/interface/emacs/khoj.el | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el index 2f6ae239..4ba6e2ca 100644 --- a/src/interface/emacs/khoj.el +++ b/src/interface/emacs/khoj.el @@ -962,9 +962,9 @@ RECEIVE-DATE is the message receive date." (defun khoj--generate-online-reference (reference) "Create `org-mode' footnotes for online REFERENCE." (setq khoj--reference-count (1+ khoj--reference-count)) - (let ((link (cdr (assoc 'link reference))) - (title (cdr (assoc 'title reference))) - (description (cdr (assoc 'description reference)))) + (let* ((link (cdr (assoc 'link reference))) + (title (or (cdr (assoc 'title reference)) link)) + (description (or (cdr (assoc 'description reference)) title))) (cons (propertize (format "^{ [fn:%x]}" khoj--reference-count) 'help-echo (format "%s\n%s" link description)) (thread-last From 427575e95864569ce2bfd45259eae45aa46f8b90 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Thu, 20 Jun 2024 21:38:02 +0530 Subject: [PATCH 8/8] Improve khoj chat new, delete session flows When create new conversation session, automatically request query. As that is expected next action after creating new session Pass session-id to khoj-chat to allow reuse from create-new-conversation func When delete conversation session, do not call load chat session. Unnecessary action. Use thread-last to improve code flow in new, delete conversation funcs --- src/interface/emacs/khoj.el | 44 +++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el index 4ba6e2ca..ce46c23c 100644 --- a/src/interface/emacs/khoj.el +++ b/src/interface/emacs/khoj.el @@ -751,14 +751,14 @@ Filter out first similar result if IS-FIND-SIMILAR set." ;; Khoj Chat ;; ---------------- -(defun khoj--chat () - "Chat with Khoj." +(defun khoj--chat (&optional session-id) + "Chat with Khoj in session with SESSION-ID." (interactive) - (when (not (get-buffer khoj--chat-buffer-name)) - (khoj--load-chat-session khoj--chat-buffer-name)) + (when (or session-id (not (get-buffer khoj--chat-buffer-name))) + (khoj--load-chat-session khoj--chat-buffer-name session-id)) (let ((query (read-string "Query: "))) (when (not (string-empty-p query)) - (khoj--query-chat-api-and-render-messages query khoj--chat-buffer-name)))) + (khoj--query-chat-api-and-render-messages query khoj--chat-buffer-name session-id)))) (defun khoj--open-side-pane (buffer-name) "Open Khoj BUFFER-NAME in right side pane." @@ -857,8 +857,8 @@ Filter out first similar result if IS-FIND-SIMILAR set." ;; show definition on hover on footnote reference (overlay-put overlay 'help-echo it))))))) -(defun khoj--query-chat-api-and-render-messages (query buffer-name) - "Send QUERY to Khoj Chat. Render the chat messages from exchange in BUFFER-NAME." +(defun khoj--query-chat-api-and-render-messages (query buffer-name &optional session-id) + "Send QUERY to Chat SESSION-ID. Render the chat messages in BUFFER-NAME." ;; render json response into formatted chat messages (with-current-buffer (get-buffer buffer-name) (let ((inhibit-read-only t) @@ -867,15 +867,19 @@ Filter out first similar result if IS-FIND-SIMILAR set." (insert (khoj--render-chat-message query "you" query-time)) (khoj--query-chat-api query + session-id #'khoj--format-chat-response #'khoj--render-chat-response buffer-name)))) -(defun khoj--query-chat-api (query callback &rest cbargs) - "Send QUERY to Khoj Chat API and call CALLBACK with the response and CBARGS." - (khoj--call-api-async "/api/chat" - "GET" - `(("q" ,query) ("n" ,khoj-results-count)) - callback cbargs)) +(defun khoj--query-chat-api (query session-id callback &rest cbargs) + "Send QUERY for SESSION-ID to Khoj Chat API. +Call CALLBACK func with response and CBARGS." + (let ((params `(("q" ,query) ("n" ,khoj-results-count)))) + (when session-id (push `("conversation_id" ,session-id) params)) + (khoj--call-api-async "/api/chat" + "GET" + params + callback cbargs))) (defun khoj--get-chat-sessions () "Get all chat sessions from Khoj server." @@ -911,9 +915,11 @@ Filter out first similar result if IS-FIND-SIMILAR set." (defun khoj--new-conversation-session () "Create new Khoj conversation session." - (let* ((session (khoj--create-chat-session)) - (new-session-id (cdr (assoc 'conversation_id session)))) - (khoj--load-chat-session khoj--chat-buffer-name new-session-id))) + (thread-last + (khoj--create-chat-session) + (assoc 'conversation_id) + (cdr) + (khoj--chat))) (defun khoj--delete-chat-session (session-id) "Delete chat session with SESSION-ID." @@ -921,9 +927,9 @@ Filter out first similar result if IS-FIND-SIMILAR set." (defun khoj--delete-conversation-session () "Delete new Khoj conversation session." - (let* ((selected-session-id (khoj--select-conversation-session "Delete")) - (session (khoj--delete-chat-session selected-session-id))) - (khoj--load-chat-session khoj--chat-buffer-name))) + (thread-last + (khoj--select-conversation-session "Delete") + (khoj--delete-chat-session))) (defun khoj--render-chat-message (message sender &optional receive-date) "Render chat messages as `org-mode' list item.