mirror of
https://github.com/khoaliber/khoj.git
synced 2026-03-04 21:29:12 +00:00
Merge branch 'master' of github.com:debanjum/khoj into create-native-gui
Conflicts:
- src/main.py
- router functions have moved to router
- move logic to handle null query perf timer variables into
router.py
- set main.py to current branch, not master
This commit is contained in:
Debanjum Singh Solanky
@@ -36,26 +36,39 @@
|
||||
(require 'url)
|
||||
(require 'json)
|
||||
|
||||
(defcustom khoj--server-url "http://localhost:8000"
|
||||
(defcustom khoj-server-url "http://localhost:8000"
|
||||
"Location of Khoj API server."
|
||||
:group 'khoj
|
||||
:type 'string)
|
||||
|
||||
(defcustom khoj--image-width 156
|
||||
(defcustom khoj-image-width 156
|
||||
"Width of rendered images returned by Khoj."
|
||||
:group 'khoj
|
||||
:type 'integer)
|
||||
|
||||
(defcustom khoj--rerank-after-idle-time 1.0
|
||||
(defcustom khoj-image-height 156
|
||||
"Height of rendered images returned by Khoj."
|
||||
:group 'khoj
|
||||
:type 'integer)
|
||||
|
||||
(defcustom khoj-rerank-after-idle-time 2.0
|
||||
"Idle time (in seconds) to trigger cross-encoder to rerank incremental search results."
|
||||
:group 'khoj
|
||||
:type 'float)
|
||||
|
||||
(defcustom khoj--results-count 5
|
||||
(defcustom khoj-results-count 5
|
||||
"Number of results to get from Khoj API for each query."
|
||||
:group 'khoj
|
||||
:type 'integer)
|
||||
|
||||
(defcustom khoj-default-search-type "org"
|
||||
"The default content type to perform search on."
|
||||
:group 'khoj
|
||||
:type '(choice (const "org")
|
||||
(const "markdown")
|
||||
(const "ledger")
|
||||
(const "music")))
|
||||
|
||||
(defvar khoj--rerank-timer nil
|
||||
"Idle timer to make cross-encoder re-rank incremental search results if user idle.")
|
||||
|
||||
@@ -71,37 +84,52 @@
|
||||
(defvar khoj--search-type "org"
|
||||
"The type of content to perform search on.")
|
||||
|
||||
(defvar khoj--keybindings-info-message
|
||||
"
|
||||
(defun khoj--keybindings-info-message ()
|
||||
(let ((enabled-content-types (khoj--get-enabled-content-types)))
|
||||
(concat
|
||||
"
|
||||
Set Search Type
|
||||
-------------------------
|
||||
C-x m | markdown
|
||||
C-x o | org-mode
|
||||
C-x l | ledger/beancount
|
||||
C-x i | images
|
||||
")
|
||||
(defun khoj--search-markdown (interactive) (setq khoj--search-type "markdown"))
|
||||
(defun khoj--search-org (interactive) (setq khoj--search-type "org"))
|
||||
(defun khoj--search-ledger (interactive) (setq khoj--search-type "ledger"))
|
||||
(defun khoj--search-images (interactive) (setq khoj--search-type "image"))
|
||||
-------------------------\n"
|
||||
(when (member 'markdown enabled-content-types)
|
||||
"C-x m | markdown\n")
|
||||
(when (member 'org enabled-content-types)
|
||||
"C-x o | org-mode\n")
|
||||
(when (member 'ledger enabled-content-types)
|
||||
"C-x l | ledger\n")
|
||||
(when (member 'image enabled-content-types)
|
||||
"C-x i | images\n")
|
||||
(when (member 'music enabled-content-types)
|
||||
"C-x M | music\n"))))
|
||||
|
||||
(defun khoj--search-markdown () (interactive) (setq khoj--search-type "markdown"))
|
||||
(defun khoj--search-org () (interactive) (setq khoj--search-type "org"))
|
||||
(defun khoj--search-ledger () (interactive) (setq khoj--search-type "ledger"))
|
||||
(defun khoj--search-images () (interactive) (setq khoj--search-type "image"))
|
||||
(defun khoj--search-music () (interactive) (setq khoj--search-type "music"))
|
||||
(defun khoj--make-search-keymap (&optional existing-keymap)
|
||||
"Setup keymap to configure Khoj search"
|
||||
(let ((kmap (or existing-keymap (make-sparse-keymap))))
|
||||
(define-key kmap (kbd "C-x m") #'khoj--search-markdown)
|
||||
(define-key kmap (kbd "C-x o") #'khoj--search-org)
|
||||
(define-key kmap (kbd "C-x l") #'khoj--search-ledger)
|
||||
(define-key kmap (kbd "C-x i") #'khoj--search-images)
|
||||
(let ((enabled-content-types (khoj--get-enabled-content-types))
|
||||
(kmap (or existing-keymap (make-sparse-keymap))))
|
||||
(when (member 'markdown enabled-content-types)
|
||||
(define-key kmap (kbd "C-x m") #'khoj--search-markdown))
|
||||
(when (member 'org enabled-content-types)
|
||||
(define-key kmap (kbd "C-x o") #'khoj--search-org))
|
||||
(when (member 'ledger enabled-content-types)
|
||||
(define-key kmap (kbd "C-x l") #'khoj--search-ledger))
|
||||
(when (member 'image enabled-content-types)
|
||||
(define-key kmap (kbd "C-x i") #'khoj--search-images))
|
||||
(when (member 'music enabled-content-types)
|
||||
(define-key kmap (kbd "C-x M") #'khoj--search-music))
|
||||
kmap))
|
||||
(defun khoj--display-keybinding-info ()
|
||||
"Display information on keybindings to customize khoj search.
|
||||
Use `which-key` if available, else display simple message in echo area"
|
||||
(if (fboundp 'which-key--create-buffer-and-show)
|
||||
(which-key--create-buffer-and-show
|
||||
(kbd "C-x")
|
||||
(symbolp (khoj--make-search-keymap))
|
||||
'(lambda (binding) (string-prefix-p "khoj--" (cdr binding)))
|
||||
"Khoj Bindings")
|
||||
(message "%s" khoj--keybindings-info-message)))
|
||||
(if (fboundp 'which-key-show-full-keymap)
|
||||
(let ((khoj--keymap (khoj--make-search-keymap)))
|
||||
(which-key--show-keymap (symbol-name 'khoj--keymap)
|
||||
(symbol-value 'khoj--keymap)
|
||||
nil t t))
|
||||
(message "%s" (khoj--keybindings-info-message))))
|
||||
|
||||
(defun khoj--extract-entries-as-markdown (json-response query)
|
||||
"Convert json response from API to markdown entries"
|
||||
@@ -124,7 +152,7 @@ Use `which-key` if available, else display simple message in echo area"
|
||||
(replace-regexp-in-string
|
||||
"^[\(\) ]" ""
|
||||
;; extract entries from response as single string and convert to entries
|
||||
(format "#+STARTUP: showall hidestars inlineimages\n* %s\n%s"
|
||||
(format "* %s\n%s\n#+STARTUP: showall hidestars inlineimages"
|
||||
query
|
||||
(mapcar
|
||||
(lambda (args)
|
||||
@@ -146,15 +174,17 @@ Use `which-key` if available, else display simple message in echo area"
|
||||
query
|
||||
(mapcar
|
||||
(lambda (args) (format
|
||||
"\n\n<h2>Score: %s Meta: %s Image: %s</h2>\n\n<a href=\"%s%s\">\n<img src=\"%s%s?%s\" width=100 height=100>\n</a>"
|
||||
"\n\n<h2>Score: %s Meta: %s Image: %s</h2>\n\n<a href=\"%s%s\">\n<img src=\"%s%s?%s\" width=%s height=%s>\n</a>"
|
||||
(cdr (assoc 'score args))
|
||||
(cdr (assoc 'metadata_score args))
|
||||
(cdr (assoc 'image_score args))
|
||||
khoj--server-url
|
||||
khoj-server-url
|
||||
(cdr (assoc 'entry args))
|
||||
khoj--server-url
|
||||
khoj-server-url
|
||||
(cdr (assoc 'entry args))
|
||||
(random 10000)))
|
||||
(random 10000)
|
||||
khoj-image-width
|
||||
khoj-image-height))
|
||||
json-response)))))
|
||||
|
||||
(defun khoj--extract-entries-as-ledger (json-response query)
|
||||
@@ -173,19 +203,34 @@ Use `which-key` if available, else display simple message in echo area"
|
||||
json-response)))))
|
||||
|
||||
(defun khoj--buffer-name-to-search-type (buffer-name)
|
||||
(let ((file-extension (file-name-extension buffer-name)))
|
||||
(let ((enabled-content-types (khoj--get-enabled-content-types))
|
||||
(file-extension (file-name-extension buffer-name)))
|
||||
(cond
|
||||
((equal buffer-name "Music.org") "music")
|
||||
((or (equal file-extension "bean") (equal file-extension "beancount")) "ledger")
|
||||
((equal file-extension "org") "org")
|
||||
((or (equal file-extension "markdown") (equal file-extension "md")) "markdown")
|
||||
(t "org"))))
|
||||
((and (member 'music enabled-content-types) (equal buffer-name "Music.org")) "music")
|
||||
((and (member 'ledger enabled-content-types) (or (equal file-extension "bean") (equal file-extension "beancount"))) "ledger")
|
||||
((and (member 'org enabled-content-types) (equal file-extension "org")) "org")
|
||||
((and (member 'markdown enabled-content-types) (or (equal file-extension "markdown") (equal file-extension "md"))) "markdown")
|
||||
(t khoj-default-search-type))))
|
||||
|
||||
(defun khoj--get-enabled-content-types ()
|
||||
"Get content types enabled for search from API"
|
||||
(let ((config-url (format "%s/config/data" khoj-server-url)))
|
||||
(with-temp-buffer
|
||||
(erase-buffer)
|
||||
(url-insert-file-contents config-url)
|
||||
(let* ((json-response (json-parse-buffer :object-type 'alist))
|
||||
(content-type (cdr (assoc 'content-type json-response))))
|
||||
;; return content-type items with configuration
|
||||
(mapcar
|
||||
'car
|
||||
(cl-remove-if-not
|
||||
'(lambda (a) (not (eq (cdr a) :null)))
|
||||
content-type))))))
|
||||
|
||||
(defun khoj--construct-api-query (query search-type &optional rerank)
|
||||
(let ((rerank (or rerank "false"))
|
||||
(results-count (or khoj--results-count 5))
|
||||
(encoded-query (url-hexify-string query)))
|
||||
(format "%s/search?q=%s&t=%s&r=%s&n=%s" khoj--server-url encoded-query search-type rerank results-count)))
|
||||
(format "%s/search?q=%s&t=%s&r=%s&n=%s" khoj-server-url encoded-query search-type rerank khoj-results-count)))
|
||||
|
||||
(defun khoj--query-api-and-render-results (query search-type query-url buffer-name)
|
||||
;; get json response from api
|
||||
@@ -240,12 +285,12 @@ Use `which-key` if available, else display simple message in echo area"
|
||||
"Delete all network connections to khoj server"
|
||||
(dolist (proc (process-list))
|
||||
(let ((proc-buf (buffer-name (process-buffer proc)))
|
||||
(khoj-network-proc-buf (string-join (split-string khoj--server-url "://") " ")))
|
||||
(khoj-network-proc-buf (string-join (split-string khoj-server-url "://") " ")))
|
||||
(when (string-match (format "%s" khoj-network-proc-buf) proc-buf)
|
||||
(delete-process proc)))))
|
||||
|
||||
(defun khoj--teardown-incremental-search ()
|
||||
(message "[Khoj]: Teardown Incremental Search")
|
||||
(message "Khoj: Teardown Incremental Search")
|
||||
;; remove advice to rerank results on normal exit from minibuffer
|
||||
(advice-remove 'exit-minibuffer #'khoj--minibuffer-exit-advice)
|
||||
;; unset khoj minibuffer window
|
||||
@@ -262,6 +307,7 @@ Use `which-key` if available, else display simple message in echo area"
|
||||
(defun khoj--minibuffer-exit-advice (&rest _args)
|
||||
(khoj--incremental-search t))
|
||||
|
||||
|
||||
;;;###autoload
|
||||
(defun khoj ()
|
||||
"Natural, Incremental Search for your personal notes, transactions and music using Khoj"
|
||||
@@ -269,8 +315,8 @@ Use `which-key` if available, else display simple message in echo area"
|
||||
(let* ((khoj-buffer-name (get-buffer-create khoj--buffer-name)))
|
||||
;; set khoj search type to last used or based on current buffer
|
||||
(setq khoj--search-type (or khoj--search-type (khoj--buffer-name-to-search-type (buffer-name))))
|
||||
;; setup rerank to improve results once user idle for KHOJ--RERANK-AFTER-IDLE-TIME seconds
|
||||
(setq khoj--rerank-timer (run-with-idle-timer khoj--rerank-after-idle-time t 'khoj--incremental-search t))
|
||||
;; setup rerank to improve results once user idle for KHOJ-RERANK-AFTER-IDLE-TIME seconds
|
||||
(setq khoj--rerank-timer (run-with-idle-timer khoj-rerank-after-idle-time t 'khoj--incremental-search t))
|
||||
;; switch to khoj results buffer
|
||||
(switch-to-buffer khoj-buffer-name)
|
||||
;; open and setup minibuffer for incremental search
|
||||
|
||||
@@ -38,6 +38,8 @@ import datetime
|
||||
from pathlib import Path
|
||||
from os.path import relpath
|
||||
|
||||
indent_regex = re.compile(r'^\s*')
|
||||
|
||||
def normalize_filename(filename):
|
||||
file_relative_to_home = f'~/{relpath(filename, start=Path.home())}'
|
||||
escaped_filename = f'{file_relative_to_home}'.replace("[","\[").replace("]","\]")
|
||||
@@ -370,7 +372,9 @@ class Orgnode(object):
|
||||
n = ''
|
||||
for _ in range(0, self.level):
|
||||
n = n + '*'
|
||||
n = n + ' ' + self.todo + ' '
|
||||
n = n + ' '
|
||||
if self.todo:
|
||||
n = n + self.todo + ' '
|
||||
if self.prty:
|
||||
n = n + '[#' + self.prty + '] '
|
||||
n = n + self.headline
|
||||
@@ -382,7 +386,12 @@ class Orgnode(object):
|
||||
n = n + closecolon
|
||||
n = n + "\n"
|
||||
|
||||
# Get body indentation from first line of body
|
||||
indent = indent_regex.match(self.body).group()
|
||||
|
||||
# Output Closed Date, Scheduled Date, Deadline Date
|
||||
if self.closed or self.scheduled or self.deadline:
|
||||
n = n + indent
|
||||
if self.closed:
|
||||
n = n + f'CLOSED: [{self.closed.strftime("%Y-%m-%d %a")}] '
|
||||
if self.scheduled:
|
||||
@@ -393,10 +402,10 @@ class Orgnode(object):
|
||||
n = n + '\n'
|
||||
|
||||
# Ouput Property Drawer
|
||||
n = n + ":PROPERTIES:\n"
|
||||
n = n + indent + ":PROPERTIES:\n"
|
||||
for key, value in self.properties.items():
|
||||
n = n + f":{key}: {value}\n"
|
||||
n = n + ":END:\n"
|
||||
n = n + indent + f":{key}: {value}\n"
|
||||
n = n + indent + ":END:\n"
|
||||
|
||||
n = n + self.body
|
||||
|
||||
|
||||
@@ -53,9 +53,11 @@ def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None, r: Opti
|
||||
print(f'No query param (q) passed in API call to initiate search')
|
||||
return {}
|
||||
|
||||
# initialize variables
|
||||
user_query = q
|
||||
results_count = n
|
||||
results = {}
|
||||
query_start, query_end, collate_start, collate_end = None, None, None, None
|
||||
|
||||
if (t == SearchType.Org or t == None) and state.model.orgmode_search:
|
||||
# query org-mode notes
|
||||
@@ -119,8 +121,10 @@ def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None, r: Opti
|
||||
collate_end = time.time()
|
||||
|
||||
if state.verbose > 1:
|
||||
print(f"Query took {query_end - query_start:.3f} seconds")
|
||||
print(f"Collating results took {collate_end - collate_start:.3f} seconds")
|
||||
if query_start and query_end:
|
||||
print(f"Query took {query_end - query_start:.3f} seconds")
|
||||
if collate_start and collate_end:
|
||||
print(f"Collating results took {collate_end - collate_start:.3f} seconds")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ def test_parse_complete_entry(tmp_path):
|
||||
"Test parsing of entry with all important fields"
|
||||
# Arrange
|
||||
entry = f'''
|
||||
*** [#A] Heading :Tag1:TAG2:tag3:
|
||||
*** DONE [#A] Heading :Tag1:TAG2:tag3:
|
||||
CLOSED: [1984-04-01 Sun 12:00] SCHEDULED: <1984-04-01 Sun 09:00> DEADLINE: <1984-04-01 Sun>
|
||||
:PROPERTIES:
|
||||
:ID: 123-456-789-4234-1231
|
||||
@@ -56,6 +56,7 @@ Body Line 2'''
|
||||
# Assert
|
||||
assert len(entries) == 1
|
||||
assert entries[0].Heading() == "Heading"
|
||||
assert entries[0].Todo() == "DONE"
|
||||
assert entries[0].Tags() == {"Tag1", "TAG2", "tag3"}
|
||||
assert entries[0].Body() == "- Clocked Log 1\nBody Line 1\nBody Line 2"
|
||||
assert entries[0].Priority() == "A"
|
||||
@@ -124,7 +125,7 @@ def test_parse_multiple_entries(tmp_path):
|
||||
"Test parsing of multiple entries"
|
||||
# Arrange
|
||||
content = f'''
|
||||
*** [#A] Heading1 :tag1:
|
||||
*** FAILED [#A] Heading1 :tag1:
|
||||
CLOSED: [1984-04-01 Sun 12:00] SCHEDULED: <1984-04-01 Sun 09:00> DEADLINE: <1984-04-01 Sun>
|
||||
:PROPERTIES:
|
||||
:ID: 123-456-789-4234-0001
|
||||
@@ -135,7 +136,7 @@ CLOCK: [1984-04-01 Sun 09:00]--[1984-04-01 Sun 12:00] => 3:00
|
||||
:END:
|
||||
Body 1
|
||||
|
||||
*** [#A] Heading2 :tag2:
|
||||
*** CANCELLED [#A] Heading2 :tag2:
|
||||
CLOSED: [1984-04-02 Sun 12:00] SCHEDULED: <1984-04-02 Sun 09:00> DEADLINE: <1984-04-02 Sun>
|
||||
:PROPERTIES:
|
||||
:ID: 123-456-789-4234-0002
|
||||
@@ -156,6 +157,7 @@ Body 2
|
||||
assert len(entries) == 2
|
||||
for index, entry in enumerate(entries):
|
||||
assert entry.Heading() == f"Heading{index+1}"
|
||||
assert entry.Todo() == "FAILED" if index == 0 else "CANCELLED"
|
||||
assert entry.Tags() == {f"tag{index+1}"}
|
||||
assert entry.Body() == f"- Clocked Log {index+1}\nBody {index+1}\n\n"
|
||||
assert entry.Priority() == "A"
|
||||
|
||||
Reference in New Issue
Block a user