Merge branch 'master' of github.com:debanjum/khoj into create-native-gui

Conflicts:
- src/main.py
  - router functions have moved to router
  - move logic to handle null query perf timer variables into
    router.py
  - set main.py to current branch, not master

src/interface/emacs/khoj.el

@@ -36,26 +36,39 @@
 (require 'url)
 (require 'json)
 
-(defcustom khoj--server-url "http://localhost:8000"
+(defcustom khoj-server-url "http://localhost:8000"
   "Location of Khoj API server."
   :group 'khoj
   :type 'string)
 
-(defcustom khoj--image-width 156
+(defcustom khoj-image-width 156
   "Width of rendered images returned by Khoj."
   :group 'khoj
   :type 'integer)
 
-(defcustom khoj--rerank-after-idle-time 1.0
+(defcustom khoj-image-height 156
+  "Height of rendered images returned by Khoj."
+  :group 'khoj
+  :type 'integer)
+
+(defcustom khoj-rerank-after-idle-time 2.0
   "Idle time (in seconds) to trigger cross-encoder to rerank incremental search results."
   :group 'khoj
   :type 'float)
 
-(defcustom khoj--results-count 5
+(defcustom khoj-results-count 5
   "Number of results to get from Khoj API for each query."
   :group 'khoj
   :type 'integer)
 
+(defcustom khoj-default-search-type "org"
+  "The default content type to perform search on."
+  :group 'khoj
+  :type '(choice (const "org")
+                 (const "markdown")
+                 (const "ledger")
+                 (const "music")))
+
 (defvar khoj--rerank-timer nil
   "Idle timer to make cross-encoder re-rank incremental search results if user idle.")
 
@@ -71,37 +84,52 @@
 (defvar khoj--search-type "org"
   "The type of content to perform search on.")
 
-(defvar khoj--keybindings-info-message
+(defun khoj--keybindings-info-message ()
-  "
+  (let ((enabled-content-types (khoj--get-enabled-content-types)))
+    (concat
+     "
      Set Search Type
--------------------------
+-------------------------\n"
-C-x m  | markdown
+     (when (member 'markdown enabled-content-types)
-C-x o  | org-mode
+         "C-x m  | markdown\n")
-C-x l  | ledger/beancount
+     (when (member 'org enabled-content-types)
-C-x i  | images
+       "C-x o  | org-mode\n")
-")
+     (when (member 'ledger enabled-content-types)
-(defun khoj--search-markdown (interactive) (setq khoj--search-type "markdown"))
+       "C-x l  | ledger\n")
-(defun khoj--search-org (interactive) (setq khoj--search-type "org"))
+     (when (member 'image enabled-content-types)
-(defun khoj--search-ledger (interactive) (setq khoj--search-type "ledger"))
+       "C-x i  | images\n")
-(defun khoj--search-images (interactive) (setq khoj--search-type "image"))
+     (when (member 'music enabled-content-types)
+       "C-x M  | music\n"))))
+
+(defun khoj--search-markdown () (interactive) (setq khoj--search-type "markdown"))
+(defun khoj--search-org () (interactive) (setq khoj--search-type "org"))
+(defun khoj--search-ledger () (interactive) (setq khoj--search-type "ledger"))
+(defun khoj--search-images () (interactive) (setq khoj--search-type "image"))
+(defun khoj--search-music () (interactive) (setq khoj--search-type "music"))
 (defun khoj--make-search-keymap (&optional existing-keymap)
   "Setup keymap to configure Khoj search"
-  (let ((kmap (or existing-keymap (make-sparse-keymap))))
+  (let ((enabled-content-types (khoj--get-enabled-content-types))
-    (define-key kmap (kbd "C-x m") #'khoj--search-markdown)
+        (kmap (or existing-keymap (make-sparse-keymap))))
-    (define-key kmap (kbd "C-x o") #'khoj--search-org)
+    (when (member 'markdown enabled-content-types)
-    (define-key kmap (kbd "C-x l") #'khoj--search-ledger)
+      (define-key kmap (kbd "C-x m") #'khoj--search-markdown))
-    (define-key kmap (kbd "C-x i") #'khoj--search-images)
+    (when (member 'org enabled-content-types)
+      (define-key kmap (kbd "C-x o") #'khoj--search-org))
+    (when (member 'ledger enabled-content-types)
+      (define-key kmap (kbd "C-x l") #'khoj--search-ledger))
+    (when (member 'image enabled-content-types)
+      (define-key kmap (kbd "C-x i") #'khoj--search-images))
+    (when (member 'music enabled-content-types)
+      (define-key kmap (kbd "C-x M") #'khoj--search-music))
     kmap))
 (defun khoj--display-keybinding-info ()
   "Display information on keybindings to customize khoj search.
 Use `which-key` if available, else display simple message in echo area"
-  (if (fboundp 'which-key--create-buffer-and-show)
+  (if (fboundp 'which-key-show-full-keymap)
-      (which-key--create-buffer-and-show
+      (let ((khoj--keymap (khoj--make-search-keymap)))
-       (kbd "C-x")
+        (which-key--show-keymap (symbol-name 'khoj--keymap)
-       (symbolp (khoj--make-search-keymap))
+                                (symbol-value 'khoj--keymap)
-       '(lambda (binding) (string-prefix-p "khoj--" (cdr binding)))
+                                nil t t))
-       "Khoj Bindings")
+    (message "%s" (khoj--keybindings-info-message))))
-    (message "%s" khoj--keybindings-info-message)))
 
 (defun khoj--extract-entries-as-markdown (json-response query)
   "Convert json response from API to markdown entries"
@@ -124,7 +152,7 @@ Use `which-key` if available, else display simple message in echo area"
   (replace-regexp-in-string
    "^[\(\) ]" ""
    ;; extract entries from response as single string and convert to entries
-   (format "#+STARTUP: showall hidestars inlineimages\n* %s\n%s"
+   (format "* %s\n%s\n#+STARTUP: showall hidestars inlineimages"
            query
            (mapcar
             (lambda (args)
@@ -146,15 +174,17 @@ Use `which-key` if available, else display simple message in echo area"
             query
             (mapcar
              (lambda (args) (format
-                             "\n\n<h2>Score: %s Meta: %s Image: %s</h2>\n\n<a href=\"%s%s\">\n<img src=\"%s%s?%s\" width=100 height=100>\n</a>"
+                             "\n\n<h2>Score: %s Meta: %s Image: %s</h2>\n\n<a href=\"%s%s\">\n<img src=\"%s%s?%s\" width=%s height=%s>\n</a>"
                              (cdr (assoc 'score args))
                              (cdr (assoc 'metadata_score args))
                              (cdr (assoc 'image_score args))
-                             khoj--server-url
+                             khoj-server-url
                              (cdr (assoc 'entry args))
-                             khoj--server-url
+                             khoj-server-url
                              (cdr (assoc 'entry args))
-                             (random 10000)))
+                             (random 10000)
+                             khoj-image-width
+                             khoj-image-height))
              json-response)))))
 
 (defun khoj--extract-entries-as-ledger (json-response query)
@@ -173,19 +203,34 @@ Use `which-key` if available, else display simple message in echo area"
              json-response)))))
 
 (defun khoj--buffer-name-to-search-type (buffer-name)
-  (let ((file-extension (file-name-extension buffer-name)))
+  (let ((enabled-content-types (khoj--get-enabled-content-types))
+        (file-extension (file-name-extension buffer-name)))
     (cond
-     ((equal buffer-name "Music.org") "music")
+     ((and (member 'music enabled-content-types) (equal buffer-name "Music.org")) "music")
-     ((or (equal file-extension "bean") (equal file-extension "beancount")) "ledger")
+     ((and (member 'ledger enabled-content-types) (or (equal file-extension "bean") (equal file-extension "beancount"))) "ledger")
-     ((equal file-extension "org") "org")
+     ((and (member 'org enabled-content-types) (equal file-extension "org")) "org")
-     ((or (equal file-extension "markdown") (equal file-extension "md")) "markdown")
+     ((and (member 'markdown enabled-content-types) (or (equal file-extension "markdown") (equal file-extension "md"))) "markdown")
-     (t "org"))))
+     (t khoj-default-search-type))))
+
+(defun khoj--get-enabled-content-types ()
+  "Get content types enabled for search from API"
+  (let ((config-url (format "%s/config/data" khoj-server-url)))
+    (with-temp-buffer
+      (erase-buffer)
+      (url-insert-file-contents config-url)
+      (let* ((json-response (json-parse-buffer :object-type 'alist))
+            (content-type (cdr (assoc 'content-type json-response))))
+        ;; return content-type items with configuration
+        (mapcar
+         'car
+         (cl-remove-if-not
+          '(lambda (a) (not (eq (cdr a) :null)))
+          content-type))))))
 
 (defun khoj--construct-api-query (query search-type &optional rerank)
   (let ((rerank (or rerank "false"))
-        (results-count (or khoj--results-count 5))
         (encoded-query (url-hexify-string query)))
-    (format "%s/search?q=%s&t=%s&r=%s&n=%s" khoj--server-url encoded-query search-type rerank results-count)))
+    (format "%s/search?q=%s&t=%s&r=%s&n=%s" khoj-server-url encoded-query search-type rerank khoj-results-count)))
 
 (defun khoj--query-api-and-render-results (query search-type query-url buffer-name)
   ;; get json response from api
@@ -240,12 +285,12 @@ Use `which-key` if available, else display simple message in echo area"
   "Delete all network connections to khoj server"
   (dolist (proc (process-list))
     (let ((proc-buf (buffer-name (process-buffer proc)))
-          (khoj-network-proc-buf (string-join (split-string khoj--server-url "://") " ")))
+          (khoj-network-proc-buf (string-join (split-string khoj-server-url "://") " ")))
       (when (string-match (format "%s" khoj-network-proc-buf) proc-buf)
         (delete-process proc)))))
 
 (defun khoj--teardown-incremental-search ()
-  (message "[Khoj]: Teardown Incremental Search")
+  (message "Khoj: Teardown Incremental Search")
   ;; remove advice to rerank results on normal exit from minibuffer
   (advice-remove 'exit-minibuffer #'khoj--minibuffer-exit-advice)
   ;; unset khoj minibuffer window
@@ -262,6 +307,7 @@ Use `which-key` if available, else display simple message in echo area"
 (defun khoj--minibuffer-exit-advice (&rest _args)
   (khoj--incremental-search t))
 
+
 ;;;###autoload
 (defun khoj ()
   "Natural, Incremental Search for your personal notes, transactions and music using Khoj"
@@ -269,8 +315,8 @@ Use `which-key` if available, else display simple message in echo area"
   (let* ((khoj-buffer-name (get-buffer-create khoj--buffer-name)))
     ;; set khoj search type to last used or based on current buffer
     (setq khoj--search-type (or khoj--search-type (khoj--buffer-name-to-search-type (buffer-name))))
-    ;; setup rerank to improve results once user idle for KHOJ--RERANK-AFTER-IDLE-TIME seconds
+    ;; setup rerank to improve results once user idle for KHOJ-RERANK-AFTER-IDLE-TIME seconds
-    (setq khoj--rerank-timer (run-with-idle-timer khoj--rerank-after-idle-time t 'khoj--incremental-search t))
+    (setq khoj--rerank-timer (run-with-idle-timer khoj-rerank-after-idle-time t 'khoj--incremental-search t))
     ;; switch to khoj results buffer
     (switch-to-buffer khoj-buffer-name)
     ;; open and setup minibuffer for incremental search

src/processor/org_mode/orgnode.py

@@ -38,6 +38,8 @@ import datetime
 from pathlib import Path
 from os.path import relpath
 
+indent_regex = re.compile(r'^\s*')
+
 def normalize_filename(filename):
    file_relative_to_home = f'~/{relpath(filename, start=Path.home())}'
    escaped_filename = f'{file_relative_to_home}'.replace("[","\[").replace("]","\]")
@@ -370,7 +372,9 @@ class Orgnode(object):
         n = ''
         for _ in range(0, self.level):
            n = n + '*'
-        n = n + ' ' + self.todo + ' '
+        n = n + ' '
+        if self.todo:
+           n = n + self.todo + ' '
         if self.prty:
            n = n +  '[#' + self.prty + '] '
         n = n + self.headline
@@ -382,7 +386,12 @@ class Orgnode(object):
         n = n + closecolon
         n = n + "\n"
 
+        # Get body indentation from first line of body
+        indent = indent_regex.match(self.body).group()
+
         # Output Closed Date, Scheduled Date, Deadline Date
+        if self.closed or self.scheduled or self.deadline:
+           n = n + indent
         if self.closed:
            n = n + f'CLOSED: [{self.closed.strftime("%Y-%m-%d %a")}] '
         if self.scheduled:
@@ -393,10 +402,10 @@ class Orgnode(object):
            n = n + '\n'
 
         # Ouput Property Drawer
-        n = n + ":PROPERTIES:\n"
+        n = n + indent + ":PROPERTIES:\n"
         for key, value in self.properties.items():
-           n = n + f":{key}: {value}\n"
+           n = n + indent + f":{key}: {value}\n"
-        n = n + ":END:\n"
+        n = n + indent + ":END:\n"
 
         n = n + self.body
 

src/router.py

@@ -53,9 +53,11 @@ def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None, r: Opti
         print(f'No query param (q) passed in API call to initiate search')
         return {}
 
+    # initialize variables
     user_query = q
     results_count = n
     results = {}
+    query_start, query_end, collate_start, collate_end = None, None, None, None
 
     if (t == SearchType.Org or t == None) and state.model.orgmode_search:
         # query org-mode notes
@@ -119,8 +121,10 @@ def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None, r: Opti
         collate_end = time.time()
 
     if state.verbose > 1:
-        print(f"Query took {query_end - query_start:.3f} seconds")
+        if query_start and query_end:
-        print(f"Collating results took {collate_end - collate_start:.3f} seconds")
+            print(f"Query took {query_end - query_start:.3f} seconds")
+        if collate_start and collate_end:
+            print(f"Collating results took {collate_end - collate_start:.3f} seconds")
 
     return results
 

tests/test_orgnode.py

@@ -37,7 +37,7 @@ def test_parse_complete_entry(tmp_path):
     "Test parsing of entry with all important fields"
     # Arrange
     entry = f'''
-*** [#A] Heading   :Tag1:TAG2:tag3:
+*** DONE [#A] Heading   :Tag1:TAG2:tag3:
 CLOSED: [1984-04-01 Sun 12:00] SCHEDULED: <1984-04-01 Sun 09:00> DEADLINE: <1984-04-01 Sun>
 :PROPERTIES:
 :ID: 123-456-789-4234-1231
@@ -56,6 +56,7 @@ Body Line 2'''
     # Assert
     assert len(entries) == 1
     assert entries[0].Heading() == "Heading"
+    assert entries[0].Todo() == "DONE"
     assert entries[0].Tags() == {"Tag1", "TAG2", "tag3"}
     assert entries[0].Body() == "- Clocked Log 1\nBody Line 1\nBody Line 2"
     assert entries[0].Priority() == "A"
@@ -124,7 +125,7 @@ def test_parse_multiple_entries(tmp_path):
     "Test parsing of multiple entries"
     # Arrange
     content = f'''
-*** [#A] Heading1   :tag1:
+*** FAILED [#A] Heading1   :tag1:
 CLOSED: [1984-04-01 Sun 12:00] SCHEDULED: <1984-04-01 Sun 09:00> DEADLINE: <1984-04-01 Sun>
 :PROPERTIES:
 :ID: 123-456-789-4234-0001
@@ -135,7 +136,7 @@ CLOCK: [1984-04-01 Sun 09:00]--[1984-04-01 Sun 12:00] => 3:00
 :END:
 Body 1
 
-*** [#A] Heading2   :tag2:
+*** CANCELLED [#A] Heading2   :tag2:
 CLOSED: [1984-04-02 Sun 12:00] SCHEDULED: <1984-04-02 Sun 09:00> DEADLINE: <1984-04-02 Sun>
 :PROPERTIES:
 :ID: 123-456-789-4234-0002
@@ -156,6 +157,7 @@ Body 2
     assert len(entries) == 2
     for index, entry in enumerate(entries):
         assert entry.Heading() == f"Heading{index+1}"
+        assert entry.Todo() == "FAILED" if index == 0 else "CANCELLED"
         assert entry.Tags() == {f"tag{index+1}"}
         assert entry.Body() == f"- Clocked Log {index+1}\nBody {index+1}\n\n"
         assert entry.Priority() == "A"