Support Natural Search on Markdown Files

- Reason:
  Allow natural search on markdown based notes, documentation,
  websites etc

- Details:
  - Create markdown processor to extract Markdown entries (identified by
    Heading) into standard jsonl format required by text_search
  - Update API, Configs to support interfacing with new markdown type
  - Update Emacs, Web clients to support interfacing with new markdown
    type via API
  - Update Readme to mentiond markdown is also supported

Closes #35

src/interface/emacs/khoj.el

@@ -3,8 +3,8 @@
 ;; Copyright (C) 2021-2022 Debanjum Singh Solanky
 
 ;; Author: Debanjum Singh Solanky <debanjum@gmail.com>
-;; Version: 0.1
-;; Keywords: search, org-mode, outlines
+;; Version: 1.0
+;; Keywords: search, org-mode, outlines, markdown, image
 ;; URL: http://github.com/debanjum/khoj/interface/emacs
 
 ;; This file is NOT part of GNU Emacs.
@@ -27,7 +27,7 @@
 ;;; Commentary:
 
 ;; This package provides natural language search on org-mode notes,
-;; beancount transactions and images.
+;; markdown files, beancount transactions and images.
 ;; It is a wrapper that interfaces with transformer based ML models.
 ;; The models search capabilities are exposed via the Khoj HTTP API
 
@@ -46,6 +46,18 @@
   :group 'khoj
   :type 'integer)
 
+(defun khoj--extract-entries-as-markdown (json-response query)
+  "Convert json response from API to markdown entries"
+  ;; remove leading (, ) or SPC from extracted entries string
+  (replace-regexp-in-string
+   "^[\(\) ]" ""
+   ;; extract entries from response as single string and convert to entries
+   (format "# %s\n%s"
+           query
+           (mapcar
+            (lambda (args) (format "%s" (cdr (assoc 'entry args))))
+            json-response))))
+
 (defun khoj--extract-entries-as-org (json-response query)
   "Convert json response from API to org-mode entries"
   ;; remove leading (, ) or SPC from extracted entries string
@@ -59,7 +71,7 @@
             json-response))))
 
 (defun khoj--extract-entries-as-images (json-response query)
-  "Convert json response from API to org-mode entries with images"
+  "Convert json response from API to html with images"
   ;; remove leading (, ) or SPC from extracted entries string
   (replace-regexp-in-string
    "[\(\) ]$" ""
@@ -103,6 +115,7 @@
      ((equal buffer-name "Music.org") "music")
      ((equal file-extension "bean") "ledger")
      ((equal file-extension "org") "notes")
+     ((or (equal file-extension "markdown") (equal file-extension "md")) "markdown")
      (t "notes"))))
 
 (defun khoj--construct-api-query (query search-type)
@@ -111,10 +124,10 @@
 
 ;;;###autoload
 (defun khoj (query)
-  "Khoj on org-mode content via khoj API"
+  "Search your content naturally using the Khoj API"
   (interactive "sQuery: ")
   (let* ((default-type (khoj--buffer-name-to-search-type (buffer-name)))
-         (search-type (completing-read "Type: " '("notes" "ledger" "music" "image") nil t default-type))
+         (search-type (completing-read "Type: " '("notes" "markdown" "ledger" "music" "image") nil t default-type))
          (url (khoj--construct-api-query query search-type))
          (buff (get-buffer-create (format "*Khoj (q:%s t:%s)*" query search-type))))
     ;; get json response from api
@@ -122,17 +135,19 @@
       (let ((inhibit-read-only t))
         (erase-buffer)
         (url-insert-file-contents url)))
-    ;; convert json response to org-mode entries
+    ;; render json response into formatted entries
     (with-current-buffer buff
       (let ((inhibit-read-only t)
             (json-response (json-parse-buffer :object-type 'alist)))
         (erase-buffer)
         (insert
          (cond ((or (equal search-type "notes") (equal search-type "music")) (khoj--extract-entries-as-org json-response query))
+               ((equal search-type "markdown") (khoj--extract-entries-as-markdown json-response query))
                ((equal search-type "ledger") (khoj--extract-entries-as-ledger json-response query))
                ((equal search-type "image") (khoj--extract-entries-as-images json-response query))
                (t (format "%s" json-response))))
       (cond ((equal search-type "notes") (org-mode))
+            ((equal search-type "markdown") (markdown-mode))
             ((equal search-type "ledger") (beancount-mode))
             ((equal search-type "music") (progn (org-mode)
                                                 (org-music-mode)))

src/interface/web/index.html

@@ -63,6 +63,7 @@
         <select id="type">
             <option value="image">Image</option>
             <option value="notes">Notes</option>
+            <option value="markdown">Markdown</option>
             <option value="ledger">Ledger</option>
             <option value="music">Music</option>
         </select>