klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-06 13:22:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Multi-User Part 4]: Authenticate using API Tokens (#513)

Browse Source 
### ✨ New
- Use API keys to authenticate from Desktop, Obsidian, Emacs clients
- Create API, UI on web app config page to CRUD API Keys
- Create user API keys table and functions to CRUD them in Database

### 🧪 Improve
- Default to better search model, [gte-small](https://huggingface.co/thenlper/gte-small), to improve search quality
- Only load chat model to GPU if enough space, throw error on load failure
- Show encoding progress, truncate headings to max chars supported
- Add instruction to create db in Django DB setup Readme

### ⚙️ Fix
- Fix error handling when configure offline chat via Web UI
- Do not warn in anon mode about Google OAuth env vars not being set
- Fix path to load static files when server started from project root
This commit is contained in:
Debanjum
2023-10-26 12:33:03 -07:00
committed by GitHub
parent 4b6ec248a6
commit 9acc722f7f
36 changed files with 692 additions and 564 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
src/interface/emacs/khoj.el
View File

@@ -93,7 +93,7 @@
:group 'khoj
:type 'number)
(defcustom khoj-server-api-key "secret"
(defcustom khoj-api-key nil
"API Key to Khoj server."
:group 'khoj
:type 'string)
@@ -246,26 +246,6 @@ for example), set this to the full interpreter path."
:type '(repeat string)
:group 'khoj)
(defcustom khoj-chat-model "gpt-3.5-turbo"
"Specify chat model to use for chat with khoj."
:type 'string
:group 'khoj)
(defcustom khoj-openai-api-key nil
"OpenAI API key used to configure chat on khoj server."
:type 'string
:group 'khoj)
(defcustom khoj-chat-offline nil
"Use offline model to chat with khoj."
:type 'boolean
:group 'khoj)
(defcustom khoj-offline-chat-model nil
"Specify chat model to use for offline chat with khoj."
:type 'string
:group 'khoj)
(defcustom khoj-auto-setup t
"Automate install, configure and start of khoj server.
Auto invokes setup steps on calling main entrypoint."
@@ -319,8 +299,7 @@ Auto invokes setup steps on calling main entrypoint."
:filter (lambda (process msg)
(cond ((string-match (format "Uvicorn running on %s" khoj-server-url) msg)
(progn
(setq khoj--server-ready? t)
(khoj--server-configure)))
(setq khoj--server-ready? t)))
((string-match "Batches: " msg)
(when (string-match "\\([0-9]+\\.[0-9]+\\|\\([0-9]+\\)\\)%?" msg)
(message "khoj.el: %s updating index %s"
@@ -383,106 +362,13 @@ Auto invokes setup steps on calling main entrypoint."
(when (not (khoj--server-started?))
(khoj--server-start)))
(defun khoj--get-directory-from-config (config keys &optional level)
"Extract directory under specified KEYS in CONFIG and trim it to LEVEL.
CONFIG is json obtained from Khoj config API."
(let ((item config))
(dolist (key keys)
(setq item (cdr (assoc key item))))
(-> item
(split-string "/")
(butlast (or level nil))
(string-join "/"))))
(defun khoj--server-configure ()
"Configure the Khoj server for search and chat."
(interactive)
(let* ((url-request-method "GET")
(current-config
(with-temp-buffer
(url-insert-file-contents (format "%s/api/config/data" khoj-server-url))
(ignore-error json-end-of-file (json-parse-buffer :object-type 'alist :array-type 'list :null-object json-null :false-object json-false))))
(default-config
(with-temp-buffer
(url-insert-file-contents (format "%s/api/config/data/default" khoj-server-url))
(ignore-error json-end-of-file (json-parse-buffer :object-type 'alist :array-type 'list :null-object json-null :false-object json-false))))
(default-chat-dir (khoj--get-directory-from-config default-config '(processor conversation conversation-logfile)))
(chat-model (or khoj-chat-model (alist-get 'chat-model (alist-get 'openai (alist-get 'conversation (alist-get 'processor default-config))))))
(enable-offline-chat (or khoj-chat-offline (alist-get 'enable-offline-chat (alist-get 'offline-chat (alist-get 'conversation (alist-get 'processor default-config))))))
(offline-chat-model (or khoj-offline-chat-model (alist-get 'chat-model (alist-get 'offline-chat (alist-get 'conversation (alist-get 'processor default-config))))))
(config (or current-config default-config)))
;; Configure processors
(cond
((not khoj-openai-api-key)
(let* ((processor (assoc 'processor config))
(conversation (assoc 'conversation processor))
(openai (assoc 'openai conversation)))
(when openai
;; Unset the `openai' field in the khoj conversation processor config
(message "khoj.el: Disable Chat using OpenAI as your OpenAI API key got removed from config")
(setcdr conversation (delq openai (cdr conversation)))
(push conversation (cdr processor))
(push processor config))))
;; If khoj backend isn't configured yet
((not current-config)
(message "khoj.el: Khoj not configured yet.")
(setq config (delq (assoc 'processor config) config))
(cl-pushnew `(processor . ((conversation . ((conversation-logfile . ,(format "%s/conversation.json" default-chat-dir))
(offline-chat . ((enable-offline-chat . ,enable-offline-chat)
(chat-model . ,offline-chat-model)))
(openai . ((chat-model . ,chat-model)
(api-key . ,khoj-openai-api-key)))))))
config))
;; Else if chat isn't configured in khoj backend
((not (alist-get 'conversation (alist-get 'processor config)))
(message "khoj.el: Chat not configured yet.")
(let ((new-processor-type (alist-get 'processor config)))
(setq new-processor-type (delq (assoc 'conversation new-processor-type) new-processor-type))
(cl-pushnew `(conversation . ((conversation-logfile . ,(format "%s/conversation.json" default-chat-dir))
(offline-chat . ((enable-offline-chat . ,enable-offline-chat)
(chat-model . ,offline-chat-model)))
(openai . ((chat-model . ,chat-model)
(api-key . ,khoj-openai-api-key)))))
new-processor-type)
(setq config (delq (assoc 'processor config) config))
(cl-pushnew `(processor . ,new-processor-type) config)))
;; Else if chat configuration in khoj backend has gone stale
((not (and (equal (alist-get 'api-key (alist-get 'openai (alist-get 'conversation (alist-get 'processor config)))) khoj-openai-api-key)
(equal (alist-get 'chat-model (alist-get 'openai (alist-get 'conversation (alist-get 'processor config)))) khoj-chat-model)
(equal (alist-get 'enable-offline-chat (alist-get 'offline-chat (alist-get 'conversation (alist-get 'processor config)))) enable-offline-chat)
(equal (alist-get 'chat-model (alist-get 'offline-chat (alist-get 'conversation (alist-get 'processor config)))) offline-chat-model)))
(message "khoj.el: Chat configuration has gone stale.")
(let* ((chat-directory (khoj--get-directory-from-config config '(processor conversation conversation-logfile)))
(new-processor-type (alist-get 'processor config)))
(setq new-processor-type (delq (assoc 'conversation new-processor-type) new-processor-type))
(cl-pushnew `(conversation . ((conversation-logfile . ,(format "%s/conversation.json" chat-directory))
(offline-chat . ((enable-offline-chat . ,enable-offline-chat)
(chat-model . ,offline-chat-model)))
(openai . ((chat-model . ,khoj-chat-model)
(api-key . ,khoj-openai-api-key)))))
new-processor-type)
(setq config (delq (assoc 'processor config) config))
(cl-pushnew `(processor . ,new-processor-type) config))))
;; Update server with latest configuration, if required
(cond ((not current-config)
(khoj--post-new-config config)
(message "khoj.el: ⚙️ Generated new khoj server configuration."))
((not (equal config current-config))
(khoj--post-new-config config)
(message "khoj.el: ⚙️ Updated khoj server configuration.")))))
(defun khoj-setup (&optional interact)
"Install, start and configure Khoj server. Get permission if INTERACT is non-nil."
"Install and start Khoj server. Get permission if INTERACT is non-nil."
(interactive "p")
;; Setup khoj server if not running
(let* ((not-started (not (khoj--server-started?)))
(permitted (if (and not-started interact)
(y-or-n-p "Could not connect to Khoj server. Should I install, start and configure it for you?")
(y-or-n-p "Could not connect to Khoj server. Should I install, start it for you?")
t)))
;; If user permits setup of khoj server from khoj.el
(when permitted
@@ -491,12 +377,9 @@ CONFIG is json obtained from Khoj config API."
(khoj--server-setup))
;; Wait until server is ready
;; As server can be started but not ready to use/configure
;; As server can be started but not ready to use
(while (not khoj--server-ready?)
(sit-for 0.5))
;; Configure server once it's ready
(khoj--server-configure))))
(sit-for 0.5)))))
;; -------------------
@@ -516,7 +399,7 @@ CONFIG is json obtained from Khoj config API."
(let ((url-request-method "POST")
(url-request-data (khoj--render-files-as-request-body files-to-index khoj--indexed-files boundary))
(url-request-extra-headers `(("content-type" . ,(format "multipart/form-data; boundary=%s" boundary))
("x-api-key" . ,khoj-server-api-key))))
("Authorization" . ,(format "Bearer %s" khoj-api-key)))))
(with-current-buffer
(url-retrieve (format "%s/api/v1/index/update?%s&force=%s&client=emacs" khoj-server-url type-query (or force "false"))
;; render response from indexing API endpoint on server
@@ -690,19 +573,22 @@ Use `BOUNDARY' to separate files. This is sent to Khoj server as a POST request.
"Configure khoj server with provided CONFIG."
;; POST provided config to khoj server
(let ((url-request-method "POST")
(url-request-extra-headers '(("Content-Type" . "application/json")))
(url-request-extra-headers `(("Content-Type" . "application/json")
("Authorization" . ,(format "Bearer %s" khoj-api-key))))
(url-request-data (encode-coding-string (json-encode-alist config) 'utf-8))
(config-url (format "%s/api/config/data" khoj-server-url)))
(with-current-buffer (url-retrieve-synchronously config-url)
(buffer-string)))
;; Update index on khoj server after configuration update
(let ((khoj--server-ready? nil))
(let ((khoj--server-ready? nil)
(url-request-extra-headers `(("Authorization" . ,(format "\"Bearer %s\"" khoj-api-key)))))
(url-retrieve (format "%s/api/update?client=emacs" khoj-server-url) #'identity)))
(defun khoj--get-enabled-content-types ()
"Get content types enabled for search from API."
(let ((config-url (format "%s/api/config/types" khoj-server-url))
(url-request-method "GET"))
(url-request-method "GET")
(url-request-extra-headers `(("Authorization" . ,(format "Bearer %s" khoj-api-key)))))
(with-temp-buffer
(url-insert-file-contents config-url)
(thread-last
@@ -722,7 +608,8 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
;; get json response from api
(with-current-buffer buffer-name
(let ((inhibit-read-only t)
(url-request-method "GET"))
(url-request-method "GET")
(url-request-extra-headers `(("Authorization" . ,(format "Bearer %s" khoj-api-key)))))
(erase-buffer)
(url-insert-file-contents query-url)))
;; render json response into formatted entries
@@ -848,6 +735,7 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
"Send QUERY to Khoj Chat API."
(let* ((url-request-method "GET")
(encoded-query (url-hexify-string query))
(url-request-extra-headers `(("Authorization" . ,(format "Bearer %s" khoj-api-key))))
(query-url (format "%s/api/chat?q=%s&n=%s&client=emacs" khoj-server-url encoded-query khoj-results-count)))
(with-temp-buffer
(condition-case ex
@@ -862,6 +750,7 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
(defun khoj--get-chat-history-api ()
"Send QUERY to Khoj Chat History API."
(let* ((url-request-method "GET")
(url-request-extra-headers `(("Authorization" . ,(format "Bearer %s" khoj-api-key))))
(query-url (format "%s/api/chat/history?client=emacs" khoj-server-url)))
(with-temp-buffer
(condition-case ex