klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-03 13:19:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add multi-user support to Khoj and use Postgres for backend storage (#549)

Browse Source 
- Adds support for multiple users to be connected to the same Khoj instance using their Google login credentials
- Moves storage solution from in-memory json data to a Postgres db. This stores all relevant information, including accounts, embeddings, chat history, server side chat configuration
- Adds the concept of a Khoj server admin for configuring instance-wide settings regarding search model, and chat configuration
- Miscellaneous updates and fixes to the UX, including chat references, colors, and an updated config page
- Adds billing to allow users to subscribe to the cloud service easily
- Adds a separate GitHub action for building the dockerized production (tag `prod`) and dev (tag `dev`) images, separate from the image used for local building. The production image uses `gunicorn` with multiple workers to run the server.
- Updates all clients (Obsidian, Emacs, Desktop) to follow the client/server architecture. The server no longer reads from the file system at all; it only accepts data via the indexer API. In line with that, removes the functionality to configure org, markdown, plaintext, or other file-specific settings in the server. Only leaves GitHub and Notion for server-side configuration.
- Changes license to GNU AGPLv3

Resolves #467 
Resolves #488 
Resolves #303 
Resolves #345 
Resolves #195 
Resolves #280 
Resolves #461 
Closes #259 
Resolves #351
Resolves #301
Resolves #296
This commit is contained in:
sabaimran
2023-11-16 11:48:01 -08:00
committed by GitHub
parent 39ad1c6ce6 1466aef554
commit e8a13f0813
147 changed files with 8704 additions and 4450 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
src/interface/emacs/khoj.el
View File

@@ -93,7 +93,7 @@
:group 'khoj
:type 'number)
(defcustom khoj-server-api-key "secret"
(defcustom khoj-api-key nil
"API Key to Khoj server."
:group 'khoj
:type 'string)
@@ -246,26 +246,6 @@ for example), set this to the full interpreter path."
:type '(repeat string)
:group 'khoj)
(defcustom khoj-chat-model "gpt-3.5-turbo"
"Specify chat model to use for chat with khoj."
:type 'string
:group 'khoj)
(defcustom khoj-openai-api-key nil
"OpenAI API key used to configure chat on khoj server."
:type 'string
:group 'khoj)
(defcustom khoj-chat-offline nil
"Use offline model to chat with khoj."
:type 'boolean
:group 'khoj)
(defcustom khoj-offline-chat-model nil
"Specify chat model to use for offline chat with khoj."
:type 'string
:group 'khoj)
(defcustom khoj-auto-setup t
"Automate install, configure and start of khoj server.
Auto invokes setup steps on calling main entrypoint."
@@ -319,8 +299,7 @@ Auto invokes setup steps on calling main entrypoint."
:filter (lambda (process msg)
(cond ((string-match (format "Uvicorn running on %s" khoj-server-url) msg)
(progn
(setq khoj--server-ready? t)
(khoj--server-configure)))
(setq khoj--server-ready? t)))
((string-match "Batches: " msg)
(when (string-match "\\([0-9]+\\.[0-9]+\\|\\([0-9]+\\)\\)%?" msg)
(message "khoj.el: %s updating index %s"
@@ -383,106 +362,13 @@ Auto invokes setup steps on calling main entrypoint."
(when (not (khoj--server-started?))
(khoj--server-start)))
(defun khoj--get-directory-from-config (config keys &optional level)
"Extract directory under specified KEYS in CONFIG and trim it to LEVEL.
CONFIG is json obtained from Khoj config API."
(let ((item config))
(dolist (key keys)
(setq item (cdr (assoc key item))))
(-> item
(split-string "/")
(butlast (or level nil))
(string-join "/"))))
(defun khoj--server-configure ()
"Configure the Khoj server for search and chat."
(interactive)
(let* ((url-request-method "GET")
(current-config
(with-temp-buffer
(url-insert-file-contents (format "%s/api/config/data" khoj-server-url))
(ignore-error json-end-of-file (json-parse-buffer :object-type 'alist :array-type 'list :null-object json-null :false-object json-false))))
(default-config
(with-temp-buffer
(url-insert-file-contents (format "%s/api/config/data/default" khoj-server-url))
(ignore-error json-end-of-file (json-parse-buffer :object-type 'alist :array-type 'list :null-object json-null :false-object json-false))))
(default-chat-dir (khoj--get-directory-from-config default-config '(processor conversation conversation-logfile)))
(chat-model (or khoj-chat-model (alist-get 'chat-model (alist-get 'openai (alist-get 'conversation (alist-get 'processor default-config))))))
(enable-offline-chat (or khoj-chat-offline (alist-get 'enable-offline-chat (alist-get 'offline-chat (alist-get 'conversation (alist-get 'processor default-config))))))
(offline-chat-model (or khoj-offline-chat-model (alist-get 'chat-model (alist-get 'offline-chat (alist-get 'conversation (alist-get 'processor default-config))))))
(config (or current-config default-config)))
;; Configure processors
(cond
((not khoj-openai-api-key)
(let* ((processor (assoc 'processor config))
(conversation (assoc 'conversation processor))
(openai (assoc 'openai conversation)))
(when openai
;; Unset the `openai' field in the khoj conversation processor config
(message "khoj.el: Disable Chat using OpenAI as your OpenAI API key got removed from config")
(setcdr conversation (delq openai (cdr conversation)))
(push conversation (cdr processor))
(push processor config))))
;; If khoj backend isn't configured yet
((not current-config)
(message "khoj.el: Khoj not configured yet.")
(setq config (delq (assoc 'processor config) config))
(cl-pushnew `(processor . ((conversation . ((conversation-logfile . ,(format "%s/conversation.json" default-chat-dir))
(offline-chat . ((enable-offline-chat . ,enable-offline-chat)
(chat-model . ,offline-chat-model)))
(openai . ((chat-model . ,chat-model)
(api-key . ,khoj-openai-api-key)))))))
config))
;; Else if chat isn't configured in khoj backend
((not (alist-get 'conversation (alist-get 'processor config)))
(message "khoj.el: Chat not configured yet.")
(let ((new-processor-type (alist-get 'processor config)))
(setq new-processor-type (delq (assoc 'conversation new-processor-type) new-processor-type))
(cl-pushnew `(conversation . ((conversation-logfile . ,(format "%s/conversation.json" default-chat-dir))
(offline-chat . ((enable-offline-chat . ,enable-offline-chat)
(chat-model . ,offline-chat-model)))
(openai . ((chat-model . ,chat-model)
(api-key . ,khoj-openai-api-key)))))
new-processor-type)
(setq config (delq (assoc 'processor config) config))
(cl-pushnew `(processor . ,new-processor-type) config)))
;; Else if chat configuration in khoj backend has gone stale
((not (and (equal (alist-get 'api-key (alist-get 'openai (alist-get 'conversation (alist-get 'processor config)))) khoj-openai-api-key)
(equal (alist-get 'chat-model (alist-get 'openai (alist-get 'conversation (alist-get 'processor config)))) khoj-chat-model)
(equal (alist-get 'enable-offline-chat (alist-get 'offline-chat (alist-get 'conversation (alist-get 'processor config)))) enable-offline-chat)
(equal (alist-get 'chat-model (alist-get 'offline-chat (alist-get 'conversation (alist-get 'processor config)))) offline-chat-model)))
(message "khoj.el: Chat configuration has gone stale.")
(let* ((chat-directory (khoj--get-directory-from-config config '(processor conversation conversation-logfile)))
(new-processor-type (alist-get 'processor config)))
(setq new-processor-type (delq (assoc 'conversation new-processor-type) new-processor-type))
(cl-pushnew `(conversation . ((conversation-logfile . ,(format "%s/conversation.json" chat-directory))
(offline-chat . ((enable-offline-chat . ,enable-offline-chat)
(chat-model . ,offline-chat-model)))
(openai . ((chat-model . ,khoj-chat-model)
(api-key . ,khoj-openai-api-key)))))
new-processor-type)
(setq config (delq (assoc 'processor config) config))
(cl-pushnew `(processor . ,new-processor-type) config))))
;; Update server with latest configuration, if required
(cond ((not current-config)
(khoj--post-new-config config)
(message "khoj.el: ⚙️ Generated new khoj server configuration."))
((not (equal config current-config))
(khoj--post-new-config config)
(message "khoj.el: ⚙️ Updated khoj server configuration.")))))
(defun khoj-setup (&optional interact)
"Install, start and configure Khoj server. Get permission if INTERACT is non-nil."
"Install and start Khoj server. Get permission if INTERACT is non-nil."
(interactive "p")
;; Setup khoj server if not running
(let* ((not-started (not (khoj--server-started?)))
(permitted (if (and not-started interact)
(y-or-n-p "Could not connect to Khoj server. Should I install, start and configure it for you?")
(y-or-n-p "Could not connect to Khoj server. Should I install, start it for you?")
t)))
;; If user permits setup of khoj server from khoj.el
(when permitted
@@ -491,12 +377,9 @@ CONFIG is json obtained from Khoj config API."
(khoj--server-setup))
;; Wait until server is ready
;; As server can be started but not ready to use/configure
;; As server can be started but not ready to use
(while (not khoj--server-ready?)
(sit-for 0.5))
;; Configure server once it's ready
(khoj--server-configure))))
(sit-for 0.5)))))
;; -------------------
@@ -516,7 +399,7 @@ CONFIG is json obtained from Khoj config API."
(let ((url-request-method "POST")
(url-request-data (khoj--render-files-as-request-body files-to-index khoj--indexed-files boundary))
(url-request-extra-headers `(("content-type" . ,(format "multipart/form-data; boundary=%s" boundary))
("x-api-key" . ,khoj-server-api-key))))
("Authorization" . ,(format "Bearer %s" khoj-api-key)))))
(with-current-buffer
(url-retrieve (format "%s/api/v1/index/update?%s&force=%s&client=emacs" khoj-server-url type-query (or force "false"))
;; render response from indexing API endpoint on server
@@ -690,19 +573,22 @@ Use `BOUNDARY' to separate files. This is sent to Khoj server as a POST request.
"Configure khoj server with provided CONFIG."
;; POST provided config to khoj server
(let ((url-request-method "POST")
(url-request-extra-headers '(("Content-Type" . "application/json")))
(url-request-extra-headers `(("Content-Type" . "application/json")
("Authorization" . ,(format "Bearer %s" khoj-api-key))))
(url-request-data (encode-coding-string (json-encode-alist config) 'utf-8))
(config-url (format "%s/api/config/data" khoj-server-url)))
(with-current-buffer (url-retrieve-synchronously config-url)
(buffer-string)))
;; Update index on khoj server after configuration update
(let ((khoj--server-ready? nil))
(let ((khoj--server-ready? nil)
(url-request-extra-headers `(("Authorization" . ,(format "\"Bearer %s\"" khoj-api-key)))))
(url-retrieve (format "%s/api/update?client=emacs" khoj-server-url) #'identity)))
(defun khoj--get-enabled-content-types ()
"Get content types enabled for search from API."
(let ((config-url (format "%s/api/config/types" khoj-server-url))
(url-request-method "GET"))
(url-request-method "GET")
(url-request-extra-headers `(("Authorization" . ,(format "Bearer %s" khoj-api-key)))))
(with-temp-buffer
(url-insert-file-contents config-url)
(thread-last
@@ -722,7 +608,8 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
;; get json response from api
(with-current-buffer buffer-name
(let ((inhibit-read-only t)
(url-request-method "GET"))
(url-request-method "GET")
(url-request-extra-headers `(("Authorization" . ,(format "Bearer %s" khoj-api-key)))))
(erase-buffer)
(url-insert-file-contents query-url)))
;; render json response into formatted entries
@@ -848,6 +735,7 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
"Send QUERY to Khoj Chat API."
(let* ((url-request-method "GET")
(encoded-query (url-hexify-string query))
(url-request-extra-headers `(("Authorization" . ,(format "Bearer %s" khoj-api-key))))
(query-url (format "%s/api/chat?q=%s&n=%s&client=emacs" khoj-server-url encoded-query khoj-results-count)))
(with-temp-buffer
(condition-case ex
@@ -862,6 +750,7 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
(defun khoj--get-chat-history-api ()
"Send QUERY to Khoj Chat History API."
(let* ((url-request-method "GET")
(url-request-extra-headers `(("Authorization" . ,(format "Bearer %s" khoj-api-key))))
(query-url (format "%s/api/chat/history?client=emacs" khoj-server-url)))
(with-temp-buffer
(condition-case ex