Add multi-user support to Khoj and use Postgres for backend storage (#549)

- Adds support for multiple users to be connected to the same Khoj instance using their Google login credentials
- Moves storage solution from in-memory json data to a Postgres db. This stores all relevant information, including accounts, embeddings, chat history, server side chat configuration
- Adds the concept of a Khoj server admin for configuring instance-wide settings regarding search model, and chat configuration
- Miscellaneous updates and fixes to the UX, including chat references, colors, and an updated config page
- Adds billing to allow users to subscribe to the cloud service easily
- Adds a separate GitHub action for building the dockerized production (tag `prod`) and dev (tag `dev`) images, separate from the image used for local building. The production image uses `gunicorn` with multiple workers to run the server.
- Updates all clients (Obsidian, Emacs, Desktop) to follow the client/server architecture. The server no longer reads from the file system at all; it only accepts data via the indexer API. In line with that, removes the functionality to configure org, markdown, plaintext, or other file-specific settings in the server. Only leaves GitHub and Notion for server-side configuration.
- Changes license to GNU AGPLv3

Resolves #467 
Resolves #488 
Resolves #303 
Resolves #345 
Resolves #195 
Resolves #280 
Resolves #461 
Closes #259 
Resolves #351
Resolves #301
Resolves #296

src/interface/obsidian/src/chat_modal.ts

@@ -142,7 +142,8 @@ export class KhojChatModal extends Modal {
     async getChatHistory(): Promise<void> {
         // Get chat history from Khoj backend
         let chatUrl = `${this.setting.khojUrl}/api/chat/history?client=obsidian`;
-        let response = await request(chatUrl);
+        let headers = { "Authorization": `Bearer ${this.setting.khojApiKey}` };
+        let response = await request({ url: chatUrl, headers: headers });
         let chatLogs = JSON.parse(response).response;
         chatLogs.forEach((chatLog: any) => {
             this.renderMessageWithReferences(chatLog.message, chatLog.by, chatLog.context, new Date(chatLog.created));
@@ -168,7 +169,8 @@ export class KhojChatModal extends Modal {
             method: "GET",
             headers: {
                 "Access-Control-Allow-Origin": "*",
-                "Content-Type": "text/event-stream"
+                "Content-Type": "text/event-stream",
+                "Authorization": `Bearer ${this.setting.khojApiKey}`,
             },
         })
 

src/interface/obsidian/src/main.ts

@@ -1,8 +1,8 @@
-import { Notice, Plugin, TFile } from 'obsidian';
+import { Notice, Plugin, request } from 'obsidian';
 import { KhojSetting, KhojSettingTab, DEFAULT_SETTINGS } from 'src/settings'
 import { KhojSearchModal } from 'src/search_modal'
 import { KhojChatModal } from 'src/chat_modal'
-import { configureKhojBackend, updateContentIndex } from './utils';
+import { updateContentIndex } from './utils';
 
 
 export default class Khoj extends Plugin {
@@ -39,9 +39,9 @@ export default class Khoj extends Plugin {
             id: 'chat',
             name: 'Chat',
             checkCallback: (checking) => {
-                if (!checking && this.settings.connectedToBackend && (!!this.settings.openaiApiKey || this.settings.enableOfflineChat))
+                if (!checking && this.settings.connectedToBackend)
                     new KhojChatModal(this.app, this.settings).open();
-                return !!this.settings.openaiApiKey || this.settings.enableOfflineChat;
+                return this.settings.connectedToBackend;
             }
         });
 
@@ -70,16 +70,27 @@ export default class Khoj extends Plugin {
         // Load khoj obsidian plugin settings
         this.settings = Object.assign({}, DEFAULT_SETTINGS, await this.loadData());
 
-        if (this.settings.autoConfigure) {
-            // Load, configure khoj server settings
-            await configureKhojBackend(this.app.vault, this.settings);
+        // Check if khoj backend is configured, note if cannot connect to backend
+        let headers = { "Authorization": `Bearer ${this.settings.khojApiKey}` };
+
+        if (this.settings.khojUrl === "https://app.khoj.dev") {
+            if (this.settings.khojApiKey === "") {
+                new Notice(`❗️Khoj API key is not configured. Please visit https://app.khoj.dev to get an API key.`);
+                return;
+            }
+
+            await request({ url: this.settings.khojUrl ,method: "GET", headers: headers })
+                .then(response => {
+                    this.settings.connectedToBackend = true;
+                })
+                .catch(error => {
+                    this.settings.connectedToBackend = false;
+                    new Notice(`❗️Ensure Khoj backend is running and Khoj URL is pointing to it in the plugin settings.\n\n${error}`);
+                });
         }
     }
 
     async saveSettings() {
-        if (this.settings.autoConfigure) {
-            await configureKhojBackend(this.app.vault, this.settings, false);
-        }
         this.saveData(this.settings);
     }
 

src/interface/obsidian/src/search_modal.ts

@@ -90,10 +90,11 @@ export class KhojSearchModal extends SuggestModal<SearchResult> {
         // Query Khoj backend for search results
         let encodedQuery = encodeURIComponent(query);
         let searchUrl = `${this.setting.khojUrl}/api/search?q=${encodedQuery}&n=${this.setting.resultsCount}&r=${this.rerank}&client=obsidian`;
+        let headers = { 'Authorization': `Bearer ${this.setting.khojApiKey}` }
 
         // Get search results for markdown and pdf files
-        let mdResponse = await request(`${searchUrl}&t=markdown`);
-        let pdfResponse = await request(`${searchUrl}&t=pdf`);
+        let mdResponse = await request({ url: `${searchUrl}&t=markdown`, headers: headers });
+        let pdfResponse = await request({ url: `${searchUrl}&t=pdf`, headers: headers });
 
         // Parse search results
         let mdData = JSON.parse(mdResponse)

src/interface/obsidian/src/settings.ts

@@ -3,22 +3,20 @@ import Khoj from 'src/main';
 import { updateContentIndex } from './utils';
 
 export interface KhojSetting {
-    enableOfflineChat: boolean;
-    openaiApiKey: string;
     resultsCount: number;
     khojUrl: string;
+    khojApiKey: string;
     connectedToBackend: boolean;
     autoConfigure: boolean;
     lastSyncedFiles: TFile[];
 }
 
 export const DEFAULT_SETTINGS: KhojSetting = {
-    enableOfflineChat: false,
     resultsCount: 6,
     khojUrl: 'http://127.0.0.1:42110',
+    khojApiKey: '',
     connectedToBackend: false,
     autoConfigure: true,
-    openaiApiKey: '',
     lastSyncedFiles: []
 }
 
@@ -49,21 +47,12 @@ export class KhojSettingTab extends PluginSettingTab {
                     containerEl.firstElementChild?.setText(this.getBackendStatusMessage());
                 }));
         new Setting(containerEl)
-            .setName('OpenAI API Key')
-            .setDesc('Use OpenAI for Khoj Chat with your API key.')
+            .setName('Khoj API Key')
+            .setDesc('Use Khoj Cloud with your Khoj API Key')
             .addText(text => text
-                .setValue(`${this.plugin.settings.openaiApiKey}`)
+                .setValue(`${this.plugin.settings.khojApiKey}`)
                 .onChange(async (value) => {
-                    this.plugin.settings.openaiApiKey = value.trim();
-                    await this.plugin.saveSettings();
-                }));
-        new Setting(containerEl)
-            .setName('Enable Offline Chat')
-            .setDesc('Chat privately without an internet connection. Enabling this will use offline chat even if OpenAI is configured.')
-            .addToggle(toggle => toggle
-                .setValue(this.plugin.settings.enableOfflineChat)
-                .onChange(async (value) => {
-                    this.plugin.settings.enableOfflineChat = value;
+                    this.plugin.settings.khojApiKey = value.trim();
                     await this.plugin.saveSettings();
                 }));
         new Setting(containerEl)
@@ -78,8 +67,8 @@ export class KhojSettingTab extends PluginSettingTab {
                     await this.plugin.saveSettings();
                 }));
         new Setting(containerEl)
-            .setName('Auto Configure')
-            .setDesc('Automatically configure the Khoj backend.')
+            .setName('Auto Sync')
+            .setDesc('Automatically index your vault with Khoj.')
             .addToggle(toggle => toggle
                 .setValue(this.plugin.settings.autoConfigure)
                 .onChange(async (value) => {
@@ -88,7 +77,7 @@ export class KhojSettingTab extends PluginSettingTab {
                 }));
         let indexVaultSetting = new Setting(containerEl);
         indexVaultSetting
-            .setName('Index Vault')
+            .setName('Force Sync')
             .setDesc('Manually force Khoj to re-index your Obsidian Vault.')
             .addButton(button => button
                 .setButtonText('Update')

src/interface/obsidian/src/utils.ts

@@ -1,4 +1,4 @@
-import { FileSystemAdapter, Notice, RequestUrlParam, request, Vault, Modal, TFile } from 'obsidian';
+import { FileSystemAdapter, Notice, Vault, Modal, TFile } from 'obsidian';
 import { KhojSetting } from 'src/settings'
 
 export function getVaultAbsolutePath(vault: Vault): string {
@@ -9,26 +9,6 @@ export function getVaultAbsolutePath(vault: Vault): string {
     return '';
 }
 
-type OpenAIType = null | {
-    "chat-model": string;
-    "api-key": string;
-};
-
-type OfflineChatType = null | {
-    "chat-model": string;
-    "enable-offline-chat": boolean;
-};
-
-interface ProcessorData {
-    conversation: {
-      "conversation-logfile": string;
-      openai: OpenAIType;
-      "offline-chat": OfflineChatType;
-      "tokenizer": null | string;
-      "max-prompt-size": null | number;
-    };
-}
-
 function fileExtensionToMimeType (extension: string): string {
     switch (extension) {
         case 'pdf':
@@ -78,7 +58,7 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las
     const response = await fetch(`${setting.khojUrl}/api/v1/index/update?force=${regenerate}&client=obsidian`, {
         method: 'POST',
         headers: {
-            'x-api-key': 'secret',
+            'Authorization': `Bearer ${setting.khojApiKey}`,
         },
         body: formData,
     });
@@ -92,100 +72,6 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las
     return files;
 }
 
-export async function configureKhojBackend(vault: Vault, setting: KhojSetting, notify: boolean = true) {
-    let khojConfigUrl = `${setting.khojUrl}/api/config/data`;
-
-    // Check if khoj backend is configured, note if cannot connect to backend
-    let khoj_already_configured = await request(khojConfigUrl)
-        .then(response => {
-            setting.connectedToBackend = true;
-            return response !== "null"
-        })
-        .catch(error => {
-            setting.connectedToBackend = false;
-            if (notify)
-                new Notice(`❗️Ensure Khoj backend is running and Khoj URL is pointing to it in the plugin settings.\n\n${error}`);
-        })
-    // Short-circuit configuring khoj if unable to connect to khoj backend
-    if (!setting.connectedToBackend) return;
-
-    // Set index name from the path of the current vault
-    // Get default config fields from khoj backend
-    let defaultConfig = await request(`${khojConfigUrl}/default`).then(response => JSON.parse(response));
-    let khojDefaultChatDirectory = getIndexDirectoryFromBackendConfig(defaultConfig["processor"]["conversation"]["conversation-logfile"]);
-    let khojDefaultOpenAIChatModelName = defaultConfig["processor"]["conversation"]["openai"]["chat-model"];
-    let khojDefaultOfflineChatModelName = defaultConfig["processor"]["conversation"]["offline-chat"]["chat-model"];
-
-    // Get current config if khoj backend configured, else get default config from khoj backend
-    await request(khoj_already_configured ? khojConfigUrl : `${khojConfigUrl}/default`)
-        .then(response => JSON.parse(response))
-        .then(data => {
-            let conversationLogFile = data?.["processor"]?.["conversation"]?.["conversation-logfile"] ?? `${khojDefaultChatDirectory}/conversation.json`;
-            let processorData: ProcessorData = {
-                "conversation": {
-                    "conversation-logfile": conversationLogFile,
-                    "openai": null,
-                    "offline-chat": {
-                        "chat-model": khojDefaultOfflineChatModelName,
-                        "enable-offline-chat": setting.enableOfflineChat,
-                    },
-                    "tokenizer": null,
-                    "max-prompt-size": null,
-                }
-            }
-
-            // If the Open AI API Key was configured in the plugin settings
-            if (!!setting.openaiApiKey) {
-                let openAIChatModel = data?.["processor"]?.["conversation"]?.["openai"]?.["chat-model"] ?? khojDefaultOpenAIChatModelName;
-                processorData = {
-                    "conversation": {
-                        "conversation-logfile": conversationLogFile,
-                        "openai": {
-                            "chat-model": openAIChatModel,
-                            "api-key": setting.openaiApiKey,
-                        },
-                        "offline-chat": {
-                            "chat-model": khojDefaultOfflineChatModelName,
-                            "enable-offline-chat": setting.enableOfflineChat,
-                        },
-                        "tokenizer": null,
-                        "max-prompt-size": null,
-                    },
-                }
-            }
-
-            // Set khoj processor config to conversation processor config
-            data["processor"] = processorData;
-
-            // Save updated config and refresh index on khoj backend
-            updateKhojBackend(setting.khojUrl, data);
-            if (!khoj_already_configured)
-                console.log(`Khoj: Created khoj backend config:\n${JSON.stringify(data)}`)
-            else
-                console.log(`Khoj: Updated khoj backend config:\n${JSON.stringify(data)}`)
-        })
-        .catch(error => {
-            if (notify)
-                new Notice(`❗️Failed to configure Khoj backend. Contact developer on Github.\n\nError: ${error}`);
-        })
-}
-
-export async function updateKhojBackend(khojUrl: string, khojConfig: Object) {
-    // POST khojConfig to khojConfigUrl
-    let requestContent: RequestUrlParam = {
-        url: `${khojUrl}/api/config/data`,
-        body: JSON.stringify(khojConfig),
-        method: 'POST',
-        contentType: 'application/json',
-    };
-    // Save khojConfig on khoj backend at khojConfigUrl
-    request(requestContent);
-}
-
-function getIndexDirectoryFromBackendConfig(filepath: string) {
-    return filepath.split("/").slice(0, -1).join("/");
-}
-
 export async function createNote(name: string, newLeaf = false): Promise<void> {
     try {
       let pathPrefix: string

src/interface/obsidian/styles.css

@@ -8,7 +8,7 @@ If your plugin does not need CSS, delete this file.
 */
 
 :root {
-    --khoj-chat-primary: #ffb300;
+    --khoj-chat-primary: #fee285;
     --khoj-chat-dark-grey: #475569;
  }