Merge branch 'master' of github.com:khoj-ai/khoj into features/add-support-for-mermaidjs

.github/workflows/desktop.yml

@@ -60,7 +60,7 @@ jobs:
 
       - name: ⏫ Upload Mac ARM App
         if: startsWith(github.ref, 'refs/tags/')
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           if-no-files-found: warn
           name: khoj-${{ github.ref_name }}-arm64.dmg
@@ -68,7 +68,7 @@ jobs:
 
       - name: ⏫ Upload Mac x64 App
         if: startsWith(github.ref, 'refs/tags/')
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           if-no-files-found: warn
           name: khoj-${{ github.ref_name }}-x64.dmg
@@ -76,7 +76,7 @@ jobs:
 
       - name: ⏫ Upload Windows App
         if: startsWith(github.ref, 'refs/tags/')
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           if-no-files-found: warn
           name: khoj-${{ github.ref_name }}-x64.exe
@@ -84,7 +84,7 @@ jobs:
 
       - name: ⏫ Upload Debian App
         if: startsWith(github.ref, 'refs/tags/')
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           if-no-files-found: warn
           name: khoj-${{ github.ref_name }}-x64.deb
@@ -92,7 +92,7 @@ jobs:
 
       - name: ⏫ Upload Linux App Image
         if: startsWith(github.ref, 'refs/tags/')
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           if-no-files-found: warn
           name: khoj-${{ github.ref_name }}-x64.AppImage

.github/workflows/github_pages_deploy.yml

@@ -17,10 +17,10 @@ jobs:
       runs-on: ubuntu-latest
       steps:
         - name: Checkout
-          uses: actions/checkout@v3
+          uses: actions/checkout@v4
         # 👇 Build steps
         - name: Set up Node.js
-          uses: actions/setup-node@v3
+          uses: actions/setup-node@v4
           with:
             node-version: 18.x
             cache: yarn
@@ -35,12 +35,12 @@ jobs:
             yarn build
         # 👆 Build steps
         - name: Setup Pages
-          uses: actions/configure-pages@v3
+          uses: actions/configure-pages@v5
         - name: Upload artifact
-          uses: actions/upload-pages-artifact@v2
+          uses: actions/upload-pages-artifact@v3
           with:
             # 👇 Specify build output path
             path: documentation/build
         - name: Deploy to GitHub Pages
           id: deployment
-          uses: actions/deploy-pages@v2
+          uses: actions/deploy-pages@v4

.github/workflows/release.yml

@@ -35,21 +35,21 @@ jobs:
           yarn run build --if-present
 
       - name: ⏫ Upload Obsidian Plugin main.js
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           if-no-files-found: error
           name: main.js
           path: src/interface/obsidian/main.js
 
       - name: ⏫ Upload Obsidian Plugin manifest.json
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           if-no-files-found: error
           name: manifest.json
           path: src/interface/obsidian/manifest.json
 
       - name: ⏫ Upload Obsidian Plugin styles.css
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           if-no-files-found: error
           name: styles.css

.github/workflows/run_evals.yml

@@ -133,7 +133,7 @@ jobs:
 
       - name: Upload Results
         if: always()  # Upload results even if tests fail
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: eval-results-${{ steps.hatch.outputs.version }}-${{ matrix.khoj_mode }}-${{ matrix.dataset }}
           path: |

docker-compose.yml

@@ -6,8 +6,6 @@ services:
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: postgres
       POSTGRES_DB: postgres
-    networks:
-      - default
     volumes:
       - khoj_db:/var/lib/postgresql/data/
     healthcheck:
@@ -17,14 +15,10 @@ services:
       retries: 5
   sandbox:
     image: ghcr.io/khoj-ai/terrarium:latest
-    restart: always
+    restart: unless-stopped
-    networks:
-      - default
   search:
     image: docker.io/searxng/searxng:latest
-    restart: always
+    restart: unless-stopped
-    networks:
-      - default
     volumes:
       - khoj_search:/etc/searxng
     environment:
@@ -35,7 +29,7 @@ services:
         condition: service_healthy
     # Use the following line to use the latest version of khoj. Otherwise, it will build from source. Set this to ghcr.io/khoj-ai/khoj-cloud:latest if you want to use the prod image.
     image: ghcr.io/khoj-ai/khoj:latest
-    restart: always
+    restart: unless-stopped
     # Uncomment the following line to build from source. This will take a few minutes. Comment the next two lines out if you want to use the official image.
     # build:
       # context: .
@@ -45,11 +39,9 @@ services:
       #   change the port in the args in the build section,
       #   as well as the port in the command section to match
       - "42110:42110"
-    working_dir: /app
-    networks:
-      - default
     extra_hosts:
       - "host.docker.internal:host-gateway"
+    working_dir: /app
     volumes:
       - khoj_config:/root/.khoj/
       - khoj_models:/root/.cache/torch/sentence_transformers

documentation/docs/advanced/admin.md

@@ -47,7 +47,7 @@ For each AI Model API you [add](http://localhost:42110/server/admin/database/aim
   ![example configuration for ai model api](/img/example_openai_processor_config.png)
 
 ### Search Model Configs
-Search models are used to generate vector embeddings of your documents for natural language search and chat. You can choose any [embeddings models on HuggingFace](https://huggingface.co/models?pipeline_tag=sentence-similarity) to try, use for your to create vector embeddings of your documents for natural language search and chat.
+Search models are used to generate vector embeddings of your documents for natural language search and chat. You can choose any [embeddings models on HuggingFace](https://huggingface.co/models?pipeline_tag=sentence-similarity) to create vector embeddings of your documents for natural language search and chat.
 
 <img src="/img/example_search_model_admin_settings.png" alt="Example Search Model Settings" style={{width: 500}} />
 
@@ -64,6 +64,9 @@ Add speech to text models with these settings. Khoj currently only supports whis
 ### Voice Model Options
 Add text to speech models with these settings. Khoj currently supports models from [ElevenLabs](https://elevenlabs.io/).
 
+### Reflective Questions
+This is a static list of starter question suggestions for each user. It is not current used in any client app. It used to be shown on the web app home page. We may turn it into a dynamic list of starter questions personalized to each users, say based on their recent conversations or synced knowledge base.
+
 ## User Data
 - Users, Entrys, Conversations, Subscriptions, Github configs, Notion configs, User search configs, User conversation configs, User voice configs
 
@@ -71,4 +74,4 @@ Add text to speech models with these settings. Khoj currently supports models fr
 - Process Locks: Persistent Locks for Automations
 - Client Applications:
 
-Client applications allow you to setup third party applications that can query your Khoj server using a client application ID + secret. The secret would go in a bearer token.
+  Client applications allow you to setup third party applications that can query your Khoj server using a client application ID + secret. The secret would go in a bearer token.

documentation/docs/data-sources/github_integration.md

@@ -1,6 +1,10 @@
 # Github integration
 
-The Github integration allows you to index as many repositories as you want. It's currently default configured to index Issues, Commits, and all Markdown/Org files in each repository. For large repositories, this takes a fairly long time, but it works well for smaller projects.
+:::warning[Unmaintained]
+The Github integration is not maintained. We are considering deprecating it. It doesn't seem used by many folks and its cumbersome for us to maintain.
+:::
+
+The Github integration allows you to index as many repositories as you want. It's currently default configured to index all Markdown/Org/Text files in each repository. For large repositories, this takes a fairly long time, but it works well for smaller projects.
 
 # Configure your settings
 
@@ -9,6 +13,6 @@ The Github integration allows you to index as many repositories as you want. It'
 ## Use the Github plugin
 
 1. Generate a [classic PAT (personal access token)](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) from [Github](https://github.com/settings/tokens) with `repo` and `admin:org` scopes at least.
-2. Navigate to [https://app.khoj.dev/settings#github](https://app.khoj.dev/settings#github) to configure your Github settings. Enter in your PAT, along with details for each repository you want to index.
+2. Navigate to [https://app.khoj.dev/settings#github](https://app.khoj.dev/settings/content/github) to configure your Github settings. Enter in your PAT, along with details for each repository you want to index.
 3. Click `Save`. Go back to the settings page and click `Configure`.
 4. Go to [https://app.khoj.dev/](https://app.khoj.dev/) and start searching!

src/interface/obsidian/src/main.ts

@@ -34,6 +34,21 @@ export default class Khoj extends Plugin {
             callback: () => { this.activateView(KhojView.CHAT); }
         });
 
+        // Add sync command to manually sync new changes
+        this.addCommand({
+            id: 'sync',
+            name: 'Sync new changes',
+            callback: async () => {
+                this.settings.lastSync = await updateContentIndex(
+                    this.app.vault,
+                    this.settings,
+                    this.settings.lastSync,
+                    false,
+                    true
+                );
+            }
+        });
+
         this.registerView(KhojView.CHAT, (leaf) => new KhojChatView(leaf, this.settings));
 
         // Create an icon in the left ribbon.
@@ -44,12 +59,32 @@ export default class Khoj extends Plugin {
         // Add a settings tab so the user can configure khoj
         this.addSettingTab(new KhojSettingTab(this.app, this));
 
-        // Add scheduled job to update index every 60 minutes
+        // Start the sync timer
+        this.startSyncTimer();
+    }
+
+    // Method to start the sync timer
+    private startSyncTimer() {
+        // Clean up the old timer if it exists
+        if (this.indexingTimer) {
+            clearInterval(this.indexingTimer);
+        }
+
+        // Start a new timer with the configured interval
         this.indexingTimer = setInterval(async () => {
             if (this.settings.autoConfigure) {
-                this.settings.lastSync = await updateContentIndex(this.app.vault, this.settings, this.settings.lastSync);
+                this.settings.lastSync = await updateContentIndex(
+                    this.app.vault,
+                    this.settings,
+                    this.settings.lastSync
+                );
             }
-        }, 60 * 60 * 1000);
+        }, this.settings.syncInterval * 60 * 1000); // Convert minutes to milliseconds
+    }
+
+    // Public method to restart the timer (called from settings)
+    public restartSyncTimer() {
+        this.startSyncTimer();
     }
 
     async loadSettings() {
@@ -62,7 +97,7 @@ export default class Khoj extends Plugin {
     }
 
     async saveSettings() {
-        this.saveData(this.settings);
+        await this.saveData(this.settings);
     }
 
     async onunload() {

src/interface/obsidian/src/search_modal.ts

@@ -1,4 +1,4 @@
-import { App, SuggestModal, request, MarkdownRenderer, Instruction, Platform } from 'obsidian';
+import { App, SuggestModal, request, MarkdownRenderer, Instruction, Platform, Notice } from 'obsidian';
 import { KhojSetting } from 'src/settings';
 import { supportedBinaryFileTypes, createNoteAndCloseModal, getFileFromPath, getLinkToEntry, supportedImageFilesTypes } from 'src/utils';
 
@@ -13,6 +13,9 @@ export class KhojSearchModal extends SuggestModal<SearchResult> {
     find_similar_notes: boolean;
     query: string = "";
     app: App;
+    currentController: AbortController | null = null;  // To cancel requests
+    isLoading: boolean = false;
+    loadingEl: HTMLElement;
 
     constructor(app: App, setting: KhojSetting, find_similar_notes: boolean = false) {
         super(app);
@@ -23,6 +26,24 @@ export class KhojSearchModal extends SuggestModal<SearchResult> {
         // Hide input element in Similar Notes mode
         this.inputEl.hidden = this.find_similar_notes;
 
+        // Create loading element
+        this.loadingEl = createDiv({ cls: "search-loading" });
+        const spinnerEl = this.loadingEl.createDiv({ cls: "search-loading-spinner" });
+
+        this.loadingEl.style.position = "absolute";
+        this.loadingEl.style.top = "50%";
+        this.loadingEl.style.left = "50%";
+        this.loadingEl.style.transform = "translate(-50%, -50%)";
+        this.loadingEl.style.zIndex = "1000";
+        this.loadingEl.style.display = "none";
+
+        // Add the element to the modal
+        this.modalEl.appendChild(this.loadingEl);
+
+        // Customize empty state message
+        // @ts-ignore - Access to private property to customize the message
+        this.emptyStateText = "";
+
         // Register Modal Keybindings to Rerank Results
         this.scope.register(['Mod'], 'Enter', async () => {
             // Re-rank when explicitly triggered by user
@@ -66,6 +87,101 @@ export class KhojSearchModal extends SuggestModal<SearchResult> {
         this.setPlaceholder('Search with Khoj...');
     }
 
+    // Check if the file exists in the vault
+    private isFileInVault(filePath: string): boolean {
+        // Normalize the path to handle different separators
+        const normalizedPath = filePath.replace(/\\/g, '/');
+
+        // Check if the file exists in the vault
+        return this.app.vault.getFiles().some(file =>
+            file.path === normalizedPath
+        );
+    }
+
+    async getSuggestions(query: string): Promise<SearchResult[]> {
+        // Do not show loading if the query is empty
+        if (!query.trim()) {
+            this.isLoading = false;
+            this.updateLoadingState();
+            return [];
+        }
+
+        // Show loading state
+        this.isLoading = true;
+        this.updateLoadingState();
+
+        // Cancel previous request if it exists
+        if (this.currentController) {
+            this.currentController.abort();
+        }
+
+        try {
+            // Create a new controller for this request
+            this.currentController = new AbortController();
+
+            // Setup Query Khoj backend for search results
+            let encodedQuery = encodeURIComponent(query);
+            let searchUrl = `${this.setting.khojUrl}/api/search?q=${encodedQuery}&n=${this.setting.resultsCount}&r=${this.rerank}&client=obsidian`;
+            let headers = {
+                'Authorization': `Bearer ${this.setting.khojApiKey}`,
+            }
+
+            // Get search results from Khoj backend
+            const response = await fetch(searchUrl, {
+                headers: headers,
+                signal: this.currentController.signal
+            });
+
+            if (!response.ok) {
+                throw new Error(`HTTP error! status: ${response.status}`);
+            }
+
+            const data = await response.json();
+
+            // Parse search results
+            let results = data
+                .filter((result: any) =>
+                    !this.find_similar_notes || !result.additional.file.endsWith(this.app.workspace.getActiveFile()?.path)
+                )
+                .map((result: any) => {
+                    return {
+                        entry: result.entry,
+                        file: result.additional.file,
+                        inVault: this.isFileInVault(result.additional.file)
+                    } as SearchResult & { inVault: boolean };
+                })
+                .sort((a: SearchResult & { inVault: boolean }, b: SearchResult & { inVault: boolean }) => {
+                    if (a.inVault === b.inVault) return 0;
+                    return a.inVault ? -1 : 1;
+                });
+
+            this.query = query;
+
+            // Hide loading state only on successful completion
+            this.isLoading = false;
+            this.updateLoadingState();
+
+            return results;
+        } catch (error) {
+            // Ignore cancellation errors and keep loading state
+            if (error.name === 'AbortError') {
+                // When cancelling, we don't want to render anything
+                return undefined as any;
+            }
+
+            // For other errors, hide loading state
+            console.error('Search error:', error);
+            this.isLoading = false;
+            this.updateLoadingState();
+            return [];
+        }
+    }
+
+    private updateLoadingState() {
+        // Show or hide loading element
+        this.loadingEl.style.display = this.isLoading ? "block" : "none";
+    }
+
     async onOpen() {
         if (this.find_similar_notes) {
             // If markdown file is currently active
@@ -86,25 +202,7 @@ export class KhojSearchModal extends SuggestModal<SearchResult> {
         }
     }
 
-    async getSuggestions(query: string): Promise<SearchResult[]> {
+    async renderSuggestion(result: SearchResult & { inVault: boolean }, el: HTMLElement) {
-        // Setup Query Khoj backend for search results
-        let encodedQuery = encodeURIComponent(query);
-        let searchUrl = `${this.setting.khojUrl}/api/search?q=${encodedQuery}&n=${this.setting.resultsCount}&r=${this.rerank}&client=obsidian`;
-        let headers = { 'Authorization': `Bearer ${this.setting.khojApiKey}` }
-
-        // Get search results from Khoj backend
-        let response = await request({ url: `${searchUrl}`, headers: headers });
-
-        // Parse search results
-        let results = JSON.parse(response)
-            .filter((result: any) => !this.find_similar_notes || !result.additional.file.endsWith(this.app.workspace.getActiveFile()?.path))
-            .map((result: any) => { return { entry: result.entry, file: result.additional.file } as SearchResult; });
-
-        this.query = query;
-        return results;
-    }
-
-    async renderSuggestion(result: SearchResult, el: HTMLElement) {
         // Max number of lines to render
         let lines_to_render = 8;
 
@@ -112,13 +210,25 @@ export class KhojSearchModal extends SuggestModal<SearchResult> {
         let os_path_separator = result.file.includes('\\') ? '\\' : '/';
         let filename = result.file.split(os_path_separator).pop();
 
-        // Show filename of each search result for context
+        // Show filename of each search result for context with appropriate color
-        el.createEl("div",{ cls: 'khoj-result-file' }).setText(filename ?? "");
+        const fileEl = el.createEl("div", {
+            cls: `khoj-result-file ${result.inVault ? 'in-vault' : 'not-in-vault'}`
+        });
+        fileEl.setText(filename ?? "");
+
+        // Add a visual indication for files not in vault
+        if (!result.inVault) {
+            fileEl.createSpan({
+                text: " (not in vault)",
+                cls: "khoj-result-file-status"
+            });
+        }
+
         let result_el = el.createEl("div", { cls: 'khoj-result-entry' })
 
         let resultToRender = "";
         let fileExtension = filename?.split(".").pop() ?? "";
-        if (supportedImageFilesTypes.includes(fileExtension) && filename) {
+        if (supportedImageFilesTypes.includes(fileExtension) && filename && result.inVault) {
             let linkToEntry: string = filename;
             let imageFiles = this.app.vault.getFiles().filter(file => supportedImageFilesTypes.includes(fileExtension));
             // Find vault file of chosen search result
@@ -140,7 +250,13 @@ export class KhojSearchModal extends SuggestModal<SearchResult> {
         MarkdownRenderer.renderMarkdown(resultToRender, result_el, result.file, null);
     }
 
-    async onChooseSuggestion(result: SearchResult, _: MouseEvent | KeyboardEvent) {
+    async onChooseSuggestion(result: SearchResult & { inVault: boolean }, _: MouseEvent | KeyboardEvent) {
+        // Only open files that are in the vault
+        if (!result.inVault) {
+            new Notice("This file is not in your vault");
+            return;
+        }
+
         // Get all markdown, pdf and image files in vault
         const mdFiles = this.app.vault.getMarkdownFiles();
         const binaryFiles = this.app.vault.getFiles().filter(file => supportedBinaryFileTypes.includes(file.extension));

src/interface/obsidian/src/settings.ts

@@ -1,4 +1,4 @@
-import { App, Notice, PluginSettingTab, Setting, TFile } from 'obsidian';
+import { App, Notice, PluginSettingTab, Setting, TFile, SuggestModal } from 'obsidian';
 import Khoj from 'src/main';
 import { canConnectToBackend, getBackendStatusMessage, updateContentIndex } from './utils';
 
@@ -15,6 +15,7 @@ interface SyncFileTypes {
     images: boolean;
     pdf: boolean;
 }
+
 export interface KhojSetting {
     resultsCount: number;
     khojUrl: string;
@@ -24,6 +25,8 @@ export interface KhojSetting {
     lastSync: Map<TFile, number>;
     syncFileType: SyncFileTypes;
     userInfo: UserInfo | null;
+    syncFolders: string[];
+    syncInterval: number;
 }
 
 export const DEFAULT_SETTINGS: KhojSetting = {
@@ -39,6 +42,8 @@ export const DEFAULT_SETTINGS: KhojSetting = {
         pdf: true,
     },
     userInfo: null,
+    syncFolders: [],
+    syncInterval: 60,
 }
 
 export class KhojSettingTab extends PluginSettingTab {
@@ -60,7 +65,8 @@ export class KhojSettingTab extends PluginSettingTab {
                 this.plugin.settings.userInfo?.email,
                 this.plugin.settings.khojUrl,
                 this.plugin.settings.khojApiKey
-            )}
+            )
+        }
         );
         let backendStatusMessage: string = '';
 
@@ -109,7 +115,7 @@ export class KhojSettingTab extends PluginSettingTab {
                 }));
 
         // Add new "Sync" heading
-        containerEl.createEl('h3', {text: 'Sync'});
+        containerEl.createEl('h3', { text: 'Sync' });
 
         // Add setting to sync markdown notes
         new Setting(containerEl)
@@ -153,6 +159,51 @@ export class KhojSettingTab extends PluginSettingTab {
                     this.plugin.settings.autoConfigure = value;
                     await this.plugin.saveSettings();
                 }));
+
+        // Add setting for sync interval
+        const syncIntervalValues = [1, 5, 10, 20, 30, 45, 60, 120, 1440];
+        new Setting(containerEl)
+            .setName('Sync Interval')
+            .setDesc('Minutes between automatic synchronizations')
+            .addDropdown(dropdown => dropdown
+                .addOptions(Object.fromEntries(
+                    syncIntervalValues.map(value => [
+                        value.toString(),
+                        value === 1 ? '1 minute' :
+                            value === 1440 ? '24 hours' :
+                                `${value} minutes`
+                    ])
+                ))
+                .setValue(this.plugin.settings.syncInterval.toString())
+                .onChange(async (value) => {
+                    this.plugin.settings.syncInterval = parseInt(value);
+                    await this.plugin.saveSettings();
+                    // Restart the timer with the new interval
+                    this.plugin.restartSyncTimer();
+                }));
+
+        // Add setting to manage sync folders
+        const syncFoldersContainer = containerEl.createDiv('sync-folders-container');
+        const foldersSetting = new Setting(syncFoldersContainer)
+            .setName('Sync Folders')
+            .setDesc('Specify folders to sync (leave empty to sync entire vault)')
+            .addButton(button => button
+                .setButtonText('Add Folder')
+                .onClick(() => {
+                    const modal = new FolderSuggestModal(this.app, (folder: string) => {
+                        if (!this.plugin.settings.syncFolders.includes(folder)) {
+                            this.plugin.settings.syncFolders.push(folder);
+                            this.plugin.saveSettings();
+                            this.updateFolderList(folderListEl);
+                        }
+                    });
+                    modal.open();
+                }));
+
+        // Create a list to display selected folders
+        const folderListEl = syncFoldersContainer.createDiv('folder-list');
+        this.updateFolderList(folderListEl);
+
         let indexVaultSetting = new Setting(containerEl);
         indexVaultSetting
             .setName('Force Sync')
@@ -200,4 +251,81 @@ export class KhojSettingTab extends PluginSettingTab {
                 })
             );
     }
+
+    // Helper method to update the folder list display
+    private updateFolderList(containerEl: HTMLElement) {
+        containerEl.empty();
+        if (this.plugin.settings.syncFolders.length === 0) {
+            containerEl.createEl('div', {
+                text: 'Syncing entire vault',
+                cls: 'folder-list-empty'
+            });
+            return;
+        }
+
+        const list = containerEl.createEl('ul', { cls: 'folder-list' });
+        this.plugin.settings.syncFolders.forEach(folder => {
+            const item = list.createEl('li', { cls: 'folder-list-item' });
+            item.createSpan({ text: folder });
+
+            const removeButton = item.createEl('button', {
+                cls: 'folder-list-remove',
+                text: '×'
+            });
+            removeButton.addEventListener('click', async () => {
+                this.plugin.settings.syncFolders = this.plugin.settings.syncFolders.filter(f => f !== folder);
+                await this.plugin.saveSettings();
+                this.updateFolderList(containerEl);
+            });
+        });
+    }
+}
+
+// Modal with folder suggestions
+class FolderSuggestModal extends SuggestModal<string> {
+    constructor(app: App, private onChoose: (folder: string) => void) {
+        super(app);
+    }
+
+    getSuggestions(query: string): string[] {
+        const folders = this.getAllFolders();
+        if (!query) return folders;
+
+        return folders.filter(folder =>
+            folder.toLowerCase().includes(query.toLowerCase())
+        );
+    }
+
+    renderSuggestion(folder: string, el: HTMLElement) {
+        el.createSpan({
+            text: folder || '/',
+            cls: 'folder-suggest-item'
+        });
+    }
+
+    onChooseSuggestion(folder: string, _: MouseEvent | KeyboardEvent) {
+        this.onChoose(folder);
+    }
+
+    private getAllFolders(): string[] {
+        const folders = new Set<string>();
+        folders.add(''); // Root folder
+
+        // Get all files and extract folder paths
+        this.app.vault.getAllLoadedFiles().forEach(file => {
+            const folderPath = file.parent?.path;
+            if (folderPath) {
+                folders.add(folderPath);
+
+                // Also add all parent folders
+                let parent = folderPath;
+                while (parent.includes('/')) {
+                    parent = parent.substring(0, parent.lastIndexOf('/'));
+                    folders.add(parent);
+                }
+            }
+        });
+
+        return Array.from(folders).sort();
+    }
 }

src/interface/obsidian/src/utils.ts

@@ -9,7 +9,7 @@ export function getVaultAbsolutePath(vault: Vault): string {
     return '';
 }
 
-function fileExtensionToMimeType (extension: string): string {
+function fileExtensionToMimeType(extension: string): string {
     switch (extension) {
         case 'pdf':
             return 'application/pdf';
@@ -28,7 +28,7 @@ function fileExtensionToMimeType (extension: string): string {
     }
 }
 
-function filenameToMimeType (filename: TFile): string {
+function filenameToMimeType(filename: TFile): string {
     switch (filename.extension) {
         case 'pdf':
             return 'application/pdf';
@@ -63,15 +63,24 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las
     // Get all markdown, pdf files in the vault
     console.log(`Khoj: Updating Khoj content index...`)
     const files = vault.getFiles()
-                        // Filter supported file types for syncing
+        // Filter supported file types for syncing
-                        .filter(file => supportedFileTypes.includes(file.extension))
+        .filter(file => supportedFileTypes.includes(file.extension))
-                        // Filter user configured file types for syncing
+        // Filter user configured file types for syncing
-                        .filter(file => {
+        .filter(file => {
-                            if (fileTypeToExtension.markdown.includes(file.extension)) return setting.syncFileType.markdown;
+            if (fileTypeToExtension.markdown.includes(file.extension)) return setting.syncFileType.markdown;
-                            if (fileTypeToExtension.pdf.includes(file.extension)) return setting.syncFileType.pdf;
+            if (fileTypeToExtension.pdf.includes(file.extension)) return setting.syncFileType.pdf;
-                            if (fileTypeToExtension.image.includes(file.extension)) return setting.syncFileType.images;
+            if (fileTypeToExtension.image.includes(file.extension)) return setting.syncFileType.images;
-                            return false;
+            return false;
-                        });
+        })
+        // Filter files based on specified folders
+        .filter(file => {
+            // If no folders are specified, sync all files
+            if (setting.syncFolders.length === 0) return true;
+            // Otherwise, check if the file is in one of the specified folders
+            return setting.syncFolders.some(folder =>
+                file.path.startsWith(folder + '/') || file.path === folder
+            );
+        });
 
     let countOfFilesToIndex = 0;
     let countOfFilesToDelete = 0;
@@ -81,7 +90,7 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las
     const fileData = [];
     for (const file of files) {
         // Only push files that have been modified since last sync if not regenerating
-        if (!regenerate && file.stat.mtime < (lastSync.get(file) ?? 0)){
+        if (!regenerate && file.stat.mtime < (lastSync.get(file) ?? 0)) {
             continue;
         }
 
@@ -89,7 +98,7 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las
         const encoding = supportedBinaryFileTypes.includes(file.extension) ? "binary" : "utf8";
         const mimeType = fileExtensionToMimeType(file.extension) + (encoding === "utf8" ? "; charset=UTF-8" : "");
         const fileContent = encoding == 'binary' ? await vault.readBinary(file) : await vault.read(file);
-        fileData.push({blob: new Blob([fileContent], { type: mimeType }), path: file.path});
+        fileData.push({ blob: new Blob([fileContent], { type: mimeType }), path: file.path });
     }
 
     // Add any previously synced files to be deleted to multipart form data
@@ -98,13 +107,13 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las
         if (!files.includes(lastSyncedFile)) {
             countOfFilesToDelete++;
             let fileObj = new Blob([""], { type: filenameToMimeType(lastSyncedFile) });
-            fileData.push({blob: fileObj, path: lastSyncedFile.path});
+            fileData.push({ blob: fileObj, path: lastSyncedFile.path });
             filesToDelete.push(lastSyncedFile);
         }
     }
 
     // Iterate through all indexable files in vault, 1000 at a time
-    let responses: string[]  = [];
+    let responses: string[] = [];
     let error_message = null;
     for (let i = 0; i < fileData.length; i += 1000) {
         const filesGroup = fileData.slice(i, i + 1000);
@@ -166,17 +175,17 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las
     }
 
     // Update last sync time for each successfully indexed file
-   files
+    files
-    .filter(file => responses.find(response => response.includes(file.path)))
+        .filter(file => responses.find(response => response.includes(file.path)))
-    .reduce((newSync, file) => {
+        .reduce((newSync, file) => {
-        newSync.set(file, new Date().getTime());
+            newSync.set(file, new Date().getTime());
-        return newSync;
+            return newSync;
-    }, lastSync);
+        }, lastSync);
 
     // Remove files that were deleted from last sync
     filesToDelete
-    .filter(file => responses.find(response => response.includes(file.path)))
+        .filter(file => responses.find(response => response.includes(file.path)))
-    .forEach(file => lastSync.delete(file));
+        .forEach(file => lastSync.delete(file));
 
     if (error_message) {
         new Notice(error_message);
@@ -188,31 +197,30 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las
     return lastSync;
 }
 
-export async function openKhojPluginSettings(): Promise<void>
+export async function openKhojPluginSettings(): Promise<void> {
- {
+    const setting = this.app.setting;
-     const setting = this.app.setting;
+    await setting.open();
-     await setting.open();
+    setting.openTabById('khoj');
-     setting.openTabById('khoj');
 }
 
 export async function createNote(name: string, newLeaf = false): Promise<void> {
     try {
-      let pathPrefix: string
+        let pathPrefix: string
-      switch (this.app.vault.getConfig('newFileLocation')) {
+        switch (this.app.vault.getConfig('newFileLocation')) {
-        case 'current':
+            case 'current':
-          pathPrefix = (this.app.workspace.getActiveFile()?.parent.path ?? '') + '/'
+                pathPrefix = (this.app.workspace.getActiveFile()?.parent.path ?? '') + '/'
-          break
+                break
-        case 'folder':
+            case 'folder':
-          pathPrefix = this.app.vault.getConfig('newFileFolderPath') + '/'
+                pathPrefix = this.app.vault.getConfig('newFileFolderPath') + '/'
-          break
+                break
-        default: // 'root'
+            default: // 'root'
-          pathPrefix = ''
+                pathPrefix = ''
-          break
+                break
-      }
+        }
-      await this.app.workspace.openLinkText(`${pathPrefix}${name}.md`, '', newLeaf)
+        await this.app.workspace.openLinkText(`${pathPrefix}${name}.md`, '', newLeaf)
     } catch (e) {
-      console.error('Khoj: Could not create note.\n' + (e as any).message);
+        console.error('Khoj: Could not create note.\n' + (e as any).message);
-      throw e
+        throw e
     }
 }
 
@@ -236,7 +244,7 @@ export async function canConnectToBackend(
     let userInfo: UserInfo | null = null;
 
     if (!!khojUrl) {
-        let headers  = !!khojApiKey ? { "Authorization": `Bearer ${khojApiKey}` } : undefined;
+        let headers = !!khojApiKey ? { "Authorization": `Bearer ${khojApiKey}` } : undefined;
         try {
             let response = await request({ url: `${khojUrl}/api/v1/user`, method: "GET", headers: headers })
             connectedToBackend = true;
@@ -387,7 +395,7 @@ function copyParentText(event: MouseEvent, message: string, originalButton: stri
 }
 
 export function createCopyParentText(message: string, originalButton: string = 'copy-plus') {
-    return function(event: MouseEvent) {
+    return function (event: MouseEvent) {
         return copyParentText(event, message, originalButton);
     }
 }
@@ -406,7 +414,7 @@ export function pasteTextAtCursor(text: string | undefined) {
     // If there is a selection, replace it with the text
     if (editor?.getSelection()) {
         editor.replaceSelection(text);
-    // If there is no selection, insert the text at the cursor position
+        // If there is no selection, insert the text at the cursor position
     } else if (cursor) {
         editor.replaceRange(text, cursor);
     }

src/interface/obsidian/styles.css

@@ -13,11 +13,12 @@ If your plugin does not need CSS, delete this file.
     --khoj-storm-grey: #475569;
     --chat-icon: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='24px' height='24px' viewBox='0 0 24 24' fill='currentColor' stroke-linecap='round' stroke-linejoin='round' class='svg-icon' version='1.1'%3E%3Cpath d='m 14.024348,9.8497703 0.04627,1.9750167' stroke='%231c274c' stroke-width='2' stroke-linecap='round' /%3E%3Cpath d='m 9.6453624,9.7953624 0.046275,1.9750166' stroke='%231c274c' stroke-width='2' stroke-linecap='round' /%3E%3Cpath d='m 11.90538,2.3619994 c -5.4939109,0 -9.6890976,4.0608185 -9.6890976,9.8578926 0,1.477202 0.2658016,2.542848 0.6989332,3.331408 0.433559,0.789293 1.0740097,1.372483 1.9230615,1.798517 1.7362861,0.87132 4.1946007,1.018626 7.0671029,1.018626 0.317997,0 0.593711,0.167879 0.784844,0.458501 0.166463,0.253124 0.238617,0.552748 0.275566,0.787233 0.07263,0.460801 0.05871,1.030165 0.04785,1.474824 v 4.8e-5 l -2.26e-4,0.0091 c -0.0085,0.348246 -0.01538,0.634247 -0.0085,0.861186 0.105589,-0.07971 0.227925,-0.185287 0.36735,-0.31735 0.348613,-0.330307 0.743513,-0.767362 1.176607,-1.246635 l 0.07837,-0.08673 c 0.452675,-0.500762 0.941688,-1.037938 1.41216,-1.473209 0.453774,-0.419787 0.969948,-0.822472 1.476003,-0.953853 1.323661,-0.343655 2.330132,-0.904027 3.005749,-1.76381 0.658957,-0.838568 1.073167,-2.051868 1.073167,-3.898667 0,-5.7970748 -4.195186,-9.8578946 -9.689097,-9.8578946 z M 0.92440678,12.219892 c 0,-7.0067939 5.05909412,-11.47090892 10.98097322,-11.47090892 5.921878,0 10.980972,4.46411502 10.980972,11.47090892 0,2.172259 -0.497596,3.825405 -1.442862,5.028357 -0.928601,1.181693 -2.218843,1.837914 -3.664937,2.213334 -0.211641,0.05502 -0.53529,0.268579 -0.969874,0.670658 -0.417861,0.386604 -0.865628,0.876836 -1.324566,1.384504 l -0.09131,0.101202 c -0.419252,0.464136 -0.849637,0.94059 -1.239338,1.309807 -0.210187,0.199169 -0.425281,0.383422 -0.635348,0.523424 -0.200911,0.133819 -0.449635,0.263369 -0.716376,0.281474 -0.327812,0.02226 -0.61539,-0.149209 -0.804998,-0.457293 -0.157614,-0.255993 -0.217622,-0.557143 -0.246564,-0.778198 -0.0542,-0.414027 -0.04101,-0.933065 -0.03027,-1.355183 l 0.0024,-0.0922 c 0.01099,-0.463865 0.01489,-0.820507 -0.01611,-1.06842 C 8.9434608,19.975238 6.3139711,19.828758 4.356743,18.84659 3.3355029,18.334136 2.4624526,17.578678 1.8500164,16.463713 1.2372016,15.348029 0.92459928,13.943803 0.92459928,12.219967 Z' clip-rule='evenodd' stroke-width='2' fill='currentColor' fill-rule='evenodd' fill-opacity='1' /%3E%3C/svg%3E%0A");
     --search-icon: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='24px' height='24px' viewBox='0 0 24 24' fill='currentColor' stroke-linecap='round' stroke-linejoin='round' class='svg-icon' version='1.1'%3E%3Cpath d='m 18.562765,17.147843 c 1.380497,-1.679442 2.307667,-4.013099 2.307667,-6.330999 C 20.870432,5.3951476 16.353958,1 10.782674,1 5.2113555,1 0.69491525,5.3951476 0.69491525,10.816844 c 0,5.421663 4.51644025,9.816844 10.08775875,9.816844 2.381867,0 4.570922,-0.803307 6.296712,-2.14673 0.508475,-0.508475 4.514633,4.192839 4.514633,4.192839 1.036377,1.008544 2.113087,-0.02559 1.07671,-1.034139 z m -7.780091,1.925408 c -4.3394583,0 -8.6708434,-4.033489 -8.6708434,-8.256407 0,-4.2229187 4.3313851,-8.2564401 8.6708434,-8.2564401 4.339458,0 8.670809,4.2369112 8.670809,8.4598301 0,4.222918 -4.331351,8.053017 -8.670809,8.053017 z' fill='currentColor' fill-rule='evenodd' clip-rule='evenodd' fill-opacity='1' stroke-width='1.10519' stroke-dasharray='none' /%3E%3Cpath d='m 13.337351,9.3402647 0.05184,2.1532893' stroke='%231c274c' stroke-width='2' stroke-linecap='round' /%3E%3Cpath d='M 8.431347,9.2809457 8.483191,11.434235' stroke='%231c274c' stroke-width='2' stroke-linecap='round' /%3E%3C/svg%3E%0A");
- }
+}
 
 .khoj-chat p {
     margin: 0;
 }
+
 .khoj-chat pre {
     text-wrap: unset;
 }
@@ -33,7 +34,8 @@ If your plugin does not need CSS, delete this file.
     font-weight: 300;
     line-height: 1.5em;
 }
-.khoj-chat > * {
+
+.khoj-chat>* {
     padding: 10px;
     margin: 10px;
 }
@@ -47,8 +49,10 @@ If your plugin does not need CSS, delete this file.
     font-size: var(--font-ui-medium);
     margin: 0px;
     line-height: 20px;
-    overflow-y: scroll; /* Make chat body scroll to see history */
+    overflow-y: scroll;
+    /* Make chat body scroll to see history */
 }
+
 /* add chat metatdata to bottom of bubble */
 .khoj-chat-message.khoj::after {
     content: attr(data-meta);
@@ -57,16 +61,19 @@ If your plugin does not need CSS, delete this file.
     color: var(--text-muted);
     margin: -12px 7px 0 0px;
 }
+
 /* move message by khoj to left */
 .khoj-chat-message.khoj {
     margin-left: auto;
     text-align: left;
 }
+
 /* move message by you to right */
 .khoj-chat-message.you {
     margin-right: auto;
     text-align: right;
 }
+
 /* basic style chat message text */
 .khoj-chat-message-text {
     margin: 10px;
@@ -80,6 +87,7 @@ If your plugin does not need CSS, delete this file.
     background-color: var(--active-bg);
     word-break: break-word;
 }
+
 /* color chat bubble by khoj blue */
 .khoj-chat-message-text.khoj {
     border-left: 2px solid var(--khoj-sun);
@@ -87,12 +95,14 @@ If your plugin does not need CSS, delete this file.
     margin-left: auto;
     white-space: pre-line;
 }
+
 /* Override white-space for ul, ol, li under khoj-chat-message-text.khoj */
 .khoj-chat-message-text.khoj ul,
 .khoj-chat-message-text.khoj ol,
 .khoj-chat-message-text.khoj li {
     white-space: normal;
 }
+
 /* add left protrusion to khoj chat bubble */
 .khoj-chat-message-text.khoj:after {
     content: '';
@@ -103,12 +113,14 @@ If your plugin does not need CSS, delete this file.
     border-bottom: 0;
     transform: rotate(-60deg);
 }
+
 /* color chat bubble by you dark grey */
 .khoj-chat-message-text.you {
     color: var(--text-normal);
     margin-right: auto;
     background-color: var(--background-modifier-cover);
 }
+
 /* add right protrusion to you chat bubble */
 .khoj-chat-message-text.you:after {
     content: '';
@@ -125,6 +137,7 @@ If your plugin does not need CSS, delete this file.
 .khoj-chat-message-text ol {
     margin: 0px 0 0;
 }
+
 .khoj-chat-message-text ol li {
     white-space: normal;
 }
@@ -146,9 +159,11 @@ code.chat-response {
 div.collapsed {
     display: none;
 }
+
 div.expanded {
     display: block;
 }
+
 div.reference {
     display: grid;
     grid-template-rows: auto;
@@ -157,6 +172,7 @@ div.reference {
     grid-row-gap: 10px;
     margin: 10px;
 }
+
 div.expanded.reference-section {
     display: grid;
     grid-template-rows: auto;
@@ -165,6 +181,7 @@ div.expanded.reference-section {
     grid-row-gap: 10px;
     margin: 10px 0;
 }
+
 button.reference-button {
     border: 1px solid var(--khoj-storm-grey);
     background-color: transparent;
@@ -183,15 +200,18 @@ button.reference-button {
     display: inline-block;
     text-wrap: inherit;
 }
+
 button.reference-button.expanded {
     height: auto;
     max-height: none;
     white-space: pre-wrap;
 }
-button.reference-button.expanded > :nth-child(2) {
+
+button.reference-button.expanded> :nth-child(2) {
     display: block;
 }
-button.reference-button.collapsed > :nth-child(2) {
+
+button.reference-button.collapsed> :nth-child(2) {
     display: none;
 }
 
@@ -201,11 +221,13 @@ button.reference-button::before {
     display: inline-block;
     transition: transform 0.1s ease-in-out;
 }
+
 button.reference-button.expanded::before,
 button.reference-button:active:before,
 button.reference-button[aria-expanded="true"]::before {
     transform: rotate(90deg);
 }
+
 button.reference-expand-button {
     background-color: transparent;
     border: 1px solid var(--khoj-storm-grey);
@@ -219,15 +241,18 @@ button.reference-expand-button {
     transition: background 0.2s ease-in-out;
     text-align: left;
 }
+
 button.reference-expand-button:hover {
     background: var(--background-modifier-active-hover);
     color: var(--text-normal);
 }
+
 a.inline-chat-link {
     color: #475569;
     text-decoration: none;
     border-bottom: 1px dotted #475569;
 }
+
 .reference-link {
     color: var(--khoj-storm-grey);
     border-bottom: 1px dotted var(--khoj-storm-grey);
@@ -247,11 +272,13 @@ div.new-conversation {
     z-index: 10;
     background-color: var(--background-primary)
 }
+
 div.conversation-header-title {
     text-align: left;
     font-size: larger;
     line-height: 1.5em;
 }
+
 div.conversation-session {
     color: var(--color-base-90);
     border: 1px solid var(--khoj-storm-grey);
@@ -298,9 +325,11 @@ div.conversation-menu {
     grid-gap: 4px;
     grid-auto-flow: column;
 }
+
 div.conversation-session:hover {
     transform: scale(1.03);
 }
+
 div.selected-conversation {
     background: var(--background-modifier-active-hover) !important;
 }
@@ -312,6 +341,7 @@ div.selected-conversation {
     grid-column-gap: 10px;
     grid-row-gap: 10px;
 }
+
 .khoj-input-row {
     display: grid;
     grid-template-columns: 32px auto 32px 32px;
@@ -324,9 +354,11 @@ div.selected-conversation {
     bottom: 0;
     z-index: 10;
 }
+
 #khoj-chat-input.option:hover {
     box-shadow: 0 0 11px var(--background-modifier-box-shadow);
 }
+
 #khoj-chat-input {
     font-size: var(--font-ui-medium);
     padding: 4px 0 0 12px;
@@ -334,6 +366,7 @@ div.selected-conversation {
     height: 32px;
     resize: none;
 }
+
 .khoj-input-row-button {
     border-radius: 50%;
     padding: 4px;
@@ -346,43 +379,55 @@ div.selected-conversation {
     padding: 0;
     position: relative;
 }
+
 #khoj-chat-send .lucide-arrow-up-circle {
     background: var(--background-modifier-active-hover);
     border-radius: 50%;
 }
+
 #khoj-chat-send .lucide-stop-circle {
     transform: rotateY(-180deg) rotateZ(-90deg);
 }
+
 #khoj-chat-send .lucide-stop-circle circle {
-    stroke-dasharray: 62px;  /* The circumference of the circle with 7px radius */
+    stroke-dasharray: 62px;
+    /* The circumference of the circle with 7px radius */
     stroke-dashoffset: 0px;
     stroke-linecap: round;
     stroke-width: 2px;
     stroke: var(--main-text-color);
     fill: none;
 }
+
 @keyframes countdown {
     from {
         stroke-dashoffset: 0px;
     }
+
     to {
-        stroke-dashoffset: -62px;  /* The circumference of the circle with 7px radius */
+        stroke-dashoffset: -62px;
+        /* The circumference of the circle with 7px radius */
     }
 }
 
-@media (pointer: coarse), (hover: none) {
+@media (pointer: coarse),
+(hover: none) {
     #khoj-chat-body.abbr[title] {
         position: relative;
-        padding-left: 4px;  /* space references out to ease tapping */
+        padding-left: 4px;
+        /* space references out to ease tapping */
     }
+
     #khoj-chat-body.abbr[title]:focus:after {
         content: attr(title);
 
         /* position tooltip */
         position: absolute;
-        left: 16px;  /* open tooltip to right of ref link, instead of on top of it */
+        left: 16px;
+        /* open tooltip to right of ref link, instead of on top of it */
         width: auto;
-        z-index: 1;  /* show tooltip above chat messages */
+        z-index: 1;
+        /* show tooltip above chat messages */
 
         /* style tooltip */
         background-color: var(--background-secondary);
@@ -398,6 +443,14 @@ div.selected-conversation {
     font-weight: 600;
 }
 
+.khoj-result-file.in-vault {
+    color: var(--color-green);
+}
+
+.khoj-result-file.not-in-vault {
+    color: var(--color-blue);
+}
+
 .khoj-result-entry {
     color: var(--text-muted);
     margin-left: 2em;
@@ -410,11 +463,11 @@ div.selected-conversation {
     white-space: normal;
 }
 
-.khoj-result-entry > * {
+.khoj-result-entry>* {
     font-size: var(--font-ui-medium);
 }
 
-.khoj-result-entry > p {
+.khoj-result-entry>p {
     margin-top: 0.2em;
     margin-bottom: 0.2em;
 }
@@ -440,9 +493,11 @@ div.khoj-header {
 a.khoj-nav {
     -webkit-app-region: no-drag;
 }
+
 div.khoj-nav {
     -webkit-app-region: no-drag;
 }
+
 nav.khoj-nav {
     display: grid;
     grid-auto-flow: column;
@@ -470,24 +525,30 @@ div.khoj-logo {
     justify-self: center;
     margin: 0;
 }
+
 .khoj-nav a:hover {
     background-color: var(--background-modifier-active-hover);
     color: var(--main-text-color);
 }
+
 a.khoj-nav-selected {
     background-color: var(--background-modifier-active-hover);
 }
+
 #similar-nav-icon-svg,
 .khoj-nav-icon {
     width: 24px;
     height: 24px;
 }
+
 .khoj-nav-icon-chat {
     background-image: var(--chat-icon);
 }
+
 .khoj-nav-icon-search {
     background-image: var(--search-icon);
 }
+
 span.khoj-nav-item-text {
     padding-left: 8px;
 }
@@ -507,12 +568,14 @@ button.chat-action-button {
     margin-top: 8px;
     float: right;
 }
+
 button.chat-action-button span {
     cursor: pointer;
     display: inline-block;
     position: relative;
     transition: 0.5s;
 }
+
 button.chat-action-button:hover {
     background-color: var(--background-modifier-active-hover);
     color: var(--text-normal);
@@ -534,6 +597,7 @@ img.copy-icon {
     box-sizing: border-box;
     animation: rotation 1s linear infinite;
 }
+
 .loader::after {
     content: '';
     box-sizing: border-box;
@@ -552,6 +616,7 @@ img.copy-icon {
     0% {
         transform: rotate(0deg);
     }
+
     100% {
         transform: rotate(360deg);
     }
@@ -564,6 +629,7 @@ img.copy-icon {
     width: 60px;
     height: 32px;
 }
+
 .lds-ellipsis div {
     position: absolute;
     top: 12px;
@@ -573,42 +639,52 @@ img.copy-icon {
     background: var(--color-base-70);
     animation-timing-function: cubic-bezier(0, 1, 1, 0);
 }
+
 .lds-ellipsis div:nth-child(1) {
     left: 8px;
     animation: lds-ellipsis1 0.6s infinite;
 }
+
 .lds-ellipsis div:nth-child(2) {
     left: 8px;
     animation: lds-ellipsis2 0.6s infinite;
 }
+
 .lds-ellipsis div:nth-child(3) {
     left: 32px;
     animation: lds-ellipsis2 0.6s infinite;
 }
+
 .lds-ellipsis div:nth-child(4) {
     left: 56px;
     animation: lds-ellipsis3 0.6s infinite;
 }
+
 @keyframes lds-ellipsis1 {
     0% {
         transform: scale(0);
     }
+
     100% {
         transform: scale(1);
     }
 }
+
 @keyframes lds-ellipsis3 {
     0% {
         transform: scale(1);
     }
+
     100% {
         transform: scale(0);
     }
 }
+
 @keyframes lds-ellipsis2 {
     0% {
         transform: translate(0, 0);
     }
+
     100% {
         transform: translate(24px, 0);
     }
@@ -633,15 +709,18 @@ img.copy-icon {
     border-radius: 50%;
     animation: pulse 3s ease-in-out infinite;
 }
+
 @keyframes pulse {
     0% {
         transform: scale(1);
         opacity: 1;
     }
+
     50% {
         transform: scale(1.2);
         opacity: 0.2;
     }
+
     100% {
         transform: scale(1);
         opacity: 1;
@@ -649,9 +728,15 @@ img.copy-icon {
 }
 
 @keyframes spin {
-    0% { transform: rotate(0deg); }
+    0% {
-    100% { transform: rotate(360deg); }
+        transform: rotate(0deg);
+    }
+
+    100% {
+        transform: rotate(360deg);
+    }
 }
+
 @media only screen and (max-width: 600px) {
     div.khoj-header {
         display: grid;
@@ -665,10 +750,112 @@ img.copy-icon {
         grid-gap: 0px;
         justify-content: space-between;
     }
+
     a.khoj-nav {
         padding: 0 16px;
     }
+
     span.khoj-nav-item-text {
         display: none;
     }
 }
+
+/* Folder list styles */
+.folder-list {
+    list-style: none;
+    padding: 0;
+    margin: 8px 0;
+}
+
+.folder-list-item {
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    padding: 6px 12px;
+    margin: 4px 0;
+    background: var(--background-secondary);
+    border-radius: 4px;
+    min-height: 32px;
+}
+
+.folder-list-remove {
+    background: none;
+    border: none;
+    color: #ff5555;
+    cursor: pointer;
+    font-size: 18px;
+    width: 24px;
+    height: 24px;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    margin-left: 8px;
+    padding: 0;
+    border-radius: 4px;
+    opacity: 0.7;
+    transition: all 0.2s ease;
+}
+
+.folder-list-remove:hover {
+    opacity: 1;
+    background-color: rgba(255, 85, 85, 0.1);
+}
+
+.folder-list-empty {
+    color: var(--text-muted);
+    font-style: italic;
+    padding: 6px 0;
+}
+
+/* Folder suggestion modal styles */
+.folder-suggest-item {
+    padding: 4px 8px;
+    display: block;
+}
+
+/* Loading animation */
+.khoj-loading {
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    padding: 20px;
+}
+
+.khoj-loading-spinner {
+    width: 24px;
+    height: 24px;
+    border: 3px solid var(--background-modifier-border);
+    border-top: 3px solid var(--text-accent);
+    border-radius: 50%;
+    animation: khoj-spin 1s linear infinite;
+}
+
+@keyframes khoj-spin {
+    0% {
+        transform: rotate(0deg);
+    }
+
+    100% {
+        transform: rotate(360deg);
+    }
+}
+
+/* Research Spinner */
+.search-loading-spinner {
+    width: 24px;
+    height: 24px;
+    border: 3px solid var(--background-modifier-border);
+    border-top: 3px solid var(--text-accent);
+    border-radius: 50%;
+    animation: search-spin 0.8s linear infinite;
+}
+
+@keyframes search-spin {
+    0% {
+        transform: rotate(0deg);
+    }
+
+    100% {
+        transform: rotate(360deg);
+    }
+}

src/interface/web/app/components/navMenu/navMenu.tsx

@@ -167,16 +167,7 @@ export default function FooterMenu({ sideBarIsOpen }: NavMenuProps) {
                                 </div>
                             </Link>
                         </DropdownMenuItem>
-                        {userData ? (
+                        {!userData ? (
-                            <DropdownMenuItem>
-                                <Link href="/auth/logout" className="no-underline w-full">
-                                    <div className="flex flex-rows">
-                                        <ArrowRight className="w-6 h-6" />
-                                        <p className="ml-3 font-semibold">Logout</p>
-                                    </div>
-                                </Link>
-                            </DropdownMenuItem>
-                        ) : (
                             <DropdownMenuItem>
                                 <Button
                                     variant={"ghost"}
@@ -189,7 +180,16 @@ export default function FooterMenu({ sideBarIsOpen }: NavMenuProps) {
                                     </div>
                                 </Button>
                             </DropdownMenuItem>
-                        )}
+                        ) : userData.username !== "default" ? (
+                            <DropdownMenuItem>
+                                <Link href="/auth/logout" className="no-underline w-full">
+                                    <div className="flex flex-rows">
+                                        <ArrowRight className="w-6 h-6" />
+                                        <p className="ml-3 font-semibold">Logout</p>
+                                    </div>
+                                </Link>
+                            </DropdownMenuItem>
+                        ) : null}
                     </DropdownMenuContent>
                 </DropdownMenu>
             </SidebarMenuItem>

src/interface/web/app/settings/settings.module.css

@@ -14,6 +14,11 @@ div.phoneInput {
     padding: 0rem;
 }
 
+:global(.dark) div.phoneInput :global(.iti__dropdown-content) {
+    --iti-dropdown-bg: hsl(var(--background));
+    --iti-hover-color: hsl(var(--accent));
+}
+
 div.phoneInput input {
     width: 100%;
     padding: 0.5rem;

src/khoj/configure.py

@@ -228,7 +228,7 @@ def configure_server(
 ):
     # Update Config
     if config == None:
-        logger.info(f"🚨 Khoj is not configured.\nInitializing it with a default config.")
+        logger.info(f"Initializing with default config.")
         config = FullConfig()
     state.config = config
 

src/khoj/interface/web/content_source_github_input.html

@@ -125,14 +125,6 @@
         event.preventDefault();
 
         const pat_token = document.getElementById("pat-token").value;
-
-        if (pat_token == "") {
-            document.getElementById("success").textContent = "❌ Please enter a Personal Access Token.";
-            document.getElementById("success").style.display = "block";
-            return;
-        }
-
-
         var cards = document.getElementById("repositories").getElementsByClassName("repo");
         var repos = [];
 

src/khoj/processor/content/github/github_to_entries.py

@@ -1,6 +1,6 @@
 import logging
 import time
-from typing import Any, Dict, List, Tuple
+from typing import Dict, List, Tuple
 
 import requests
 from magika import Magika
@@ -11,7 +11,7 @@ from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntrie
 from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
 from khoj.processor.content.plaintext.plaintext_to_entries import PlaintextToEntries
 from khoj.processor.content.text_to_entries import TextToEntries
-from khoj.utils.helpers import timer
+from khoj.utils.helpers import is_none_or_empty, timer
 from khoj.utils.rawconfig import GithubContentConfig, GithubRepoConfig
 
 logger = logging.getLogger(__name__)
@@ -36,7 +36,8 @@ class GithubToEntries(TextToEntries):
             repos=repos,
         )
         self.session = requests.Session()
-        self.session.headers.update({"Authorization": f"token {self.config.pat_token}"})
+        if not is_none_or_empty(self.config.pat_token):
+            self.session.headers.update({"Authorization": f"token {self.config.pat_token}"})
 
     @staticmethod
     def wait_for_rate_limit_reset(response, func, *args, **kwargs):
@@ -49,9 +50,10 @@ class GithubToEntries(TextToEntries):
             return
 
     def process(self, files: dict[str, str], user: KhojUser, regenerate: bool = False) -> Tuple[int, int]:
-        if self.config.pat_token is None or self.config.pat_token == "":
+        if is_none_or_empty(self.config.pat_token):
-            logger.error(f"Github PAT token is not set. Skipping github content")
+            logger.warning(
-            raise ValueError("Github PAT token is not set. Skipping github content")
+                f"Github PAT token is not set. Private repositories cannot be indexed and lower rate limits apply."
+            )
         current_entries = []
         for repo in self.config.repos:
             current_entries += self.process_repo(repo)
@@ -114,7 +116,9 @@ class GithubToEntries(TextToEntries):
     def get_files(self, repo_url: str, repo: GithubRepoConfig):
         # Get the contents of the repository
         repo_content_url = f"{repo_url}/git/trees/{repo.branch}"
-        headers = {"Authorization": f"token {self.config.pat_token}"}
+        headers = {}
+        if not is_none_or_empty(self.config.pat_token):
+            headers = {"Authorization": f"token {self.config.pat_token}"}
         params = {"recursive": "true"}
         response = requests.get(repo_content_url, headers=headers, params=params)
         contents = response.json()

src/khoj/processor/conversation/openai/utils.py

@@ -19,7 +19,11 @@ from khoj.processor.conversation.utils import (
     ThreadedGenerator,
     commit_conversation_trace,
 )
-from khoj.utils.helpers import get_chat_usage_metrics, is_promptrace_enabled
+from khoj.utils.helpers import (
+    get_chat_usage_metrics,
+    get_openai_client,
+    is_promptrace_enabled,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -51,10 +55,7 @@ def completion_with_backoff(
     client_key = f"{openai_api_key}--{api_base_url}"
     client: openai.OpenAI | None = openai_clients.get(client_key)
     if not client:
-        client = openai.OpenAI(
+        client = get_openai_client(openai_api_key, api_base_url)
-            api_key=openai_api_key,
-            base_url=api_base_url,
-        )
         openai_clients[client_key] = client
 
     formatted_messages = [{"role": message.role, "content": message.content} for message in messages]
@@ -161,14 +162,11 @@ def llm_thread(
 ):
     try:
         client_key = f"{openai_api_key}--{api_base_url}"
-        if client_key not in openai_clients:
+        if client_key in openai_clients:
-            client = openai.OpenAI(
-                api_key=openai_api_key,
-                base_url=api_base_url,
-            )
-            openai_clients[client_key] = client
-        else:
             client = openai_clients[client_key]
+        else:
+            client = get_openai_client(openai_api_key, api_base_url)
+            openai_clients[client_key] = client
 
         formatted_messages = [{"role": message.role, "content": message.content} for message in messages]
 

src/khoj/processor/conversation/prompts.py

@@ -74,7 +74,7 @@ no_online_results_found = PromptTemplate.from_template(
 
 no_entries_found = PromptTemplate.from_template(
     """
-    It looks like you haven't added any notes yet. No worries, you can fix that by downloading the Khoj app from <a href=https://khoj.dev/downloads>here</a>.
+    It looks like you haven't synced any notes yet. No worries, you can fix that by downloading the Khoj app from <a href=https://khoj.dev/downloads#desktop>here</a>.
 """.strip()
 )
 

src/khoj/routers/helpers.py

@@ -1263,6 +1263,7 @@ def send_message_to_model_wrapper_sync(
 
     elif chat_model.model_type == ChatModel.ModelType.OPENAI:
         api_key = chat_model.ai_model_api.api_key
+        api_base_url = chat_model.ai_model_api.api_base_url
         truncated_messages = generate_chatml_messages_with_context(
             user_message=message,
             system_message=system_message,
@@ -1277,6 +1278,7 @@ def send_message_to_model_wrapper_sync(
         openai_response = send_message_to_model(
             messages=truncated_messages,
             api_key=api_key,
+            api_base_url=api_base_url,
             model=chat_model_name,
             response_type=response_type,
             tracer=tracer,

src/khoj/utils/helpers.py

@@ -22,6 +22,7 @@ from time import perf_counter
 from typing import TYPE_CHECKING, Any, Optional, Union
 from urllib.parse import urlparse
 
+import openai
 import psutil
 import requests
 import torch
@@ -596,3 +597,20 @@ def get_chat_usage_metrics(
         "output_tokens": prev_usage["output_tokens"] + output_tokens,
         "cost": cost or get_cost_of_chat_message(model_name, input_tokens, output_tokens, prev_cost=prev_usage["cost"]),
     }
+
+
+def get_openai_client(api_key: str, api_base_url: str) -> Union[openai.OpenAI, openai.AzureOpenAI]:
+    """Get OpenAI or AzureOpenAI client based on the API Base URL"""
+    parsed_url = urlparse(api_base_url)
+    if parsed_url.hostname and parsed_url.hostname.endswith(".openai.azure.com"):
+        client = openai.AzureOpenAI(
+            api_key=api_key,
+            azure_endpoint=api_base_url,
+            api_version="2024-10-21",
+        )
+    else:
+        client = openai.OpenAI(
+            api_key=api_key,
+            base_url=api_base_url,
+        )
+    return client

src/khoj/utils/rawconfig.py

@@ -66,7 +66,7 @@ class GithubRepoConfig(ConfigBase):
 
 
 class GithubContentConfig(ConfigBase):
-    pat_token: str
+    pat_token: Optional[str] = None
     repos: List[GithubRepoConfig]