Merge branch 'master' of github.com:khoj-ai/khoj into features/add-a-knowledge-base-page

pyproject.toml

@@ -91,6 +91,7 @@ dependencies = [
     "google-generativeai == 0.8.3",
     "pyjson5 == 1.6.7",
     "resend == 1.0.1",
+    "email-validator == 2.2.0",
 ]
 dynamic = ["version"]
 

src/interface/obsidian/src/main.ts

@@ -34,6 +34,21 @@ export default class Khoj extends Plugin {
             callback: () => { this.activateView(KhojView.CHAT); }
         });
 
+        // Add sync command to manually sync new changes
+        this.addCommand({
+            id: 'sync',
+            name: 'Sync new changes',
+            callback: async () => {
+                this.settings.lastSync = await updateContentIndex(
+                    this.app.vault,
+                    this.settings,
+                    this.settings.lastSync,
+                    false,
+                    true
+                );
+            }
+        });
+
         this.registerView(KhojView.CHAT, (leaf) => new KhojChatView(leaf, this.settings));
 
         // Create an icon in the left ribbon.
@@ -44,12 +59,32 @@ export default class Khoj extends Plugin {
         // Add a settings tab so the user can configure khoj
         this.addSettingTab(new KhojSettingTab(this.app, this));
 
-        // Add scheduled job to update index every 60 minutes
+        // Start the sync timer
+        this.startSyncTimer();
+    }
+
+    // Method to start the sync timer
+    private startSyncTimer() {
+        // Clean up the old timer if it exists
+        if (this.indexingTimer) {
+            clearInterval(this.indexingTimer);
+        }
+
+        // Start a new timer with the configured interval
         this.indexingTimer = setInterval(async () => {
             if (this.settings.autoConfigure) {
-                this.settings.lastSync = await updateContentIndex(this.app.vault, this.settings, this.settings.lastSync);
+                this.settings.lastSync = await updateContentIndex(
+                    this.app.vault,
+                    this.settings,
+                    this.settings.lastSync
+                );
             }
-        }, 60 * 60 * 1000);
+        }, this.settings.syncInterval * 60 * 1000); // Convert minutes to milliseconds
+    }
+
+    // Public method to restart the timer (called from settings)
+    public restartSyncTimer() {
+        this.startSyncTimer();
     }
 
     async loadSettings() {
@@ -62,7 +97,7 @@ export default class Khoj extends Plugin {
     }
 
     async saveSettings() {
-        this.saveData(this.settings);
+        await this.saveData(this.settings);
     }
 
     async onunload() {

src/interface/obsidian/src/search_modal.ts

@@ -1,4 +1,4 @@
-import { App, SuggestModal, request, MarkdownRenderer, Instruction, Platform } from 'obsidian';
+import { App, SuggestModal, request, MarkdownRenderer, Instruction, Platform, Notice } from 'obsidian';
 import { KhojSetting } from 'src/settings';
 import { supportedBinaryFileTypes, createNoteAndCloseModal, getFileFromPath, getLinkToEntry, supportedImageFilesTypes } from 'src/utils';
 
@@ -13,6 +13,9 @@ export class KhojSearchModal extends SuggestModal<SearchResult> {
     find_similar_notes: boolean;
     query: string = "";
     app: App;
+    currentController: AbortController | null = null;  // To cancel requests
+    isLoading: boolean = false;
+    loadingEl: HTMLElement;
 
     constructor(app: App, setting: KhojSetting, find_similar_notes: boolean = false) {
         super(app);
@@ -23,6 +26,24 @@ export class KhojSearchModal extends SuggestModal<SearchResult> {
         // Hide input element in Similar Notes mode
         this.inputEl.hidden = this.find_similar_notes;
 
+        // Create loading element
+        this.loadingEl = createDiv({ cls: "search-loading" });
+        const spinnerEl = this.loadingEl.createDiv({ cls: "search-loading-spinner" });
+
+        this.loadingEl.style.position = "absolute";
+        this.loadingEl.style.top = "50%";
+        this.loadingEl.style.left = "50%";
+        this.loadingEl.style.transform = "translate(-50%, -50%)";
+        this.loadingEl.style.zIndex = "1000";
+        this.loadingEl.style.display = "none";
+
+        // Add the element to the modal
+        this.modalEl.appendChild(this.loadingEl);
+
+        // Customize empty state message
+        // @ts-ignore - Access to private property to customize the message
+        this.emptyStateText = "";
+
         // Register Modal Keybindings to Rerank Results
         this.scope.register(['Mod'], 'Enter', async () => {
             // Re-rank when explicitly triggered by user
@@ -66,6 +87,101 @@ export class KhojSearchModal extends SuggestModal<SearchResult> {
         this.setPlaceholder('Search with Khoj...');
     }
 
+    // Check if the file exists in the vault
+    private isFileInVault(filePath: string): boolean {
+        // Normalize the path to handle different separators
+        const normalizedPath = filePath.replace(/\\/g, '/');
+
+        // Check if the file exists in the vault
+        return this.app.vault.getFiles().some(file =>
+            file.path === normalizedPath
+        );
+    }
+
+    async getSuggestions(query: string): Promise<SearchResult[]> {
+        // Do not show loading if the query is empty
+        if (!query.trim()) {
+            this.isLoading = false;
+            this.updateLoadingState();
+            return [];
+        }
+
+        // Show loading state
+        this.isLoading = true;
+        this.updateLoadingState();
+
+        // Cancel previous request if it exists
+        if (this.currentController) {
+            this.currentController.abort();
+        }
+
+        try {
+            // Create a new controller for this request
+            this.currentController = new AbortController();
+
+            // Setup Query Khoj backend for search results
+            let encodedQuery = encodeURIComponent(query);
+            let searchUrl = `${this.setting.khojUrl}/api/search?q=${encodedQuery}&n=${this.setting.resultsCount}&r=${this.rerank}&client=obsidian`;
+            let headers = {
+                'Authorization': `Bearer ${this.setting.khojApiKey}`,
+            }
+
+            // Get search results from Khoj backend
+            const response = await fetch(searchUrl, {
+                headers: headers,
+                signal: this.currentController.signal
+            });
+
+            if (!response.ok) {
+                throw new Error(`HTTP error! status: ${response.status}`);
+            }
+
+            const data = await response.json();
+
+            // Parse search results
+            let results = data
+                .filter((result: any) =>
+                    !this.find_similar_notes || !result.additional.file.endsWith(this.app.workspace.getActiveFile()?.path)
+                )
+                .map((result: any) => {
+                    return {
+                        entry: result.entry,
+                        file: result.additional.file,
+                        inVault: this.isFileInVault(result.additional.file)
+                    } as SearchResult & { inVault: boolean };
+                })
+                .sort((a: SearchResult & { inVault: boolean }, b: SearchResult & { inVault: boolean }) => {
+                    if (a.inVault === b.inVault) return 0;
+                    return a.inVault ? -1 : 1;
+                });
+
+            this.query = query;
+
+            // Hide loading state only on successful completion
+            this.isLoading = false;
+            this.updateLoadingState();
+
+            return results;
+        } catch (error) {
+            // Ignore cancellation errors and keep loading state
+            if (error.name === 'AbortError') {
+                // When cancelling, we don't want to render anything
+                return undefined as any;
+            }
+
+            // For other errors, hide loading state
+            console.error('Search error:', error);
+            this.isLoading = false;
+            this.updateLoadingState();
+            return [];
+        }
+    }
+
+    private updateLoadingState() {
+        // Show or hide loading element
+        this.loadingEl.style.display = this.isLoading ? "block" : "none";
+    }
+
     async onOpen() {
         if (this.find_similar_notes) {
             // If markdown file is currently active
@@ -86,25 +202,7 @@ export class KhojSearchModal extends SuggestModal<SearchResult> {
         }
     }
 
-    async getSuggestions(query: string): Promise<SearchResult[]> {
+    async renderSuggestion(result: SearchResult & { inVault: boolean }, el: HTMLElement) {
-        // Setup Query Khoj backend for search results
-        let encodedQuery = encodeURIComponent(query);
-        let searchUrl = `${this.setting.khojUrl}/api/search?q=${encodedQuery}&n=${this.setting.resultsCount}&r=${this.rerank}&client=obsidian`;
-        let headers = { 'Authorization': `Bearer ${this.setting.khojApiKey}` }
-
-        // Get search results from Khoj backend
-        let response = await request({ url: `${searchUrl}`, headers: headers });
-
-        // Parse search results
-        let results = JSON.parse(response)
-            .filter((result: any) => !this.find_similar_notes || !result.additional.file.endsWith(this.app.workspace.getActiveFile()?.path))
-            .map((result: any) => { return { entry: result.entry, file: result.additional.file } as SearchResult; });
-
-        this.query = query;
-        return results;
-    }
-
-    async renderSuggestion(result: SearchResult, el: HTMLElement) {
         // Max number of lines to render
         let lines_to_render = 8;
 
@@ -112,13 +210,25 @@ export class KhojSearchModal extends SuggestModal<SearchResult> {
         let os_path_separator = result.file.includes('\\') ? '\\' : '/';
         let filename = result.file.split(os_path_separator).pop();
 
-        // Show filename of each search result for context
+        // Show filename of each search result for context with appropriate color
-        el.createEl("div",{ cls: 'khoj-result-file' }).setText(filename ?? "");
+        const fileEl = el.createEl("div", {
+            cls: `khoj-result-file ${result.inVault ? 'in-vault' : 'not-in-vault'}`
+        });
+        fileEl.setText(filename ?? "");
+
+        // Add a visual indication for files not in vault
+        if (!result.inVault) {
+            fileEl.createSpan({
+                text: " (not in vault)",
+                cls: "khoj-result-file-status"
+            });
+        }
+
         let result_el = el.createEl("div", { cls: 'khoj-result-entry' })
 
         let resultToRender = "";
         let fileExtension = filename?.split(".").pop() ?? "";
-        if (supportedImageFilesTypes.includes(fileExtension) && filename) {
+        if (supportedImageFilesTypes.includes(fileExtension) && filename && result.inVault) {
             let linkToEntry: string = filename;
             let imageFiles = this.app.vault.getFiles().filter(file => supportedImageFilesTypes.includes(fileExtension));
             // Find vault file of chosen search result
@@ -140,7 +250,13 @@ export class KhojSearchModal extends SuggestModal<SearchResult> {
         MarkdownRenderer.renderMarkdown(resultToRender, result_el, result.file, null);
     }
 
-    async onChooseSuggestion(result: SearchResult, _: MouseEvent | KeyboardEvent) {
+    async onChooseSuggestion(result: SearchResult & { inVault: boolean }, _: MouseEvent | KeyboardEvent) {
+        // Only open files that are in the vault
+        if (!result.inVault) {
+            new Notice("This file is not in your vault");
+            return;
+        }
+
         // Get all markdown, pdf and image files in vault
         const mdFiles = this.app.vault.getMarkdownFiles();
         const binaryFiles = this.app.vault.getFiles().filter(file => supportedBinaryFileTypes.includes(file.extension));

src/interface/obsidian/src/settings.ts

@@ -1,4 +1,4 @@
-import { App, Notice, PluginSettingTab, Setting, TFile } from 'obsidian';
+import { App, Notice, PluginSettingTab, Setting, TFile, SuggestModal } from 'obsidian';
 import Khoj from 'src/main';
 import { canConnectToBackend, getBackendStatusMessage, updateContentIndex } from './utils';
 
@@ -15,6 +15,7 @@ interface SyncFileTypes {
     images: boolean;
     pdf: boolean;
 }
+
 export interface KhojSetting {
     resultsCount: number;
     khojUrl: string;
@@ -24,6 +25,8 @@ export interface KhojSetting {
     lastSync: Map<TFile, number>;
     syncFileType: SyncFileTypes;
     userInfo: UserInfo | null;
+    syncFolders: string[];
+    syncInterval: number;
 }
 
 export const DEFAULT_SETTINGS: KhojSetting = {
@@ -39,6 +42,8 @@ export const DEFAULT_SETTINGS: KhojSetting = {
         pdf: true,
     },
     userInfo: null,
+    syncFolders: [],
+    syncInterval: 60,
 }
 
 export class KhojSettingTab extends PluginSettingTab {
@@ -60,7 +65,8 @@ export class KhojSettingTab extends PluginSettingTab {
                 this.plugin.settings.userInfo?.email,
                 this.plugin.settings.khojUrl,
                 this.plugin.settings.khojApiKey
-            )}
+            )
+        }
         );
         let backendStatusMessage: string = '';
 
@@ -153,6 +159,51 @@ export class KhojSettingTab extends PluginSettingTab {
                     this.plugin.settings.autoConfigure = value;
                     await this.plugin.saveSettings();
                 }));
+
+        // Add setting for sync interval
+        const syncIntervalValues = [1, 5, 10, 20, 30, 45, 60, 120, 1440];
+        new Setting(containerEl)
+            .setName('Sync Interval')
+            .setDesc('Minutes between automatic synchronizations')
+            .addDropdown(dropdown => dropdown
+                .addOptions(Object.fromEntries(
+                    syncIntervalValues.map(value => [
+                        value.toString(),
+                        value === 1 ? '1 minute' :
+                            value === 1440 ? '24 hours' :
+                                `${value} minutes`
+                    ])
+                ))
+                .setValue(this.plugin.settings.syncInterval.toString())
+                .onChange(async (value) => {
+                    this.plugin.settings.syncInterval = parseInt(value);
+                    await this.plugin.saveSettings();
+                    // Restart the timer with the new interval
+                    this.plugin.restartSyncTimer();
+                }));
+
+        // Add setting to manage sync folders
+        const syncFoldersContainer = containerEl.createDiv('sync-folders-container');
+        const foldersSetting = new Setting(syncFoldersContainer)
+            .setName('Sync Folders')
+            .setDesc('Specify folders to sync (leave empty to sync entire vault)')
+            .addButton(button => button
+                .setButtonText('Add Folder')
+                .onClick(() => {
+                    const modal = new FolderSuggestModal(this.app, (folder: string) => {
+                        if (!this.plugin.settings.syncFolders.includes(folder)) {
+                            this.plugin.settings.syncFolders.push(folder);
+                            this.plugin.saveSettings();
+                            this.updateFolderList(folderListEl);
+                        }
+                    });
+                    modal.open();
+                }));
+
+        // Create a list to display selected folders
+        const folderListEl = syncFoldersContainer.createDiv('folder-list');
+        this.updateFolderList(folderListEl);
+
         let indexVaultSetting = new Setting(containerEl);
         indexVaultSetting
             .setName('Force Sync')
@@ -200,4 +251,81 @@ export class KhojSettingTab extends PluginSettingTab {
                 })
             );
     }
+
+    // Helper method to update the folder list display
+    private updateFolderList(containerEl: HTMLElement) {
+        containerEl.empty();
+        if (this.plugin.settings.syncFolders.length === 0) {
+            containerEl.createEl('div', {
+                text: 'Syncing entire vault',
+                cls: 'folder-list-empty'
+            });
+            return;
+        }
+
+        const list = containerEl.createEl('ul', { cls: 'folder-list' });
+        this.plugin.settings.syncFolders.forEach(folder => {
+            const item = list.createEl('li', { cls: 'folder-list-item' });
+            item.createSpan({ text: folder });
+
+            const removeButton = item.createEl('button', {
+                cls: 'folder-list-remove',
+                text: '×'
+            });
+            removeButton.addEventListener('click', async () => {
+                this.plugin.settings.syncFolders = this.plugin.settings.syncFolders.filter(f => f !== folder);
+                await this.plugin.saveSettings();
+                this.updateFolderList(containerEl);
+            });
+        });
+    }
+}
+
+// Modal with folder suggestions
+class FolderSuggestModal extends SuggestModal<string> {
+    constructor(app: App, private onChoose: (folder: string) => void) {
+        super(app);
+    }
+
+    getSuggestions(query: string): string[] {
+        const folders = this.getAllFolders();
+        if (!query) return folders;
+
+        return folders.filter(folder =>
+            folder.toLowerCase().includes(query.toLowerCase())
+        );
+    }
+
+    renderSuggestion(folder: string, el: HTMLElement) {
+        el.createSpan({
+            text: folder || '/',
+            cls: 'folder-suggest-item'
+        });
+    }
+
+    onChooseSuggestion(folder: string, _: MouseEvent | KeyboardEvent) {
+        this.onChoose(folder);
+    }
+
+    private getAllFolders(): string[] {
+        const folders = new Set<string>();
+        folders.add(''); // Root folder
+
+        // Get all files and extract folder paths
+        this.app.vault.getAllLoadedFiles().forEach(file => {
+            const folderPath = file.parent?.path;
+            if (folderPath) {
+                folders.add(folderPath);
+
+                // Also add all parent folders
+                let parent = folderPath;
+                while (parent.includes('/')) {
+                    parent = parent.substring(0, parent.lastIndexOf('/'));
+                    folders.add(parent);
+                }
+            }
+        });
+
+        return Array.from(folders).sort();
+    }
 }

src/interface/obsidian/src/utils.ts

@@ -71,6 +71,15 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las
             if (fileTypeToExtension.pdf.includes(file.extension)) return setting.syncFileType.pdf;
             if (fileTypeToExtension.image.includes(file.extension)) return setting.syncFileType.images;
             return false;
+        })
+        // Filter files based on specified folders
+        .filter(file => {
+            // If no folders are specified, sync all files
+            if (setting.syncFolders.length === 0) return true;
+            // Otherwise, check if the file is in one of the specified folders
+            return setting.syncFolders.some(folder =>
+                file.path.startsWith(folder + '/') || file.path === folder
+            );
         });
 
     let countOfFilesToIndex = 0;
@@ -188,8 +197,7 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las
     return lastSync;
 }
 
-export async function openKhojPluginSettings(): Promise<void>
+export async function openKhojPluginSettings(): Promise<void> {
- {
     const setting = this.app.setting;
     await setting.open();
     setting.openTabById('khoj');

src/interface/obsidian/styles.css

@@ -18,6 +18,7 @@ If your plugin does not need CSS, delete this file.
 .khoj-chat p {
     margin: 0;
 }
+
 .khoj-chat pre {
     text-wrap: unset;
 }
@@ -33,6 +34,7 @@ If your plugin does not need CSS, delete this file.
     font-weight: 300;
     line-height: 1.5em;
 }
+
 .khoj-chat>* {
     padding: 10px;
     margin: 10px;
@@ -47,8 +49,10 @@ If your plugin does not need CSS, delete this file.
     font-size: var(--font-ui-medium);
     margin: 0px;
     line-height: 20px;
-    overflow-y: scroll; /* Make chat body scroll to see history */
+    overflow-y: scroll;
+    /* Make chat body scroll to see history */
 }
+
 /* add chat metatdata to bottom of bubble */
 .khoj-chat-message.khoj::after {
     content: attr(data-meta);
@@ -57,16 +61,19 @@ If your plugin does not need CSS, delete this file.
     color: var(--text-muted);
     margin: -12px 7px 0 0px;
 }
+
 /* move message by khoj to left */
 .khoj-chat-message.khoj {
     margin-left: auto;
     text-align: left;
 }
+
 /* move message by you to right */
 .khoj-chat-message.you {
     margin-right: auto;
     text-align: right;
 }
+
 /* basic style chat message text */
 .khoj-chat-message-text {
     margin: 10px;
@@ -80,6 +87,7 @@ If your plugin does not need CSS, delete this file.
     background-color: var(--active-bg);
     word-break: break-word;
 }
+
 /* color chat bubble by khoj blue */
 .khoj-chat-message-text.khoj {
     border-left: 2px solid var(--khoj-sun);
@@ -87,12 +95,14 @@ If your plugin does not need CSS, delete this file.
     margin-left: auto;
     white-space: pre-line;
 }
+
 /* Override white-space for ul, ol, li under khoj-chat-message-text.khoj */
 .khoj-chat-message-text.khoj ul,
 .khoj-chat-message-text.khoj ol,
 .khoj-chat-message-text.khoj li {
     white-space: normal;
 }
+
 /* add left protrusion to khoj chat bubble */
 .khoj-chat-message-text.khoj:after {
     content: '';
@@ -103,12 +113,14 @@ If your plugin does not need CSS, delete this file.
     border-bottom: 0;
     transform: rotate(-60deg);
 }
+
 /* color chat bubble by you dark grey */
 .khoj-chat-message-text.you {
     color: var(--text-normal);
     margin-right: auto;
     background-color: var(--background-modifier-cover);
 }
+
 /* add right protrusion to you chat bubble */
 .khoj-chat-message-text.you:after {
     content: '';
@@ -125,6 +137,7 @@ If your plugin does not need CSS, delete this file.
 .khoj-chat-message-text ol {
     margin: 0px 0 0;
 }
+
 .khoj-chat-message-text ol li {
     white-space: normal;
 }
@@ -146,9 +159,11 @@ code.chat-response {
 div.collapsed {
     display: none;
 }
+
 div.expanded {
     display: block;
 }
+
 div.reference {
     display: grid;
     grid-template-rows: auto;
@@ -157,6 +172,7 @@ div.reference {
     grid-row-gap: 10px;
     margin: 10px;
 }
+
 div.expanded.reference-section {
     display: grid;
     grid-template-rows: auto;
@@ -165,6 +181,7 @@ div.expanded.reference-section {
     grid-row-gap: 10px;
     margin: 10px 0;
 }
+
 button.reference-button {
     border: 1px solid var(--khoj-storm-grey);
     background-color: transparent;
@@ -183,14 +200,17 @@ button.reference-button {
     display: inline-block;
     text-wrap: inherit;
 }
+
 button.reference-button.expanded {
     height: auto;
     max-height: none;
     white-space: pre-wrap;
 }
+
 button.reference-button.expanded> :nth-child(2) {
     display: block;
 }
+
 button.reference-button.collapsed> :nth-child(2) {
     display: none;
 }
@@ -201,11 +221,13 @@ button.reference-button::before {
     display: inline-block;
     transition: transform 0.1s ease-in-out;
 }
+
 button.reference-button.expanded::before,
 button.reference-button:active:before,
 button.reference-button[aria-expanded="true"]::before {
     transform: rotate(90deg);
 }
+
 button.reference-expand-button {
     background-color: transparent;
     border: 1px solid var(--khoj-storm-grey);
@@ -219,15 +241,18 @@ button.reference-expand-button {
     transition: background 0.2s ease-in-out;
     text-align: left;
 }
+
 button.reference-expand-button:hover {
     background: var(--background-modifier-active-hover);
     color: var(--text-normal);
 }
+
 a.inline-chat-link {
     color: #475569;
     text-decoration: none;
     border-bottom: 1px dotted #475569;
 }
+
 .reference-link {
     color: var(--khoj-storm-grey);
     border-bottom: 1px dotted var(--khoj-storm-grey);
@@ -247,11 +272,13 @@ div.new-conversation {
     z-index: 10;
     background-color: var(--background-primary)
 }
+
 div.conversation-header-title {
     text-align: left;
     font-size: larger;
     line-height: 1.5em;
 }
+
 div.conversation-session {
     color: var(--color-base-90);
     border: 1px solid var(--khoj-storm-grey);
@@ -298,9 +325,11 @@ div.conversation-menu {
     grid-gap: 4px;
     grid-auto-flow: column;
 }
+
 div.conversation-session:hover {
     transform: scale(1.03);
 }
+
 div.selected-conversation {
     background: var(--background-modifier-active-hover) !important;
 }
@@ -312,6 +341,7 @@ div.selected-conversation {
     grid-column-gap: 10px;
     grid-row-gap: 10px;
 }
+
 .khoj-input-row {
     display: grid;
     grid-template-columns: 32px auto 32px 32px;
@@ -324,9 +354,11 @@ div.selected-conversation {
     bottom: 0;
     z-index: 10;
 }
+
 #khoj-chat-input.option:hover {
     box-shadow: 0 0 11px var(--background-modifier-box-shadow);
 }
+
 #khoj-chat-input {
     font-size: var(--font-ui-medium);
     padding: 4px 0 0 12px;
@@ -334,6 +366,7 @@ div.selected-conversation {
     height: 32px;
     resize: none;
 }
+
 .khoj-input-row-button {
     border-radius: 50%;
     padding: 4px;
@@ -346,43 +379,55 @@ div.selected-conversation {
     padding: 0;
     position: relative;
 }
+
 #khoj-chat-send .lucide-arrow-up-circle {
     background: var(--background-modifier-active-hover);
     border-radius: 50%;
 }
+
 #khoj-chat-send .lucide-stop-circle {
     transform: rotateY(-180deg) rotateZ(-90deg);
 }
+
 #khoj-chat-send .lucide-stop-circle circle {
-    stroke-dasharray: 62px;  /* The circumference of the circle with 7px radius */
+    stroke-dasharray: 62px;
+    /* The circumference of the circle with 7px radius */
     stroke-dashoffset: 0px;
     stroke-linecap: round;
     stroke-width: 2px;
     stroke: var(--main-text-color);
     fill: none;
 }
+
 @keyframes countdown {
     from {
         stroke-dashoffset: 0px;
     }
+
     to {
-        stroke-dashoffset: -62px;  /* The circumference of the circle with 7px radius */
+        stroke-dashoffset: -62px;
+        /* The circumference of the circle with 7px radius */
     }
 }
 
-@media (pointer: coarse), (hover: none) {
+@media (pointer: coarse),
+(hover: none) {
     #khoj-chat-body.abbr[title] {
         position: relative;
-        padding-left: 4px;  /* space references out to ease tapping */
+        padding-left: 4px;
+        /* space references out to ease tapping */
     }
+
     #khoj-chat-body.abbr[title]:focus:after {
         content: attr(title);
 
         /* position tooltip */
         position: absolute;
-        left: 16px;  /* open tooltip to right of ref link, instead of on top of it */
+        left: 16px;
+        /* open tooltip to right of ref link, instead of on top of it */
         width: auto;
-        z-index: 1;  /* show tooltip above chat messages */
+        z-index: 1;
+        /* show tooltip above chat messages */
 
         /* style tooltip */
         background-color: var(--background-secondary);
@@ -398,6 +443,14 @@ div.selected-conversation {
     font-weight: 600;
 }
 
+.khoj-result-file.in-vault {
+    color: var(--color-green);
+}
+
+.khoj-result-file.not-in-vault {
+    color: var(--color-blue);
+}
+
 .khoj-result-entry {
     color: var(--text-muted);
     margin-left: 2em;
@@ -440,9 +493,11 @@ div.khoj-header {
 a.khoj-nav {
     -webkit-app-region: no-drag;
 }
+
 div.khoj-nav {
     -webkit-app-region: no-drag;
 }
+
 nav.khoj-nav {
     display: grid;
     grid-auto-flow: column;
@@ -470,24 +525,30 @@ div.khoj-logo {
     justify-self: center;
     margin: 0;
 }
+
 .khoj-nav a:hover {
     background-color: var(--background-modifier-active-hover);
     color: var(--main-text-color);
 }
+
 a.khoj-nav-selected {
     background-color: var(--background-modifier-active-hover);
 }
+
 #similar-nav-icon-svg,
 .khoj-nav-icon {
     width: 24px;
     height: 24px;
 }
+
 .khoj-nav-icon-chat {
     background-image: var(--chat-icon);
 }
+
 .khoj-nav-icon-search {
     background-image: var(--search-icon);
 }
+
 span.khoj-nav-item-text {
     padding-left: 8px;
 }
@@ -507,12 +568,14 @@ button.chat-action-button {
     margin-top: 8px;
     float: right;
 }
+
 button.chat-action-button span {
     cursor: pointer;
     display: inline-block;
     position: relative;
     transition: 0.5s;
 }
+
 button.chat-action-button:hover {
     background-color: var(--background-modifier-active-hover);
     color: var(--text-normal);
@@ -534,6 +597,7 @@ img.copy-icon {
     box-sizing: border-box;
     animation: rotation 1s linear infinite;
 }
+
 .loader::after {
     content: '';
     box-sizing: border-box;
@@ -552,6 +616,7 @@ img.copy-icon {
     0% {
         transform: rotate(0deg);
     }
+
     100% {
         transform: rotate(360deg);
     }
@@ -564,6 +629,7 @@ img.copy-icon {
     width: 60px;
     height: 32px;
 }
+
 .lds-ellipsis div {
     position: absolute;
     top: 12px;
@@ -573,42 +639,52 @@ img.copy-icon {
     background: var(--color-base-70);
     animation-timing-function: cubic-bezier(0, 1, 1, 0);
 }
+
 .lds-ellipsis div:nth-child(1) {
     left: 8px;
     animation: lds-ellipsis1 0.6s infinite;
 }
+
 .lds-ellipsis div:nth-child(2) {
     left: 8px;
     animation: lds-ellipsis2 0.6s infinite;
 }
+
 .lds-ellipsis div:nth-child(3) {
     left: 32px;
     animation: lds-ellipsis2 0.6s infinite;
 }
+
 .lds-ellipsis div:nth-child(4) {
     left: 56px;
     animation: lds-ellipsis3 0.6s infinite;
 }
+
 @keyframes lds-ellipsis1 {
     0% {
         transform: scale(0);
     }
+
     100% {
         transform: scale(1);
     }
 }
+
 @keyframes lds-ellipsis3 {
     0% {
         transform: scale(1);
     }
+
     100% {
         transform: scale(0);
     }
 }
+
 @keyframes lds-ellipsis2 {
     0% {
         transform: translate(0, 0);
     }
+
     100% {
         transform: translate(24px, 0);
     }
@@ -633,15 +709,18 @@ img.copy-icon {
     border-radius: 50%;
     animation: pulse 3s ease-in-out infinite;
 }
+
 @keyframes pulse {
     0% {
         transform: scale(1);
         opacity: 1;
     }
+
     50% {
         transform: scale(1.2);
         opacity: 0.2;
     }
+
     100% {
         transform: scale(1);
         opacity: 1;
@@ -649,9 +728,15 @@ img.copy-icon {
 }
 
 @keyframes spin {
-    0% { transform: rotate(0deg); }
+    0% {
-    100% { transform: rotate(360deg); }
+        transform: rotate(0deg);
     }
+
+    100% {
+        transform: rotate(360deg);
+    }
+}
+
 @media only screen and (max-width: 600px) {
     div.khoj-header {
         display: grid;
@@ -665,10 +750,112 @@ img.copy-icon {
         grid-gap: 0px;
         justify-content: space-between;
     }
+
     a.khoj-nav {
         padding: 0 16px;
     }
+
     span.khoj-nav-item-text {
         display: none;
     }
 }
+
+/* Folder list styles */
+.folder-list {
+    list-style: none;
+    padding: 0;
+    margin: 8px 0;
+}
+
+.folder-list-item {
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    padding: 6px 12px;
+    margin: 4px 0;
+    background: var(--background-secondary);
+    border-radius: 4px;
+    min-height: 32px;
+}
+
+.folder-list-remove {
+    background: none;
+    border: none;
+    color: #ff5555;
+    cursor: pointer;
+    font-size: 18px;
+    width: 24px;
+    height: 24px;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    margin-left: 8px;
+    padding: 0;
+    border-radius: 4px;
+    opacity: 0.7;
+    transition: all 0.2s ease;
+}
+
+.folder-list-remove:hover {
+    opacity: 1;
+    background-color: rgba(255, 85, 85, 0.1);
+}
+
+.folder-list-empty {
+    color: var(--text-muted);
+    font-style: italic;
+    padding: 6px 0;
+}
+
+/* Folder suggestion modal styles */
+.folder-suggest-item {
+    padding: 4px 8px;
+    display: block;
+}
+
+/* Loading animation */
+.khoj-loading {
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    padding: 20px;
+}
+
+.khoj-loading-spinner {
+    width: 24px;
+    height: 24px;
+    border: 3px solid var(--background-modifier-border);
+    border-top: 3px solid var(--text-accent);
+    border-radius: 50%;
+    animation: khoj-spin 1s linear infinite;
+}
+
+@keyframes khoj-spin {
+    0% {
+        transform: rotate(0deg);
+    }
+
+    100% {
+        transform: rotate(360deg);
+    }
+}
+
+/* Research Spinner */
+.search-loading-spinner {
+    width: 24px;
+    height: 24px;
+    border: 3px solid var(--background-modifier-border);
+    border-top: 3px solid var(--text-accent);
+    border-radius: 50%;
+    animation: search-spin 0.8s linear infinite;
+}
+
+@keyframes search-spin {
+    0% {
+        transform: rotate(0deg);
+    }
+
+    100% {
+        transform: rotate(360deg);
+    }
+}

src/khoj/database/adapters/__init__.py

@@ -72,6 +72,7 @@ from khoj.utils.helpers import (
     generate_random_name,
     in_debug_mode,
     is_none_or_empty,
+    normalize_email,
     timer,
 )
 
@@ -231,13 +232,18 @@ async def acreate_user_by_phone_number(phone_number: str) -> KhojUser:
     return user
 
 
-async def aget_or_create_user_by_email(email: str) -> tuple[KhojUser, bool]:
+async def aget_or_create_user_by_email(input_email: str) -> tuple[KhojUser, bool]:
+    email, is_valid_email = normalize_email(input_email)
+    is_existing_user = await KhojUser.objects.filter(email=email).aexists()
+    # Validate email address of new users
+    if not is_existing_user and not is_valid_email:
+        logger.error(f"Account creation failed. Invalid email address: {email}")
+        return None, False
+
     user, is_new = await KhojUser.objects.filter(email=email).aupdate_or_create(
         defaults={"username": email, "email": email}
     )
-    await user.asave()
 
-    if user:
     # Generate a secure 6-digit numeric code
     user.email_verification_code = f"{secrets.randbelow(1000000):06}"
     user.email_verification_code_expiry = datetime.now(tz=timezone.utc) + timedelta(minutes=5)
@@ -270,10 +276,15 @@ async def astart_trial_subscription(user: KhojUser) -> Subscription:
 
 
 async def aget_user_validated_by_email_verification_code(code: str, email: str) -> tuple[Optional[KhojUser], bool]:
-    user = await KhojUser.objects.filter(email_verification_code=code, email=email).afirst()
+    # Normalize the email address
+    normalized_email, _ = normalize_email(email)
+
+    # Check if verification code exists for the user
+    user = await KhojUser.objects.filter(email_verification_code=code, email=normalized_email).afirst()
     if not user:
         return None, False
 
+    # Check if the code has expired
     if user.email_verification_code_expiry < datetime.now(tz=timezone.utc):
         return user, True
 
@@ -348,6 +359,8 @@ async def set_user_subscription(
 ) -> tuple[Optional[Subscription], bool]:
     # Get or create the user object and their subscription
     user, is_new = await aget_or_create_user_by_email(email)
+    if not user:
+        return None, is_new
     user_subscription = await Subscription.objects.filter(user=user).afirst()
 
     # Update the user subscription state

src/khoj/routers/auth.py

@@ -86,12 +86,11 @@ async def login_magic_link(request: Request, form: MagicLinkForm):
         # Clear the session if user is already authenticated
         request.session.pop("user", None)
 
-    email = form.email
+    user, is_new = await aget_or_create_user_by_email(form.email)
-    user, is_new = await aget_or_create_user_by_email(email)
-    unique_id = user.email_verification_code
 
     if user:
-        await send_magic_link_email(email, unique_id, request.base_url)
+        unique_id = user.email_verification_code
+        await send_magic_link_email(user.email, unique_id, request.base_url)
         if is_new:
             update_telemetry_state(
                 request=request,

src/khoj/routers/helpers.py

@@ -1135,6 +1135,7 @@ def send_message_to_model_wrapper_sync(
 
     elif chat_model.model_type == ChatModel.ModelType.OPENAI:
         api_key = chat_model.ai_model_api.api_key
+        api_base_url = chat_model.ai_model_api.api_base_url
         truncated_messages = generate_chatml_messages_with_context(
             user_message=message,
             system_message=system_message,
@@ -1149,6 +1150,7 @@ def send_message_to_model_wrapper_sync(
         openai_response = send_message_to_model(
             messages=truncated_messages,
             api_key=api_key,
+            api_base_url=api_base_url,
             model=chat_model_name,
             response_type=response_type,
             tracer=tracer,

src/khoj/utils/helpers.py

@@ -27,6 +27,7 @@ import psutil
 import requests
 import torch
 from asgiref.sync import sync_to_async
+from email_validator import EmailNotValidError, EmailUndeliverableError, validate_email
 from magika import Magika
 from PIL import Image
 from pytz import country_names, country_timezones
@@ -614,3 +615,13 @@ def get_openai_client(api_key: str, api_base_url: str) -> Union[openai.OpenAI, o
             base_url=api_base_url,
         )
     return client
+
+
+def normalize_email(email: str, check_deliverability=False) -> tuple[str, bool]:
+    """Normalize, validate and check deliverability of email address"""
+    lower_email = email.lower()
+    try:
+        valid_email = validate_email(lower_email, check_deliverability=check_deliverability)
+        return valid_email.normalized, True
+    except (EmailNotValidError, EmailUndeliverableError):
+        return lower_email, False