Improve Syncing Obsidian Vault, Invalidate Static Assets in Browser Cache in Web Client (#657)

- Improve
  - Only send files modified since their last sync for indexing on server from the Obsidian client
- Fix 
  - Invalidate static asset browser cache in Web client when Khoj version changes

src/interface/desktop/main.js

@@ -106,6 +106,7 @@ function filenameToMimeType (filename) {
         case 'org':
             return 'text/org';
         default:
+            console.warn(`Unknown file type: ${extension}. Defaulting to text/plain.`);
             return 'text/plain';
     }
 }

src/interface/obsidian/src/main.ts

@@ -44,9 +44,7 @@ export default class Khoj extends Plugin {
         // Add scheduled job to update index every 60 minutes
         this.indexingTimer = setInterval(async () => {
             if (this.settings.autoConfigure) {
-                this.settings.lastSyncedFiles = await updateContentIndex(
-                    this.app.vault, this.settings, this.settings.lastSyncedFiles
-                );
+                this.settings.lastSync = await updateContentIndex(this.app.vault, this.settings, this.settings.lastSync);
             }
         }, 60 * 60 * 1000);
     }

src/interface/obsidian/src/settings.ts

@@ -8,7 +8,7 @@ export interface KhojSetting {
     khojApiKey: string;
     connectedToBackend: boolean;
     autoConfigure: boolean;
-    lastSyncedFiles: TFile[];
+    lastSync: Map<TFile, number>;
     userEmail: string;
 }
 
@@ -18,7 +18,7 @@ export const DEFAULT_SETTINGS: KhojSetting = {
     khojApiKey: '',
     connectedToBackend: false,
     autoConfigure: true,
-    lastSyncedFiles: [],
+    lastSync: new Map(),
     userEmail: '',
 }
 
@@ -132,8 +132,8 @@ export class KhojSettingTab extends PluginSettingTab {
                     }, 300);
                     this.plugin.registerInterval(progress_indicator);
 
-                    this.plugin.settings.lastSyncedFiles = await updateContentIndex(
-                        this.app.vault, this.plugin.settings, this.plugin.settings.lastSyncedFiles, true
+                    this.plugin.settings.lastSync = await updateContentIndex(
+                        this.app.vault, this.plugin.settings, this.plugin.settings.lastSync, true
                     );
                     new Notice('✅ Updated Khoj index.');
 

src/interface/obsidian/src/utils.ts

@@ -28,17 +28,43 @@ function fileExtensionToMimeType (extension: string): string {
     }
 }
 
-export async function updateContentIndex(vault: Vault, setting: KhojSetting, lastSyncedFiles: TFile[], regenerate: boolean = false): Promise<TFile[]> {
+function filenameToMimeType (filename: TFile): string {
+    switch (filename.extension) {
+        case 'pdf':
+            return 'application/pdf';
+        case 'png':
+            return 'image/png';
+        case 'jpg':
+        case 'jpeg':
+            return 'image/jpeg';
+        case 'md':
+        case 'markdown':
+            return 'text/markdown';
+        case 'org':
+            return 'text/org';
+        default:
+            console.warn(`Unknown file type: ${filename.extension}. Defaulting to text/plain.`);
+            return 'text/plain';
+    }
+}
+
+export async function updateContentIndex(vault: Vault, setting: KhojSetting, lastSync: Map<TFile, number>, regenerate: boolean = false): Promise<Map<TFile, number>> {
     // Get all markdown, pdf files in the vault
     console.log(`Khoj: Updating Khoj content index...`)
     const files = vault.getFiles().filter(file => file.extension === 'md' || file.extension === 'markdown' || file.extension === 'pdf');
     const binaryFileTypes = ['pdf']
     let countOfFilesToIndex = 0;
     let countOfFilesToDelete = 0;
+    lastSync = lastSync.size > 0 ? lastSync : new Map<TFile, number>();
 
     // Add all files to index as multipart form data
     const fileData = [];
     for (const file of files) {
+        // Only push files that have been modified since last sync if not regenerating
+        if (!regenerate && file.stat.mtime < (lastSync.get(file) ?? 0)){
+            continue;
+        }
+
         countOfFilesToIndex++;
         const encoding = binaryFileTypes.includes(file.extension) ? "binary" : "utf8";
         const mimeType = fileExtensionToMimeType(file.extension) + (encoding === "utf8" ? "; charset=UTF-8" : "");
@@ -47,14 +73,18 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las
     }
 
     // Add any previously synced files to be deleted to multipart form data
-    for (const lastSyncedFile of lastSyncedFiles) {
+    let filesToDelete: TFile[] = [];
+    for (const lastSyncedFile of lastSync.keys()) {
         if (!files.includes(lastSyncedFile)) {
             countOfFilesToDelete++;
-            fileData.push({blob: new Blob([]), path: lastSyncedFile.path});
+            let fileObj = new Blob([""], { type: filenameToMimeType(lastSyncedFile) });
+            fileData.push({blob: fileObj, path: lastSyncedFile.path});
+            filesToDelete.push(lastSyncedFile);
         }
     }
 
     // Iterate through all indexable files in vault, 1000 at a time
+    let responses: string[]  = [];
     let error_message = null;
     for (let i = 0; i < fileData.length; i += 1000) {
         const filesGroup = fileData.slice(i, i + 1000);
@@ -79,16 +109,31 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las
             } else {
                 error_message = `❗️Failed to sync your content with Khoj server. Raise issue on Khoj Discord or Github\nError: ${response.statusText}`;
             }
+        } else {
+            responses.push(await response.text());
         }
     }
 
+    // Update last sync time for each successfully indexed file
+   files
+    .filter(file => responses.find(response => response.includes(file.path)))
+    .reduce((newSync, file) => {
+        newSync.set(file, new Date().getTime());
+        return newSync;
+    }, lastSync);
+
+    // Remove files that were deleted from last sync
+    filesToDelete
+    .filter(file => responses.find(response => response.includes(file.path)))
+    .forEach(file => lastSync.delete(file));
+
     if (error_message) {
         new Notice(error_message);
     } else {
         console.log(`✅ Refreshed Khoj content index. Updated: ${countOfFilesToIndex} files, Deleted: ${countOfFilesToDelete} files.`);
     }
 
-    return files;
+    return lastSync;
 }
 
 export async function createNote(name: string, newLeaf = false): Promise<void> {