Set File Types to Sync from Obsidian via Khoj Plugin Settings Page (#904)

Limit file types to sync with Khoj from Obsidian to:
- Avoid hitting per user index-able data limits, especially for folks on the Khoj cloud free tier. E.g by excluding images in Obsidian vault from being synced
- Improve context used by Khoj to generate responses

src/interface/obsidian/src/settings.ts

@@ -10,6 +10,11 @@ export interface UserInfo {
     email?: string;
 }
 
+interface SyncFileTypes {
+    markdown: boolean;
+    images: boolean;
+    pdf: boolean;
+}
 export interface KhojSetting {
     resultsCount: number;
     khojUrl: string;
@@ -17,6 +22,7 @@ export interface KhojSetting {
     connectedToBackend: boolean;
     autoConfigure: boolean;
     lastSync: Map<TFile, number>;
+    syncFileType: SyncFileTypes;
     userInfo: UserInfo | null;
 }
 
@@ -27,6 +33,11 @@ export const DEFAULT_SETTINGS: KhojSetting = {
     connectedToBackend: false,
     autoConfigure: true,
     lastSync: new Map(),
+    syncFileType: {
+        markdown: true,
+        images: true,
+        pdf: true,
+    },
     userInfo: null,
 }
 
@@ -96,6 +107,43 @@ export class KhojSettingTab extends PluginSettingTab {
                     this.plugin.settings.resultsCount = value;
                     await this.plugin.saveSettings();
                 }));
+
+        // Add new "Sync" heading
+        containerEl.createEl('h3', {text: 'Sync'});
+
+        // Add setting to sync markdown notes
+        new Setting(containerEl)
+            .setName('Sync Notes')
+            .setDesc('Index Markdown files in your vault with Khoj.')
+            .addToggle(toggle => toggle
+                .setValue(this.plugin.settings.syncFileType.markdown)
+                .onChange(async (value) => {
+                    this.plugin.settings.syncFileType.markdown = value;
+                    await this.plugin.saveSettings();
+                }));
+
+        // Add setting to sync images
+        new Setting(containerEl)
+            .setName('Sync Images')
+            .setDesc('Index images in your vault with Khoj.')
+            .addToggle(toggle => toggle
+                .setValue(this.plugin.settings.syncFileType.images)
+                .onChange(async (value) => {
+                    this.plugin.settings.syncFileType.images = value;
+                    await this.plugin.saveSettings();
+                }));
+
+        // Add setting to sync PDFs
+        new Setting(containerEl)
+            .setName('Sync PDFs')
+            .setDesc('Index PDF files in your vault with Khoj.')
+            .addToggle(toggle => toggle
+                .setValue(this.plugin.settings.syncFileType.pdf)
+                .onChange(async (value) => {
+                    this.plugin.settings.syncFileType.pdf = value;
+                    await this.plugin.saveSettings();
+                }));
+
         new Setting(containerEl)
             .setName('Auto Sync')
             .setDesc('Automatically index your vault with Khoj.')
@@ -141,9 +189,8 @@ export class KhojSettingTab extends PluginSettingTab {
                     this.plugin.registerInterval(progress_indicator);
 
                     this.plugin.settings.lastSync = await updateContentIndex(
-                        this.app.vault, this.plugin.settings, this.plugin.settings.lastSync, true
+                        this.app.vault, this.plugin.settings, this.plugin.settings.lastSync, true, true
                     );
-                    new Notice('✅ Updated Khoj index.');
 
                     // Reset button once index is updated
                     window.clearInterval(progress_indicator);

src/interface/obsidian/src/utils.ts

@@ -48,14 +48,29 @@ function filenameToMimeType (filename: TFile): string {
     }
 }
 
-export const supportedImageFilesTypes = ['png', 'jpg', 'jpeg'];
-export const supportedBinaryFileTypes = ['pdf'].concat(supportedImageFilesTypes);
-export const supportedFileTypes = ['md', 'markdown'].concat(supportedBinaryFileTypes);
+export const fileTypeToExtension = {
+    'pdf': ['pdf'],
+    'image': ['png', 'jpg', 'jpeg'],
+    'markdown': ['md', 'markdown'],
+};
+export const supportedImageFilesTypes = fileTypeToExtension.image;
+export const supportedBinaryFileTypes = fileTypeToExtension.pdf.concat(supportedImageFilesTypes);
+export const supportedFileTypes = fileTypeToExtension.markdown.concat(supportedBinaryFileTypes);
 
-export async function updateContentIndex(vault: Vault, setting: KhojSetting, lastSync: Map<TFile, number>, regenerate: boolean = false): Promise<Map<TFile, number>> {
+export async function updateContentIndex(vault: Vault, setting: KhojSetting, lastSync: Map<TFile, number>, regenerate: boolean = false, userTriggered: boolean = false): Promise<Map<TFile, number>> {
     // Get all markdown, pdf files in the vault
     console.log(`Khoj: Updating Khoj content index...`)
-    const files = vault.getFiles().filter(file => supportedFileTypes.includes(file.extension));
+    const files = vault.getFiles()
+                        // Filter supported file types for syncing
+                        .filter(file => supportedFileTypes.includes(file.extension))
+                        // Filter user configured file types for syncing
+                        .filter(file => {
+                            if (fileTypeToExtension.markdown.includes(file.extension)) return setting.syncFileType.markdown;
+                            if (fileTypeToExtension.pdf.includes(file.extension)) return setting.syncFileType.pdf;
+                            if (fileTypeToExtension.image.includes(file.extension)) return setting.syncFileType.images;
+                            return false;
+                        });
+
     let countOfFilesToIndex = 0;
     let countOfFilesToDelete = 0;
     lastSync = lastSync.size > 0 ? lastSync : new Map<TFile, number>();
@@ -105,7 +120,37 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las
 
         if (!response.ok) {
             if (response.status === 429) {
-                error_message = `❗️Failed to sync your content with Khoj server. Requests were throttled. Upgrade your subscription or try again later.`;
+                let response_text = await response.text();
+                if (response_text.includes("Too much data")) {
+                    const errorFragment = document.createDocumentFragment();
+                    errorFragment.appendChild(document.createTextNode("❗️Exceeded data sync limits. To resolve this either:"));
+                    const bulletList = document.createElement('ul');
+
+                    const limitFilesItem = document.createElement('li');
+                    const settingsPrefixText = document.createTextNode("Limit files to sync from ");
+                    const settingsLink = document.createElement('a');
+                    settingsLink.textContent = "Khoj settings";
+                    settingsLink.href = "#";
+                    settingsLink.addEventListener('click', (e) => {
+                        e.preventDefault();
+                        openKhojPluginSettings();
+                    });
+                    limitFilesItem.appendChild(settingsPrefixText);
+                    limitFilesItem.appendChild(settingsLink);
+                    bulletList.appendChild(limitFilesItem);
+
+                    const upgradeItem = document.createElement('li');
+                    const upgradeLink = document.createElement('a');
+                    upgradeLink.href = `${setting.khojUrl}/settings#subscription`;
+                    upgradeLink.textContent = 'Upgrade your subscription';
+                    upgradeLink.target = '_blank';
+                    upgradeItem.appendChild(upgradeLink);
+                    bulletList.appendChild(upgradeItem);
+                    errorFragment.appendChild(bulletList);
+                    error_message = errorFragment;
+                } else {
+                    error_message = `❗️Failed to sync your content with Khoj server. Requests were throttled. Upgrade your subscription or try again later.`;
+                }
                 break;
             } else if (response.status === 404) {
                 error_message = `❗️Could not connect to Khoj server. Ensure you can connect to it.`;
@@ -134,12 +179,20 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las
     if (error_message) {
         new Notice(error_message);
     } else {
+        if (userTriggered) new Notice('✅ Updated Khoj index.');
         console.log(`✅ Refreshed Khoj content index. Updated: ${countOfFilesToIndex} files, Deleted: ${countOfFilesToDelete} files.`);
     }
 
     return lastSync;
 }
 
+export async function openKhojPluginSettings(): Promise<void>
+ {
+     const setting = this.app.setting;
+     await setting.open();
+     setting.openTabById('khoj');
+}
+
 export async function createNote(name: string, newLeaf = false): Promise<void> {
     try {
       let pathPrefix: string