diff --git a/src/interface/desktop/main.js b/src/interface/desktop/main.js index 83a19f36..62493f54 100644 --- a/src/interface/desktop/main.js +++ b/src/interface/desktop/main.js @@ -8,7 +8,6 @@ const {dialog} = require('electron'); const cron = require('cron').CronJob; const axios = require('axios'); -const { Readable } = require('stream'); const KHOJ_URL = 'http://127.0.0.1:42110' @@ -65,7 +64,7 @@ const schema = { var state = {} -const store = new Store({schema}); +const store = new Store({ schema }); console.log(store); @@ -86,37 +85,48 @@ function handleSetTitle (event, title) { }); } +function filenameToMimeType (filename) { + const extension = filename.split('.').pop(); + switch (extension) { + case 'pdf': + return 'application/pdf'; + case 'png': + return 'image/png'; + case 'jpg': + return 'image/jpeg'; + case 'jpeg': + return 'image/jpeg'; + case 'markdown': + return 'text/markdown'; + case 'org': + return 'text/org'; + default: + return 'text/plain'; + } +} + function pushDataToKhoj (regenerate = false) { let filesToPush = []; - const files = store.get('files'); - const folders = store.get('folders'); - state = { - completed: true + const files = store.get('files') || []; + const folders = store.get('folders') || []; + state = { completed: true } + + for (const file of files) { + filesToPush.push(file.path); } - if (files) { - for (file of files) { - filesToPush.push(file.path); - } - } - if (folders) { - for (folder of folders) { - const files = fs.readdirSync(folder.path, { withFileTypes: true }); - for (file of files) { - if (file.isFile() && validFileTypes.includes(file.name.split('.').pop())) { - filesToPush.push(path.join(folder.path, file.name)); - } + for (const folder of folders) { + const files = fs.readdirSync(folder.path, { withFileTypes: true }); + for (const file of files) { + if (file.isFile() && validFileTypes.includes(file.name.split('.').pop())) { + filesToPush.push(path.join(folder.path, file.name)); } } } - let data = { - files: [] - } - const lastSync = store.get('lastSync') || []; - - for (file of filesToPush) { + const formData = new FormData(); + for (const file of filesToPush) { const stats = fs.statSync(file); if (!regenerate) { if (stats.mtime.toISOString() < lastSync.find((syncedFile) => syncedFile.path === file)?.datetime) { @@ -125,18 +135,10 @@ function pushDataToKhoj (regenerate = false) { } try { - let rawData; - // If the file is a PDF or IMG file, read it as a binary file - if (binaryFileTypes.includes(file.split('.').pop())) { - rawData = fs.readFileSync(file).toString('base64'); - } else { - rawData = fs.readFileSync(file, 'utf8'); - } - - data.files.push({ - path: file, - content: rawData - }); + encoding = binaryFileTypes.includes(file.split('.').pop()) ? "binary" : "utf8"; + mimeType = filenameToMimeType(file) + (encoding === "utf8" ? "; charset=UTF-8" : ""); + fileObj = new Blob([fs.createReadStream(file, encoding)], { type: mimeType }); + formData.append('files', fileObj, file); state[file] = { success: true, } @@ -151,44 +153,37 @@ function pushDataToKhoj (regenerate = false) { for (const syncedFile of lastSync) { if (!filesToPush.includes(syncedFile.path)) { - data.files.push({ - path: syncedFile.path, - content: "" - }); + fileObj = new Blob([""], { type: filenameToMimeType(syncedFile.path) }); + formData.append('files', fileObj, syncedFile.path); } } - const headers = { 'x-api-key': 'secret', 'Content-Type': 'application/json' }; - - const stream = new Readable({ - read() { - this.push(JSON.stringify(data)); - this.push(null); - } - }); - - const hostURL = store.get('hostURL') || KHOJ_URL; - - axios.post(`${hostURL}/api/v1/indexer/batch?regenerate=${regenerate}`, stream, { headers }) - .then(response => { - console.log(response.data); - const win = BrowserWindow.getAllWindows()[0]; - win.webContents.send('update-state', state); - let lastSync = []; - for (const file of filesToPush) { - lastSync.push({ - path: file, - datetime: new Date().toISOString() - }); - } - store.set('lastSync', lastSync); - }) - .catch(error => { - console.error(error); - state['completed'] = false - const win = BrowserWindow.getAllWindows()[0]; - win.webContents.send('update-state', state); - }); + if (!!formData?.entries()?.next().value) { + const hostURL = store.get('hostURL') || KHOJ_URL; + const headers = { + 'x-api-key': 'secret' + }; + axios.post(`${hostURL}/api/v1/indexer/batch?regenerate=${regenerate}`, formData, { headers }) + .then(response => { + console.log(response.data); + const win = BrowserWindow.getAllWindows()[0]; + win.webContents.send('update-state', state); + let lastSync = []; + for (const file of filesToPush) { + lastSync.push({ + path: file, + datetime: new Date().toISOString() + }); + } + store.set('lastSync', lastSync); + }) + .catch(error => { + console.error(error); + state['completed'] = false + const win = BrowserWindow.getAllWindows()[0]; + win.webContents.send('update-state', state); + }); + } } pushDataToKhoj();