klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-07 13:23:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Push 1000 files at a time from the Desktop client for indexing

Browse Source 
FastAPI API endpoints only support uploading 1000 files at a time.
So split all files to index into groups of 1000 for upload to
index/update API endpoint
This commit is contained in:
Debanjum Singh Solanky
2024-01-03 23:08:20 +05:30
parent fca7a5ff32
commit efe41aaaca
1 changed files with 36 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
src/interface/desktop/main.js
View File

@@ -151,7 +151,7 @@ function pushDataToKhoj (regenerate = false) {
} }
const lastSync = store.get('lastSync') || []; const lastSync = store.get('lastSync') || [];
const formData = new FormData(); const filesDataToPush = [];
for (const file of filesToPush) { for (const file of filesToPush) {
const stats = fs.statSync(file); const stats = fs.statSync(file);
if (!regenerate) { if (!regenerate) {
@@ -167,7 +167,7 @@ function pushDataToKhoj (regenerate = false) {
let mimeType = filenameToMimeType(file) + (encoding === "utf8" ? "; charset=UTF-8" : ""); let mimeType = filenameToMimeType(file) + (encoding === "utf8" ? "; charset=UTF-8" : "");
let fileContent = Buffer.from(fs.readFileSync(file, { encoding: encoding }), encoding); let fileContent = Buffer.from(fs.readFileSync(file, { encoding: encoding }), encoding);
let fileObj = new Blob([fileContent], { type: mimeType }); let fileObj = new Blob([fileContent], { type: mimeType });
formData.append('files', fileObj, file); filesDataToPush.push({blob: fileObj, path: file});
state[file] = { state[file] = {
success: true, success: true,
} }
@@ -184,49 +184,48 @@ function pushDataToKhoj (regenerate = false) {
for (const syncedFile of lastSync) { for (const syncedFile of lastSync) {
if (!filesToPush.includes(syncedFile.path)) { if (!filesToPush.includes(syncedFile.path)) {
fileObj = new Blob([""], { type: filenameToMimeType(syncedFile.path) }); fileObj = new Blob([""], { type: filenameToMimeType(syncedFile.path) });
formData.append('files', fileObj, syncedFile.path); filesDataToPush.push({blob: fileObj, path: syncedFile.path});
} }
} }
// Send collected files to Khoj server for indexing // Send collected files to Khoj server for indexing
if (!!formData?.entries()?.next().value) { const hostURL = store.get('hostURL') || KHOJ_URL;
const hostURL = store.get('hostURL') || KHOJ_URL; const headers = { 'Authorization': `Bearer ${store.get("khojToken")}` };
const headers = { let requests = [];
'Authorization': `Bearer ${store.get("khojToken")}`
}; // Request indexing files on server. With upto 1000 files in each request
axios.post(`${hostURL}/api/v1/index/update?force=${regenerate}&client=desktop`, formData, { headers }) for (let i = 0; i < filesDataToPush.length; i += 1000) {
.then(response => { const filesDataGroup = filesDataToPush.slice(i, i + 1000);
console.log(response.data); const formData = new FormData();
let lastSync = []; filesDataGroup.forEach(fileData => { formData.append('files', fileData.blob, fileData.path) });
for (const file of filesToPush) { let request = axios.post(`${hostURL}/api/v1/index/update?force=${regenerate}&client=desktop`, formData, { headers });
lastSync.push({ requests.push(request);
path: file, }
datetime: new Date().toISOString()
}); // Wait for requests batch to finish
} Promise
store.set('lastSync', lastSync); .all(requests)
}) .then(responses => {
.catch(error => { const lastSync = filesToPush
console.error(error); .filter(file => responses.find(response => response.data.includes(file)))
if (error.response.status == 429) { .map(file => ({ path: file, datetime: new Date().toISOString() }));
const win = BrowserWindow.getAllWindows()[0]; store.set('lastSync', lastSync);
if (win) win.webContents.send('needsSubscription', true); })
if (win) win.webContents.send('update-state', state); .catch(error => {
} console.error(error);
state['completed'] = false state["completed"] = false;
}) if (error.response.status === 429) {
.finally(() => { win = BrowserWindow.getAllWindows()[0]
// Syncing complete if (win) win.webContents.send('needsSubscription', true);
syncing = false; if (win) win.webContents.send('update-state', state);
const win = BrowserWindow.getAllWindows()[0]; }
if (win) win.webContents.send('update-state', state); })
}); .finally(() => {
} else {
// Syncing complete // Syncing complete
syncing = false; syncing = false;
const win = BrowserWindow.getAllWindows()[0]; const win = BrowserWindow.getAllWindows()[0];
if (win) win.webContents.send('update-state', state); if (win) win.webContents.send('update-state', state);
} });
} }
pushDataToKhoj(); pushDataToKhoj();