mirror of
https://github.com/khoaliber/khoj.git
synced 2026-03-02 21:19:12 +00:00
Add front-end Electron application for Khoj local file syncing (#473)
* Initial version - setup a file-push architecture for generating embeddings with Khoj * Use state.host and state.port for configuring the URL for the indexer * Fix parsing of PDF files * Read markdown files from streamed data and update unit tests * On application startup, load in embeddings from configurations files, rather than regenerating the corpus based on file system * Init: refactor indexer/batch endpoint to support a generic file ingestion format * Add features to better support indexing from files sent by the desktop client * Initial commit with Electron application - Adds electron app * Add import for pymupdf, remove import for pypdf * Allow user to configure khoj host URL * Remove search type configuration from index.html * Use v1 path for current indexer routes
This commit is contained in:
sabaimran
359
src/interface/desktop/main.js
Normal file
359
src/interface/desktop/main.js
Normal file
@@ -0,0 +1,359 @@
|
||||
const { app, BrowserWindow, ipcMain } = require('electron');
|
||||
const todesktop = require("@todesktop/runtime");
|
||||
|
||||
todesktop.init();
|
||||
|
||||
const fs = require('fs');
|
||||
const {dialog} = require('electron');
|
||||
|
||||
const cron = require('cron').CronJob;
|
||||
const axios = require('axios');
|
||||
const { Readable } = require('stream');
|
||||
|
||||
const KHOJ_URL = 'http://127.0.0.1:42110'
|
||||
|
||||
const Store = require('electron-store');
|
||||
|
||||
const validFileTypes = ['org', 'md', 'markdown', 'txt', 'html', 'xml', 'pdf']
|
||||
|
||||
const binaryFileTypes = ['pdf', 'png', 'jpg', 'jpeg']
|
||||
|
||||
const schema = {
|
||||
files: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
path: {
|
||||
type: 'string'
|
||||
}
|
||||
}
|
||||
},
|
||||
default: []
|
||||
},
|
||||
folders: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
path: {
|
||||
type: 'string'
|
||||
}
|
||||
}
|
||||
},
|
||||
default: []
|
||||
},
|
||||
hostURL: {
|
||||
type: 'string',
|
||||
default: KHOJ_URL
|
||||
},
|
||||
lastSync: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
path: {
|
||||
type: 'string'
|
||||
},
|
||||
datetime: {
|
||||
type: 'string'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
var state = {}
|
||||
|
||||
const store = new Store({schema});
|
||||
|
||||
console.log(store);
|
||||
|
||||
// include the Node.js 'path' module at the top of your file
|
||||
const path = require('path');
|
||||
|
||||
function handleSetTitle (event, title) {
|
||||
const webContents = event.sender
|
||||
const win = BrowserWindow.fromWebContents(webContents)
|
||||
win.setTitle(title)
|
||||
dialog.showOpenDialog({properties: ['openFile', 'openDirectory'] }).then(function (response) {
|
||||
if (!response.canceled) {
|
||||
// handle fully qualified file name
|
||||
console.log(response.filePaths[0]);
|
||||
} else {
|
||||
console.log("no file selected");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function pushDataToKhoj () {
|
||||
let filesToPush = [];
|
||||
const files = store.get('files');
|
||||
const folders = store.get('folders');
|
||||
state = {
|
||||
completed: true
|
||||
}
|
||||
|
||||
if (files) {
|
||||
for (file of files) {
|
||||
filesToPush.push(file.path);
|
||||
}
|
||||
}
|
||||
if (folders) {
|
||||
for (folder of folders) {
|
||||
const files = fs.readdirSync(folder.path, { withFileTypes: true });
|
||||
for (file of files) {
|
||||
if (file.isFile() && validFileTypes.includes(file.name.split('.').pop())) {
|
||||
filesToPush.push(path.join(folder.path, file.name));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let data = {
|
||||
files: []
|
||||
}
|
||||
|
||||
const lastSync = store.get('lastSync') || [];
|
||||
|
||||
for (file of filesToPush) {
|
||||
const stats = fs.statSync(file);
|
||||
if (stats.mtime.toISOString() < lastSync.find((syncedFile) => syncedFile.path === file)?.datetime) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
let rawData;
|
||||
// If the file is a PDF or IMG file, read it as a binary file
|
||||
if (binaryFileTypes.includes(file.split('.').pop())) {
|
||||
rawData = fs.readFileSync(file).toString('base64');
|
||||
} else {
|
||||
rawData = fs.readFileSync(file, 'utf8');
|
||||
}
|
||||
|
||||
data.files.push({
|
||||
path: file,
|
||||
content: rawData
|
||||
});
|
||||
state[file] = {
|
||||
success: true,
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
state[file] = {
|
||||
success: false,
|
||||
error: err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const syncedFile of lastSync) {
|
||||
if (!filesToPush.includes(syncedFile.path)) {
|
||||
data.files.push({
|
||||
path: syncedFile.path,
|
||||
content: ""
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const headers = { 'x-api-key': 'secret', 'Content-Type': 'application/json' };
|
||||
|
||||
const stream = new Readable({
|
||||
read() {
|
||||
this.push(JSON.stringify(data));
|
||||
this.push(null);
|
||||
}
|
||||
});
|
||||
|
||||
const hostURL = store.get('hostURL') || KHOJ_URL;
|
||||
|
||||
axios.post(`${hostURL}/v1/indexer/batch`, stream, { headers })
|
||||
.then(response => {
|
||||
console.log(response.data);
|
||||
const win = BrowserWindow.getAllWindows()[0];
|
||||
win.webContents.send('update-state', state);
|
||||
let lastSync = [];
|
||||
for (const file of filesToPush) {
|
||||
lastSync.push({
|
||||
path: file,
|
||||
datetime: new Date().toISOString()
|
||||
});
|
||||
}
|
||||
store.set('lastSync', lastSync);
|
||||
})
|
||||
.catch(error => {
|
||||
console.error(error);
|
||||
state['completed'] = false
|
||||
const win = BrowserWindow.getAllWindows()[0];
|
||||
win.webContents.send('update-state', state);
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
pushDataToKhoj();
|
||||
|
||||
async function handleFileOpen (event, key) {
|
||||
const { canceled, filePaths } = await dialog.showOpenDialog({properties: ['openFile', 'openDirectory'], filters: [{ name: "Valid Khoj Files", extensions: validFileTypes}] });
|
||||
if (!canceled) {
|
||||
const files = store.get('files') || [];
|
||||
const folders = store.get('folders') || [];
|
||||
|
||||
for (const filePath of filePaths) {
|
||||
console.log(filePath);
|
||||
if (fs.existsSync(filePath)) {
|
||||
const stats = fs.statSync(filePath);
|
||||
if (stats.isFile()) {
|
||||
console.log(`${filePath} is a file.`);
|
||||
|
||||
if (files.find((file) => file.path === filePath)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
files.push({path: filePath});
|
||||
store.set('files', files);
|
||||
} else if (stats.isDirectory()) {
|
||||
console.log(`${filePath} is a directory.`);
|
||||
|
||||
if (folders.find((folder) => folder.path === filePath)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
folders.push({path: filePath});
|
||||
store.set('folders', folders);
|
||||
}
|
||||
|
||||
} else {
|
||||
console.log(`${filePath} does not exist.`);
|
||||
}
|
||||
}
|
||||
return {
|
||||
files: store.get('files'),
|
||||
folders: store.get('folders')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function getFiles () {
|
||||
return store.get('files');
|
||||
}
|
||||
|
||||
async function getFolders () {
|
||||
return store.get('folders');
|
||||
}
|
||||
|
||||
async function setURL (event, url) {
|
||||
store.set('hostURL', url);
|
||||
return store.get('hostURL');
|
||||
}
|
||||
|
||||
async function getURL () {
|
||||
return store.get('hostURL');
|
||||
}
|
||||
|
||||
async function removeFile (event, filePath) {
|
||||
const files = store.get('files');
|
||||
const newFiles = files.filter((file) => file.path !== filePath);
|
||||
store.set('files', newFiles);
|
||||
return newFiles;
|
||||
}
|
||||
|
||||
async function removeFolder (event, folderPath) {
|
||||
const folders = store.get('folders');
|
||||
const newFolders = folders.filter((folder) => folder.path !== folderPath);
|
||||
store.set('folders', newFolders);
|
||||
return newFolders;
|
||||
}
|
||||
|
||||
async function syncData () {
|
||||
try {
|
||||
pushDataToKhoj();
|
||||
const date = new Date();
|
||||
console.log('Pushing data to Khoj at: ', date);
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
}
|
||||
}
|
||||
|
||||
const createWindow = () => {
|
||||
const win = new BrowserWindow({
|
||||
width: 800,
|
||||
height: 800,
|
||||
// titleBarStyle: 'hidden',
|
||||
webPreferences: {
|
||||
preload: path.join(__dirname, 'preload.js'),
|
||||
nodeIntegration: true,
|
||||
}
|
||||
})
|
||||
|
||||
const job = new cron('0 */10 * * * *', function() {
|
||||
try {
|
||||
pushDataToKhoj();
|
||||
const date = new Date();
|
||||
console.log('Pushing data to Khoj at: ', date);
|
||||
win.webContents.send('update-state', state);
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
}
|
||||
});
|
||||
|
||||
win.setResizable(true);
|
||||
win.setOpacity(0.95);
|
||||
win.setBackgroundColor('#FFFFFF');
|
||||
win.setHasShadow(true);
|
||||
|
||||
job.start();
|
||||
|
||||
win.loadFile('index.html')
|
||||
}
|
||||
|
||||
app.whenReady().then(() => {
|
||||
ipcMain.on('set-title', handleSetTitle);
|
||||
|
||||
ipcMain.handle('getStoreValue', handleFileOpen);
|
||||
|
||||
ipcMain.on('update-state', (event, arg) => {
|
||||
console.log(arg);
|
||||
event.reply('update-state', arg);
|
||||
});
|
||||
|
||||
ipcMain.handle('getFiles', getFiles);
|
||||
ipcMain.handle('getFolders', getFolders);
|
||||
|
||||
ipcMain.handle('removeFile', removeFile);
|
||||
ipcMain.handle('removeFolder', removeFolder);
|
||||
|
||||
ipcMain.handle('setURL', setURL);
|
||||
ipcMain.handle('getURL', getURL);
|
||||
|
||||
ipcMain.handle('syncData', syncData);
|
||||
|
||||
createWindow()
|
||||
|
||||
app.setAboutPanelOptions({
|
||||
applicationName: "Khoj",
|
||||
applicationVersion: "0.0.1",
|
||||
version: "0.0.1",
|
||||
authors: "Khoj Team",
|
||||
website: "https://khoj.dev",
|
||||
iconPath: path.join(__dirname, 'assets', 'khoj.png')
|
||||
});
|
||||
|
||||
app.on('ready', async() => {
|
||||
try {
|
||||
const result = await todesktop.autoUpdater.checkForUpdates();
|
||||
if (result.updateInfo) {
|
||||
console.log("Update found:", result.updateInfo.version);
|
||||
todesktop.autoUpdater.restartAndInstall();
|
||||
}
|
||||
} catch (e) {
|
||||
console.log("Update check failed:", e);
|
||||
}
|
||||
})
|
||||
|
||||
app.on('activate', () => {
|
||||
if (BrowserWindow.getAllWindows().length === 0) createWindow()
|
||||
})
|
||||
})
|
||||
|
||||
app.on('window-all-closed', () => {
|
||||
if (process.platform !== 'darwin') app.quit()
|
||||
})
|
||||
Reference in New Issue
Block a user