[Multi-User Part 4]: Authenticate using API Tokens (#513)

###  New
- Use API keys to authenticate from Desktop, Obsidian, Emacs clients
- Create API, UI on web app config page to CRUD API Keys
- Create user API keys table and functions to CRUD them in Database

### 🧪 Improve
- Default to better search model, [gte-small](https://huggingface.co/thenlper/gte-small), to improve search quality
- Only load chat model to GPU if enough space, throw error on load failure
- Show encoding progress, truncate headings to max chars supported
- Add instruction to create db in Django DB setup Readme

### ⚙️ Fix
- Fix error handling when configure offline chat via Web UI
- Do not warn in anon mode about Google OAuth env vars not being set
- Fix path to load static files when server started from project root
This commit is contained in:
Debanjum
2023-10-26 12:33:03 -07:00
committed by GitHub
parent 4b6ec248a6
commit 9acc722f7f
36 changed files with 692 additions and 564 deletions

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.6 KiB

View File

@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<svg width="800px" height="800px" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" clip-rule="evenodd" d="M22 8.29344C22 11.7692 19.1708 14.5869 15.6807 14.5869C15.0439 14.5869 13.5939 14.4405 12.8885 13.8551L12.0067 14.7333C11.4883 15.2496 11.6283 15.4016 11.8589 15.652C11.9551 15.7565 12.0672 15.8781 12.1537 16.0505C12.1537 16.0505 12.8885 17.075 12.1537 18.0995C11.7128 18.6849 10.4783 19.5045 9.06754 18.0995L8.77362 18.3922C8.77362 18.3922 9.65538 19.4167 8.92058 20.4412C8.4797 21.0267 7.30403 21.6121 6.27531 20.5876L5.2466 21.6121C4.54119 22.3146 3.67905 21.9048 3.33616 21.6121L2.45441 20.7339C1.63143 19.9143 2.1115 19.0264 2.45441 18.6849L10.0963 11.0743C10.0963 11.0743 9.3615 9.90338 9.3615 8.29344C9.3615 4.81767 12.1907 2 15.6807 2C19.1708 2 22 4.81767 22 8.29344ZM15.681 10.4889C16.8984 10.4889 17.8853 9.50601 17.8853 8.29353C17.8853 7.08105 16.8984 6.09814 15.681 6.09814C14.4635 6.09814 13.4766 7.08105 13.4766 8.29353C13.4766 9.50601 14.4635 10.4889 15.681 10.4889Z" fill="#1C274C"/>
</svg>

After

Width:  |  Height:  |  Size: 1.1 KiB

View File

@@ -1,5 +1,4 @@
<?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
<?xml version="1.0" encoding="utf-8"?>
<svg width="800px" height="800px" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M15.197 3.35462C16.8703 1.67483 19.4476 1.53865 20.9536 3.05046C22.4596 4.56228 22.3239 7.14956 20.6506 8.82935L18.2268 11.2626M10.0464 14C8.54044 12.4882 8.67609 9.90087 10.3494 8.22108L12.5 6.06212" stroke="#1C274C" stroke-width="1.5" stroke-linecap="round"/>
<path d="M13.9536 10C15.4596 11.5118 15.3239 14.0991 13.6506 15.7789L11.2268 18.2121L8.80299 20.6454C7.12969 22.3252 4.55237 22.4613 3.0464 20.9495C1.54043 19.4377 1.67609 16.8504 3.34939 15.1706L5.77323 12.7373" stroke="#1C274C" stroke-width="1.5" stroke-linecap="round"/>
<path d="M9.16488 17.6505C8.92513 17.8743 8.73958 18.0241 8.54996 18.1336C7.62175 18.6695 6.47816 18.6695 5.54996 18.1336C5.20791 17.9361 4.87912 17.6073 4.22153 16.9498C3.56394 16.2922 3.23514 15.9634 3.03767 15.6213C2.50177 14.6931 2.50177 13.5495 3.03767 12.6213C3.23514 12.2793 3.56394 11.9505 4.22153 11.2929L7.04996 8.46448C7.70755 7.80689 8.03634 7.47809 8.37838 7.28062C9.30659 6.74472 10.4502 6.74472 11.3784 7.28061C11.7204 7.47809 12.0492 7.80689 12.7068 8.46448C13.3644 9.12207 13.6932 9.45086 13.8907 9.7929C14.4266 10.7211 14.4266 11.8647 13.8907 12.7929C13.7812 12.9825 13.6314 13.1681 13.4075 13.4078M10.5919 10.5922C10.368 10.8319 10.2182 11.0175 10.1087 11.2071C9.57284 12.1353 9.57284 13.2789 10.1087 14.2071C10.3062 14.5492 10.635 14.878 11.2926 15.5355C11.9502 16.1931 12.279 16.5219 12.621 16.7194C13.5492 17.2553 14.6928 17.2553 15.621 16.7194C15.9631 16.5219 16.2919 16.1931 16.9495 15.5355L19.7779 12.7071C20.4355 12.0495 20.7643 11.7207 20.9617 11.3787C21.4976 10.4505 21.4976 9.30689 20.9617 8.37869C20.7643 8.03665 20.4355 7.70785 19.7779 7.05026C19.1203 6.39267 18.7915 6.06388 18.4495 5.8664C17.5212 5.3305 16.3777 5.3305 15.4495 5.8664C15.2598 5.97588 15.0743 6.12571 14.8345 6.34955" stroke="#000000" stroke-width="2" stroke-linecap="round"/>
</svg>

Before

Width:  |  Height:  |  Size: 777 B

After

Width:  |  Height:  |  Size: 1.4 KiB

View File

@@ -89,6 +89,8 @@
// Generate backend API URL to execute query
let url = `${hostURL}/api/chat?q=${encodeURIComponent(query)}&n=${resultsCount}&client=web&stream=true`;
const khojToken = await window.tokenAPI.getToken();
const headers = { 'Authorization': `Bearer ${khojToken}` };
let chat_body = document.getElementById("chat-body");
let new_response = document.createElement("div");
@@ -113,7 +115,7 @@
chatInput.classList.remove("option-enabled");
// Call specified Khoj API which returns a streamed response of type text/plain
fetch(url)
fetch(url, { headers })
.then(response => {
const reader = response.body.getReader();
const decoder = new TextDecoder();
@@ -217,7 +219,10 @@
async function loadChat() {
const hostURL = await window.hostURLAPI.getURL();
fetch(`${hostURL}/api/chat/history?client=web`)
const khojToken = await window.tokenAPI.getToken();
const headers = { 'Authorization': `Bearer ${khojToken}` };
fetch(`${hostURL}/api/chat/history?client=web`, { headers })
.then(response => response.json())
.then(data => {
if (data.detail) {
@@ -243,7 +248,7 @@
return;
});
fetch(`${hostURL}/api/chat/options`)
fetch(`${hostURL}/api/chat/options`, { headers })
.then(response => response.json())
.then(data => {
// Render chat options, if any
@@ -272,9 +277,9 @@
<img class="khoj-logo" src="./assets/icons/khoj-logo-sideways-500.png" alt="Khoj"></img>
</a>
<nav class="khoj-nav">
<a class="khoj-nav khoj-nav-selected" href="./chat.html">Chat</a>
<a class="khoj-nav" href="./index.html">Search</a>
<a class="khoj-nav" href="./config.html">⚙️</a>
<a class="khoj-nav khoj-nav-selected" href="./chat.html">💬 Chat</a>
<a class="khoj-nav" href="./index.html">🔎 Search</a>
<a class="khoj-nav" href="./config.html">⚙️ Settings</a>
</nav>
</div>

View File

@@ -12,66 +12,85 @@
<script type="text/javascript" src="./assets/markdown-it.min.js"></script>
<body>
<div class="page">
<!--Add Header Logo and Nav Pane-->
<div class="khoj-header">
<a class="khoj-logo" href="./index.html">
<img class="khoj-logo" src="./assets/icons/khoj-logo-sideways-500.png" alt="Khoj"></img>
</a>
<nav class="khoj-nav">
<a class="khoj-nav" href="./chat.html">Chat</a>
<a class="khoj-nav" href="./index.html">Search</a>
<a class="khoj-nav khoj-nav-selected" href="./config.html">⚙️</a>
</nav>
</div>
<!--Add Header Logo and Nav Pane-->
<div class="khoj-header">
<a class="khoj-logo" href="./index.html">
<img class="khoj-logo" src="./assets/icons/khoj-logo-sideways-500.png" alt="Khoj"></img>
</a>
<nav class="khoj-nav">
<a class="khoj-nav" href="./chat.html">💬 Chat</a>
<a class="khoj-nav" href="./index.html">🔎 Search</a>
<a class="khoj-nav khoj-nav-selected" href="./config.html">⚙️ Settings</a>
</nav>
</div>
<div class="section-cards">
<div class="card configuration">
<div class="card-title-row">
<img class="card-icon" src="./assets/icons/link.svg" alt="File">
<h3 class="card-title">
Host
</h3>
<div class="card-description-row">
<div class="card configuration">
<div class="card-title-row">
<img class="card-icon" src="./assets/icons/link.svg" alt="Khoj Server URL">
<h3 class="card-title">
Server URL
</h3>
</div>
<div class="card-description-row">
<input id="khoj-host-url" class="card-input" type="text">
</div>
<div class="card-title-row">
<img class="card-icon" src="./assets/icons/key.svg" alt="Khoj Access Key">
<h3 class="card-title">
Access Key
</h3>
</div>
<div class="card-description-row">
<input id="khoj-access-key" class="card-input" type="text" placeholder="Enter key to access your Khoj">
</div>
</div>
<div class="card-description-row">
<input id="khoj-host-url" class="card-input" type="text">
</div>
<div class="card-title-row">
<img class="card-icon" src="./assets/icons/plaintext.svg" alt="File">
<h3 class="card-title">
Files
<button id="toggle-files" class="card-button">
<svg id="toggle-files-svg" xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 5v14M5 12l7 7 7-7"></path></svg>
</div>
<div class="card-description-row">
<div class="card configuration">
<div class="card-title-row">
<img class="card-icon" src="./assets/icons/plaintext.svg" alt="File">
<h3 class="card-title">
Files
<button id="toggle-files" class="card-button">
<svg id="toggle-files-svg" xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 5v14M5 12l7 7 7-7"></path></svg>
</button>
</h3>
</div>
<div class="card-description-row">
<div id="current-files"></div>
</div>
<div class="card-action-row">
<button id="update-file" class="card-button">
Add
<img class="add-files-icon" src="./assets/icons/circular-add.svg" alt="Add">
</button>
</h3>
</div>
</div>
<div class="card-description-row">
<div id="current-files"></div>
</div>
<div class="card-action-row">
<button id="update-file" class="card-button">
Add
<img class="add-files-icon" src="./assets/icons/circular-add.svg" alt="Add">
</button>
</div>
<div class="card-title-row">
<img class="card-icon" src="./assets/icons/folder.svg" alt="Folder">
<h3 class="card-title">
Folders
<button id="toggle-folders" class="card-button">
<svg id="toggle-folders-svg" xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 5v14M5 12l7 7 7-7"></path></svg>
</div>
<div class="card-description-row">
<div class="card configuration">
<div class="card-title-row">
<img class="card-icon" src="./assets/icons/folder.svg" alt="Folder">
<h3 class="card-title">
Folders
<button id="toggle-folders" class="card-button">
<svg id="toggle-folders-svg" xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 5v14M5 12l7 7 7-7"></path></svg>
</button>
</h3>
</div>
<div class="card-description-row">
<div id="current-folders"></div>
</div>
<div class="card-action-row">
<button id="update-folder" class="card-button">
Add
<img class="add-files-icon" src="./assets/icons/circular-add.svg" alt="Add">
</button>
</h3>
</div>
<div class="card-description-row">
<div id="current-folders"></div>
</div>
<div class="card-action-row">
<button id="update-folder" class="card-button">
Add
<img class="add-files-icon" src="./assets/icons/circular-add.svg" alt="Add">
</button>
</div>
</div>
</div>
<div class="section-action-row">
<div class="card-description-row">
<button id="sync-data">Sync</button>
</div>
@@ -79,11 +98,10 @@
<input id="sync-force" type="checkbox" name="sync-force" value="force">
<label for="sync-force">Force Sync</label>
</div>
<div id="loading-bar" style="display: none;">
</div>
<div class="card-description-row">
<div id="sync-status"></div>
</div>
</div>
<div id="loading-bar" style="display: none;"></div>
<div class="card-description-row">
<div id="sync-status"></div>
</div>
</div>
</body>
@@ -93,7 +111,7 @@
body {
display: grid;
grid-template-columns: 1fr;
grid-template-rows: 1fr auto auto auto minmax(80px, 100%);
grid-template-rows: 1fr auto;
font-size: small!important;
}
body > * {
@@ -104,8 +122,7 @@
body {
display: grid;
grid-template-columns: 1fr min(70vw, 100%) 1fr;
grid-template-rows: 1fr auto auto auto minmax(80px, 100%);
padding-top: 60vw;
grid-template-rows: 80px auto;
}
body > * {
grid-column: 2;
@@ -126,11 +143,6 @@
margin: 10px;
}
div.page {
padding: 0px;
margin: 0px;
}
svg {
transition: transform 0.3s ease-in-out;
}
@@ -167,18 +179,18 @@
}
}
#khoj-host-url {
.card-input {
padding: 4px;
box-shadow: 0 0 2px 1px rgba(0, 0, 0, 0.2);
border: none;
width: 450px;
}
.card {
display: grid;
/* grid-template-rows: repeat(3, 1fr); */
gap: 8px;
padding: 24px 16px;
width: 100%;
width: 450px;
background: white;
border: 1px solid rgb(229, 229, 229);
border-radius: 4px;
@@ -188,15 +200,15 @@
.section-cards {
display: grid;
grid-template-columns: repeat(1, 1fr);
gap: 16px;
justify-items: start;
justify-items: center;
margin: 0;
width: auto;
}
div.configuration {
width: auto;
.section-action-row {
display: grid;
grid-auto-flow: column;
gap: 16px;
height: fit-content;
}
.card-title-row {
@@ -302,7 +314,6 @@
}
div.content-name {
width: 500px;
overflow-wrap: break-word;
}
@@ -347,6 +358,12 @@
background-color: #ffcc00;
box-shadow: 0px 3px 0px #f9f5de;
}
.sync-force-toggle {
align-content: center;
display: grid;
grid-auto-flow: column;
gap: 4px;
}
</style>
<script>
var khojBannerSubmit = document.getElementById("khoj-banner-submit");

View File

@@ -170,7 +170,10 @@
// Execute Search and Render Results
url = await createRequestUrl(query, type, results_count || 5, rerank);
fetch(url)
const khojToken = await window.tokenAPI.getToken();
const headers = { 'Authorization': `Bearer ${khojToken}` };
fetch(url, { headers })
.then(response => response.json())
.then(data => {
console.log(data);
@@ -192,9 +195,11 @@
async function populate_type_dropdown() {
const hostURL = await window.hostURLAPI.getURL();
const khojToken = await window.tokenAPI.getToken();
const headers = { 'Authorization': `Bearer ${khojToken}` };
// Populate type dropdown field with enabled content types only
fetch(`${hostURL}/api/config/types`)
fetch(`${hostURL}/api/config/types`, { headers })
.then(response => response.json())
.then(enabled_types => {
// Show warning if no content types are enabled
@@ -247,9 +252,9 @@
<img class="khoj-logo" src="./assets/icons/khoj-logo-sideways-500.png" alt="Khoj"></img>
</a>
<nav class="khoj-nav">
<a class="khoj-nav" href="./chat.html">Chat</a>
<a class="khoj-nav khoj-nav-selected" href="./index.html">Search</a>
<a class="khoj-nav" href="./config.html">⚙️</a>
<a class="khoj-nav" href="./chat.html">💬 Chat</a>
<a class="khoj-nav khoj-nav-selected" href="./index.html">🔎 Search</a>
<a class="khoj-nav" href="./config.html">⚙️ Settings</a>
</nav>
</div>

View File

@@ -1,4 +1,4 @@
const { app, BrowserWindow, ipcMain } = require('electron');
const { app, BrowserWindow, ipcMain, Tray, Menu, nativeImage } = require('electron');
const todesktop = require("@todesktop/runtime");
todesktop.init();
@@ -42,6 +42,10 @@ const schema = {
},
default: []
},
khojToken: {
type: 'string',
default: ''
},
hostURL: {
type: 'string',
default: KHOJ_URL
@@ -63,7 +67,6 @@ const schema = {
};
var state = {}
const store = new Store({ schema });
console.log(store);
@@ -168,7 +171,7 @@ function pushDataToKhoj (regenerate = false) {
if (!!formData?.entries()?.next().value) {
const hostURL = store.get('hostURL') || KHOJ_URL;
const headers = {
'x-api-key': 'secret'
'Authorization': `Bearer ${store.get("khojToken")}`
};
axios.post(`${hostURL}/api/v1/index/update?force=${regenerate}&client=desktop`, formData, { headers })
.then(response => {
@@ -246,6 +249,15 @@ async function handleFileOpen (type) {
}
}
async function getToken () {
return store.get('khojToken');
}
async function setToken (event, token) {
store.set('khojToken', token);
return store.get('khojToken');
}
async function getFiles () {
return store.get('files');
}
@@ -287,8 +299,9 @@ async function syncData (regenerate = false) {
}
}
const createWindow = () => {
const win = new BrowserWindow({
let win = null;
const createWindow = (tab = 'index.html') => {
win = new BrowserWindow({
width: 800,
height: 800,
// titleBarStyle: 'hidden',
@@ -316,7 +329,7 @@ const createWindow = () => {
job.start();
win.loadFile('index.html')
win.loadFile(tab)
}
app.whenReady().then(() => {
@@ -340,6 +353,9 @@ app.whenReady().then(() => {
ipcMain.handle('setURL', setURL);
ipcMain.handle('getURL', getURL);
ipcMain.handle('setToken', setToken);
ipcMain.handle('getToken', getToken);
ipcMain.handle('syncData', (event, regenerate) => {
syncData(regenerate);
});
@@ -375,3 +391,33 @@ app.whenReady().then(() => {
app.on('window-all-closed', () => {
if (process.platform !== 'darwin') app.quit()
})
/*
** System Tray Icon
*/
let tray
openWindow = (page) => {
if (BrowserWindow.getAllWindows().length === 0) {
createWindow(page);
} else {
win.loadFile(page); win.show();
}
}
app.whenReady().then(() => {
const icon = nativeImage.createFromPath('assets/icons/favicon-20x20.png')
tray = new Tray(icon)
const contextMenu = Menu.buildFromTemplate([
{ label: 'Chat', type: 'normal', click: () => { openWindow('chat.html'); }},
{ label: 'Search', type: 'normal', click: () => { openWindow('index.html') }},
{ label: 'Configure', type: 'normal', click: () => { openWindow('config.html') }},
{ type: 'separator' },
{ label: 'Quit', type: 'normal', click: () => { app.quit() } }
])
tray.setToolTip('Khoj')
tray.setContextMenu(contextMenu)
})

View File

@@ -47,3 +47,8 @@ contextBridge.exposeInMainWorld('hostURLAPI', {
contextBridge.exposeInMainWorld('syncDataAPI', {
syncData: (regenerate) => ipcRenderer.invoke('syncData', regenerate)
})
contextBridge.exposeInMainWorld('tokenAPI', {
setToken: (token) => ipcRenderer.invoke('setToken', token),
getToken: () => ipcRenderer.invoke('getToken')
})

View File

@@ -181,6 +181,17 @@ urlInput.addEventListener('blur', async () => {
urlInput.value = url;
});
const khojKeyInput = document.getElementById('khoj-access-key');
(async function() {
const token = await window.tokenAPI.getToken();
khojKeyInput.value = token;
})();
khojKeyInput.addEventListener('blur', async () => {
const token = await window.tokenAPI.setToken(khojKeyInput.value.trim());
khojKeyInput.value = token;
});
const syncButton = document.getElementById('sync-data');
const syncForceToggle = document.getElementById('sync-force');
syncButton.addEventListener('click', async () => {