klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-08 05:39:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update the configuration page to make config management easier

Browse Source 
- Add a central configuration management page to make management of config details easier
- Add relevant api endpoints both for client and server to update/request data as necessary
- Attempt to update the favicon
This commit is contained in:
sabaimran
2023-06-17 15:21:28 -07:00
parent c68cde4803
commit ded3100caf
21 changed files with 470 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/khoj/interface/desktop/main_window.py
View File

@@ -194,6 +194,8 @@ class MainWindow(QtWidgets.QMainWindow):
# Search Type (re)-Enabled # Search Type (re)-Enabled
if child.isChecked(): if child.isChecked():
current_search_config = self.current_config["content-type"].get(child.search_type, {}) current_search_config = self.current_config["content-type"].get(child.search_type, {})
if current_search_config == None:
current_search_config = {}
default_search_config = self.get_default_config(search_type=child.search_type) default_search_config = self.get_default_config(search_type=child.search_type)
self.new_config["content-type"][child.search_type.value] = merge_dicts( self.new_config["content-type"][child.search_type.value] = merge_dicts(
current_search_config, default_search_config current_search_config, default_search_config

22
src/khoj/interface/web/404.html Normal file
View File

@@ -0,0 +1,22 @@
<!DOCTYPE html>
<html>
<head>
<title>Khoj: An AI Personal Assistant for your digital brain</title>
<link rel=”stylesheet” href=”static/styles.css”>
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@picocss/pico@1/css/pico.min.css">
</head>
<body class="not-found">
<header class=”header”>
<h1>Oops, this is awkward. That page couldn't be found.</h1>
</header>
<a href="/config">Go Home</a>
<footer class=”footer”>
</footer>
</body>
<style>
body.not-found {
padding: 0 10%
}
</style>
</html>

29
src/khoj/interface/web/assets/config.css
View File

@@ -1,29 +0,0 @@
:root {
--primary-color: #ffffff;
--bold-color: #2073ee;
--complementary-color: #124408;
--accent-color-0: #57f0b5;
}
input[type=text] {
width: 40%;
}
div.config-element {
color: var(--bold-color);
margin: 8px;
}
div.config-title {
font-weight: bold;
}
span.config-element-value {
color: var(--complementary-color);
font-weight: normal;
cursor: pointer;
}
button {
cursor: pointer;
}

125
src/khoj/interface/web/assets/config.js
View File

@@ -1,125 +0,0 @@
// Retrieve elements from the DOM.
var showConfig = document.getElementById("show-config");
var configForm = document.getElementById("config-form");
var regenerateButton = document.getElementById("config-regenerate");
// Global variables.
var rawConfig = {};
var emptyValueDefault = "🖊️";
/**
* Fetch the existing config file.
*/
fetch("/api/config/data")
.then(response => response.json())
.then(data => {
rawConfig = data;
configForm.style.display = "block";
processChildren(configForm, data);
var submitButton = document.createElement("button");
submitButton.type = "submit";
submitButton.innerHTML = "update";
configForm.appendChild(submitButton);
// The config form's submit handler.
configForm.addEventListener("submit", (event) => {
event.preventDefault();
console.log(rawConfig);
fetch("/api/config/data", {
method: "POST",
credentials: "same-origin",
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(rawConfig)
})
.then(response => response.json())
.then(data => console.log(data));
});
});
/**
* The click handler for the Regenerate button.
*/
regenerateButton.addEventListener("click", (event) => {
event.preventDefault();
regenerateButton.style.cursor = "progress";
regenerateButton.disabled = true;
fetch("/api/update?force=true&client=web")
.then(response => response.json())
.then(data => {
regenerateButton.style.cursor = "pointer";
regenerateButton.disabled = false;
console.log(data);
});
})
/**
* Adds config elements to the DOM representing the sub-components
* of one of the fields in the raw config file.
* @param {the parent element} element
* @param {the data to be rendered for this element and its children} data
*/
function processChildren(element, data) {
for (let key in data) {
var child = document.createElement("div");
child.id = key;
child.className = "config-element";
child.appendChild(document.createTextNode(key + ": "));
if (data[key] === Object(data[key]) && !Array.isArray(data[key])) {
child.className+=" config-title";
processChildren(child, data[key]);
} else {
child.appendChild(createValueNode(data, key));
}
element.appendChild(child);
}
}
/**
* Takes an element, and replaces it with an editable
* element with the same data in place.
* @param {the original element to be replaced} original
* @param {the source data to be rendered for the new element} data
* @param {the key for this input in the source data} key
*/
function makeElementEditable(original, data, key) {
original.addEventListener("click", () => {
var inputNewText = document.createElement("input");
inputNewText.type = "text";
inputNewText.className = "config-element-edit";
inputNewText.value = (original.textContent == emptyValueDefault) ? "" : original.textContent;
fixInputOnFocusOut(inputNewText, data, key);
original.parentNode.replaceChild(inputNewText, original);
inputNewText.focus();
});
}
/**
* Creates a node corresponding to the value of a config element.
* @param {the source data} data
* @param {the key corresponding to this node's data} key
* @returns A new element which corresponds to the value in some field.
*/
function createValueNode(data, key) {
var valueElement = document.createElement("span");
valueElement.className = "config-element-value";
valueElement.textContent = !data[key] ? emptyValueDefault : data[key];
makeElementEditable(valueElement, data, key);
return valueElement;
}
/**
* Replaces an existing input element with an element with the same data, which is not an input.
* If the input data for this element was changed, update the corresponding data in the raw config.
* @param {the original element to be replaced} original
* @param {the source data} data
* @param {the key corresponding to this node's data} key
*/
function fixInputOnFocusOut(original, data, key) {
original.addEventListener("blur", () => {
data[key] = (original.value != emptyValueDefault) ? original.value : "";
original.parentNode.replaceChild(createValueNode(data, key), original);
})
}

BIN
src/khoj/interface/web/assets/icons/favicon-128x128.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 26 KiB

BIN
src/khoj/interface/web/assets/icons/favicon-144x144.ico
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 159 KiB

After

Width:  |  Height:  |  Size: 200 KiB

BIN
src/khoj/interface/web/assets/icons/favicon-144x144.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 29 KiB

After

Width:  |  Height:  |  Size: 12 KiB

BIN
src/khoj/interface/web/assets/icons/favicon.icns
View File

Binary file not shown.

35
src/khoj/interface/web/base_config.html Normal file
View File

@@ -0,0 +1,35 @@
<!DOCTYPE html>
<head>
<meta charset="utf-8">
<link rel="icon" href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 100 100%22><text y=%22.9em%22 font-size=%2290%22>🦅</text></svg>">
<title>Khoj - Configure App</title>
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@picocss/pico@1/css/pico.min.css">
</head>
<body class="khoj-configure">
<header>
<h1>Configure Khoj</h1>
<div>
<h2>Ready?</h2>
<div id="actions">
<button onclick="window.location.href='/';" >
Search
</button>
<button onclick="window.location.href='/chat';">
Chat
</button>
</div>
</div>
</header>
<div class=”content”>
{% block content %}
{% endblock %}
</div>
</body>
<style>
header {
display: grid;
grid-template-columns: 1fr 1fr;
grid-gap: 12px;
}
</style>
</html>

27
src/khoj/interface/web/base_data_integration.html Normal file
View File

@@ -0,0 +1,27 @@
<!DOCTYPE html>
<html>
<head>
<title>Khoj: Configuration for data integrations</title>
<link rel=”stylesheet” href=”static/styles.css”>
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@picocss/pico@1/css/pico.min.css">
</head>
<body class="data-integration">
<header class=”header”>
<h1>Configure your data integrations for Khoj</h1>
</header>
<a href="/config">Go back</a>
<div class=”content”>
{% block content %}
{% endblock %}
</div>
<footer class=”footer”>
</footer>
</body>
<style>
body.data-integration {
padding: 0 10%
}
</style>
</html>

27
src/khoj/interface/web/base_processor_integration.html Normal file
View File

@@ -0,0 +1,27 @@
<!DOCTYPE html>
<html>
<head>
<title>Khoj: Configuration for processor integrations</title>
<link rel=”stylesheet” href=”static/styles.css”>
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@picocss/pico@1/css/pico.min.css">
</head>
<body class="data-integration">
<header class=”header”>
<h1>Configure your processor integrations for Khoj</h1>
</header>
<a href="/config">Go back</a>
<div class=”content”>
{% block content %}
{% endblock %}
</div>
<footer class=”footer”>
</footer>
</body>
<style>
body.data-integration {
padding: 0 10%
}
</style>
</html>

46
src/khoj/interface/web/config.html
View File

@@ -1,14 +1,32 @@
<!DOCTYPE html> {% extends "base_config.html" %}
<head> {% block content %}
<meta charset="utf-8"> <h2>Content Types</h2>
<link rel="icon" href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 100 100%22><text y=%22.9em%22 font-size=%2290%22>🦅</text></svg>"> <div id="content-configuration">
<link rel="stylesheet" href="static/assets/config.css"> <button onclick="window.location.href='/config/content_type/pdf';">
<title>Khoj - Configure App</title> Configure PDF
</head> </button>
<body> <button onclick="window.location.href='/config/content_type/markdown';">
<form id="config-form"> Configure Markdown
</form> </button>
<button id="config-regenerate">regenerate</button> <button onclick="window.location.href='/config/content_type/org';">
</body> Configure Org
<script src="static/assets/config.js"></script> </button>
</html> <button onclick="window.location.href='/config/content_type/ledger';">
Configure Ledger
</button>
</div>
<h2>Processors</h2>
<button onclick="window.location.href='/config/processor/conversation/';">
Configure Conversation
</button>
<style>
body.khoj-configure {
padding: 0 10%
}
div#content-configuration, div#actions {
display: grid;
grid-template-columns: 1fr 1fr;
grid-gap: 12px;
}
</style>
{% endblock %}

151
src/khoj/interface/web/content_type_input.html Normal file
View File

@@ -0,0 +1,151 @@
{% extends "base_data_integration.html" %}
{% block content %}
<h2>{{ content_type }}</h2>
<form id="config-form">
<table>
<tr>
<th>Field</th>
<th>Value</th>
</tr>
<tr>
<td>
<label for="input-files">Input Files</label>
</td>
<td id="input-files-cell">
{% if current_config['input_files'] is none %}
<input type="text" id="input-files" name="input-files">
{% else %}
{% for input_file in current_config['input_files'] %}
<input type="text" id="input-files" name="input-files" value="{{ input_file }}">
{% endfor %}
{% endif %}
</td>
<td>
<button type="button" id="input-files-button">Add</button>
</td>
</tr>
<tr>
<td>
<label for="input-filter">Input Filter</label>
</td>
<td id="input-filter-cell">
{% if current_config['input_filter'] is none %}
<input type="text" id="input-filter" name="input-filter">
{% else %}
{% for input_filter in current_config['input_filter'] %}
<input type="text" id="input-filter" name="input-filter" value="{{ input_filter }}">
{% endfor %}
{% endif %}
</td>
<td>
<button type="button" id="input-filter-button">Add</button>
</td>
</tr>
</table>
<h4>You probably don't need to edit these.</h4>
<table>
<tr>
<th>Field</th>
<th>Value</th>
</tr>
<tr>
<td>
<label for="compressed-jsonl">Compressed JSONL (Output)</label>
</td>
<td>
<input type="text" id="compressed-jsonl" name="compressed-jsonl" value="{{ current_config['compressed_jsonl'] }}">
</td>
</tr>
<tr>
<td>
<label for="embeddings-file">Embeddings File (Output)</label>
</td>
<td>
<input type="text" id="embeddings-file" name="embeddings-file" value="{{ current_config['embeddings_file'] }}">
</td>
</tr>
<tr>
<td>
<label for="index-heading-entries">Index Heading Entries</label>
</td>
<td>
<input type="text" id="index-heading-entries" name="index-heading-entries" value="{{ current_config['index_heading_entries'] }}">
</td>
</tr>
</table>
<button id="submit" type="submit">Submit</button>
</form>
<script>
function addButtonEventListener(fieldName) {
var button = document.getElementById(fieldName + "-button");
button.addEventListener("click", function(event) {
var cell = document.getElementById(fieldName + "-cell");
var newInput = document.createElement("input");
newInput.setAttribute("type", "text");
newInput.setAttribute("name", fieldName);
cell.appendChild(newInput);
})
}
addButtonEventListener("input-files");
addButtonEventListener("input-filter");
function getValidInputNodes(nodes) {
var validNodes = [];
for (var i = 0; i < nodes.length; i++) {
const nodeValue = nodes[i].value;
if (nodeValue === "" || nodeValue === null || nodeValue === undefined || nodeValue === "None") {
continue;
}
validNodes.push(nodes[i]);
}
return validNodes;
}
submit.addEventListener("click", function(event) {
event.preventDefault();
var inputFileNodes = document.getElementsByName("input-files");
var input_files = getValidInputNodes(inputFileNodes).map(node => node.value);
var inputFilterNodes = document.getElementsByName("input-filter");
var input_filter = getValidInputNodes(inputFilterNodes).map(node => node.value);
if (input_files.length === 0 && input_filter.length === 0) {
alert("You must specify at least one input file or input filter.");
return;
}
if (input_files.length == 0) {
input_files = null;
}
if (input_filter.length == 0) {
input_filter = null;
}
var compressed_jsonl = document.getElementById("compressed-jsonl").value;
var embeddings_file = document.getElementById("embeddings-file").value;
var index_heading_entries = document.getElementById("index-heading-entries").value;
fetch('/api/config/data/content_type/{{ content_type }}', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
"input_files": input_files,
"input_filter": input_filter,
"compressed_jsonl": compressed_jsonl,
"embeddings_file": embeddings_file,
"index_heading_entries": index_heading_entries
})
})
.then(response => response.json())
.then(data => {
console.log('Success:', data);
})
});
</script>
{% endblock %}

2
src/khoj/interface/web/khoj.webmanifest
View File

@@ -5,7 +5,7 @@
"icons": [ "icons": [
{ {
"src": "/static/assets/icons/favicon-144x144.png", "src": "/static/assets/icons/favicon-144x144.png",
"sizes": "144x144", "sizes": "128x128",
"type": "image/png" "type": "image/png"
} }
], ],

2
src/khoj/interface/web/khoj_chat.webmanifest
View File

@@ -5,7 +5,7 @@
"icons": [ "icons": [
{ {
"src": "/static/assets/icons/favicon-144x144.png", "src": "/static/assets/icons/favicon-144x144.png",
"sizes": "144x144", "sizes": "128x128",
"type": "image/png" "type": "image/png"
} }
], ],

72
src/khoj/interface/web/processor_conversation_input.html Normal file
View File

@@ -0,0 +1,72 @@
{% extends "base_processor_integration.html" %}
{% block content %}
<h2>Conversation</h2>
<form id="config-form">
<table>
<tr>
<td>
<label for="openai-api-key">Open AI API key</label>
</td>
<td>
<input type="text" id="openai-api-key" name="openai-api-key" value="{{ current_config['openai_api_key'] }}">
</td>
</tr>
</table>
<h4>You probably don't need to edit these.</h4>
<table>
<tr>
<td>
<label for="conversation-logfile">Conversation Logfile</label>
</td>
<td>
<input type="text" id="conversation-logfile" name="conversation-logfile" value="{{ current_config['conversation_logfile'] }}">
</td>
</tr>
<tr>
<td>
<label for="model">Model</label>
</td>
<td>
<input type="text" id="model" name="model" value="{{ current_config['model'] }}">
</td>
</tr>
<tr>
<td>
<label for="chat-model">Chat Model</label>
</td>
<td>
<input type="text" id="chat-model" name="chat-model" value="{{ current_config['chat_model'] }}">
</td>
</tr>
</table>
<button id="submit" type="submit">Submit</button>
</form>
<script>
submit.addEventListener("click", function(event) {
event.preventDefault();
var openai_api_key = document.getElementById("openai-api-key").value;
var conversation_logfile = document.getElementById("conversation-logfile").value;
var model = document.getElementById("model").value;
var chat_model = document.getElementById("chat-model").value;
fetch('/api/config/data/processor/conversation', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
"openai_api_key": openai_api_key,
"conversation_logfile": conversation_logfile,
"model": model,
"chat_model": chat_model
})
})
.then(response => response.json())
.then(data => {
console.log('Success:', data);
})
});
</script>
{% endblock %}

15
src/khoj/routers/api.py
View File

@@ -15,9 +15,10 @@ from khoj.processor.conversation.gpt import converse, extract_questions
from khoj.processor.conversation.utils import message_to_log, message_to_prompt from khoj.processor.conversation.utils import message_to_log, message_to_prompt
from khoj.search_type import image_search, text_search from khoj.search_type import image_search, text_search
from khoj.utils.helpers import log_telemetry, timer from khoj.utils.helpers import log_telemetry, timer
from khoj.utils.rawconfig import FullConfig, SearchResponse from khoj.utils.rawconfig import FullConfig, SearchResponse, TextContentConfig, ConversationProcessorConfig
from khoj.utils.state import SearchType from khoj.utils.state import SearchType
from khoj.utils import state, constants from khoj.utils import state, constants
from khoj.utils.yaml import save_config_to_file_updated_state
# Initialize Router # Initialize Router
api = APIRouter() api = APIRouter()
@@ -62,6 +63,18 @@ async def set_config_data(updated_config: FullConfig):
return state.config return state.config
@api.post("/config/data/content_type/{content_type}")
async def set_content_config_data(content_type: str, updated_config: TextContentConfig):
state.config.content_type[content_type] = updated_config
save_config_to_file_updated_state()
@api.post("/config/data/processor/conversation")
async def set_processor_conversation_config_data(updated_config: ConversationProcessorConfig):
state.config.processor.conversation = updated_config
save_config_to_file_updated_state()
@api.get("/search", response_model=List[SearchResponse]) @api.get("/search", response_model=List[SearchResponse])
def search( def search(
q: str, q: str,

71
src/khoj/routers/web_client.py
View File

@@ -3,15 +3,23 @@ from fastapi import APIRouter
from fastapi import Request from fastapi import Request
from fastapi.responses import HTMLResponse, FileResponse from fastapi.responses import HTMLResponse, FileResponse
from fastapi.templating import Jinja2Templates from fastapi.templating import Jinja2Templates
from khoj.utils.rawconfig import TextContentConfig, ConversationProcessorConfig
# Internal Packages # Internal Packages
from khoj.utils import constants from khoj.utils import constants, state
import logging
import json
logger = logging.getLogger("khoj")
# Initialize Router # Initialize Router
web_client = APIRouter() web_client = APIRouter()
templates = Jinja2Templates(directory=constants.web_directory) templates = Jinja2Templates(directory=constants.web_directory)
VALID_CONTENT_TYPES = ["org", "ledger", "image", "music", "markdown", "pdf"]
# Create Routes # Create Routes
@web_client.get("/", response_class=FileResponse) @web_client.get("/", response_class=FileResponse)
@@ -24,6 +32,67 @@ def config_page(request: Request):
return templates.TemplateResponse("config.html", context={"request": request}) return templates.TemplateResponse("config.html", context={"request": request})
@web_client.get("/test-child", response_class=HTMLResponse)
def test_child(request: Request):
return templates.TemplateResponse("child0.html", context={"request": request, "config": constants.default_config})
@web_client.get("/config/content_type/{content_type}", response_class=HTMLResponse)
def content_config_page(request: Request, content_type: str):
if content_type not in VALID_CONTENT_TYPES:
return templates.TemplateResponse("config.html", context={"request": request})
default_copy = constants.default_config.copy()
default_content_type = default_copy["content-type"][content_type]
default_config = TextContentConfig(
compressed_jsonl=default_content_type["compressed-jsonl"],
embeddings_file=default_content_type["embeddings-file"],
)
current_config = (
state.config.content_type[content_type]
if state.config.content_type[content_type] is not None
else default_config
)
current_config = json.loads(current_config.json())
return templates.TemplateResponse(
"content_type_input.html",
context={
"request": request,
"current_config": current_config,
"content_type": content_type,
},
)
@web_client.get("/config/processor/conversation", response_class=HTMLResponse)
def conversation_processor_config_page(request: Request):
default_copy = constants.default_config.copy()
default_processor_config = default_copy["processor"]["conversation"]
default_processor_config = ConversationProcessorConfig(
openai_api_key="",
model=default_processor_config["model"],
conversation_logfile=default_processor_config["conversation-logfile"],
chat_model=default_processor_config["chat-model"],
)
current_processor_conversation_config = (
state.config.processor.conversation
if state.config.processor.conversation is not None
else default_processor_config
)
current_processor_conversation_config = json.loads(current_processor_conversation_config.json())
return templates.TemplateResponse(
"processor_conversation_input.html",
context={
"request": request,
"current_config": current_processor_conversation_config,
},
)
@web_client.get("/chat", response_class=FileResponse) @web_client.get("/chat", response_class=FileResponse)
def chat_page(): def chat_page():
return FileResponse(constants.web_directory / "chat.html") return FileResponse(constants.web_directory / "chat.html")

3
src/khoj/utils/constants.py
View File

@@ -14,7 +14,7 @@ default_config = {
"input-filter": None, "input-filter": None,
"compressed-jsonl": "~/.khoj/content/org/org.jsonl.gz", "compressed-jsonl": "~/.khoj/content/org/org.jsonl.gz",
"embeddings-file": "~/.khoj/content/org/org_embeddings.pt", "embeddings-file": "~/.khoj/content/org/org_embeddings.pt",
"index_heading_entries": False, "index-heading-entries": False,
}, },
"markdown": { "markdown": {
"input-files": None, "input-files": None,
@@ -66,6 +66,7 @@ default_config = {
"openai-api-key": None, "openai-api-key": None,
"model": "text-davinci-003", "model": "text-davinci-003",
"conversation-logfile": "~/.khoj/processor/conversation/conversation_logs.json", "conversation-logfile": "~/.khoj/processor/conversation/conversation_logs.json",
"chat-model": "gpt-3.5-turbo",
} }
}, },
} }

6
src/khoj/utils/rawconfig.py
View File

@@ -15,6 +15,12 @@ class ConfigBase(BaseModel):
alias_generator = to_snake_case_from_dash alias_generator = to_snake_case_from_dash
allow_population_by_field_name = True allow_population_by_field_name = True
def __getitem__(self, item):
return getattr(self, item)
def __setitem__(self, key, value):
return setattr(self, key, value)
class TextContentConfig(ConfigBase): class TextContentConfig(ConfigBase):
input_files: Optional[List[Path]] input_files: Optional[List[Path]]

8
src/khoj/utils/yaml.py
View File

@@ -6,12 +6,20 @@ import yaml
# Internal Packages # Internal Packages
from khoj.utils.rawconfig import FullConfig from khoj.utils.rawconfig import FullConfig
from khoj.utils import state
# Do not emit tags when dumping to YAML # Do not emit tags when dumping to YAML
yaml.emitter.Emitter.process_tag = lambda self, *args, **kwargs: None # type: ignore[assignment] yaml.emitter.Emitter.process_tag = lambda self, *args, **kwargs: None # type: ignore[assignment]
def save_config_to_file_updated_state():
with open(state.config_file, "w") as outfile:
yaml.dump(yaml.safe_load(state.config.json(by_alias=True)), outfile)
outfile.close()
return state.config
def save_config_to_file(yaml_config: dict, yaml_config_file: Path): def save_config_to_file(yaml_config: dict, yaml_config_file: Path):
"Write config to YML file" "Write config to YML file"
# Create output directory, if it doesn't exist # Create output directory, if it doesn't exist