Use the src/ layout to fix packaging Khoj for PyPi

- Why
  The khoj pypi packages should be installed in `khoj' directory.
  Previously it was being installed into `src' directory, which is a
  generic top level directory name that is discouraged from being used

- Changes
 - move src/* to src/khoj/*
 - update `setup.py' to `find_packages' in `src' instead of project root
 - rename imports to form `from khoj.*' in complete project
 - update `constants.web_directory' path to use `khoj' directory
 - rename root logger to `khoj' in `main.py'
 - fix image_search tests to use the newly rename `khoj' logger
 - update config, docs, workflows to reference new path `src/khoj'
This commit is contained in:
Debanjum Singh Solanky
2023-02-14 14:50:51 -06:00
parent cc31cd070d
commit 25a749ca1d
104 changed files with 209 additions and 208 deletions

View File

@@ -1,29 +0,0 @@
:root {
--primary-color: #ffffff;
--bold-color: #2073ee;
--complementary-color: #124408;
--accent-color-0: #57f0b5;
}
input[type=text] {
width: 40%;
}
div.config-element {
color: var(--bold-color);
margin: 8px;
}
div.config-title {
font-weight: bold;
}
span.config-element-value {
color: var(--complementary-color);
font-weight: normal;
cursor: pointer;
}
button {
cursor: pointer;
}

View File

@@ -1,125 +0,0 @@
// Retrieve elements from the DOM.
var showConfig = document.getElementById("show-config");
var configForm = document.getElementById("config-form");
var regenerateButton = document.getElementById("config-regenerate");
// Global variables.
var rawConfig = {};
var emptyValueDefault = "🖊️";
/**
* Fetch the existing config file.
*/
fetch("/api/config/data")
.then(response => response.json())
.then(data => {
rawConfig = data;
configForm.style.display = "block";
processChildren(configForm, data);
var submitButton = document.createElement("button");
submitButton.type = "submit";
submitButton.innerHTML = "update";
configForm.appendChild(submitButton);
// The config form's submit handler.
configForm.addEventListener("submit", (event) => {
event.preventDefault();
console.log(rawConfig);
fetch("/api/config/data", {
method: "POST",
credentials: "same-origin",
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(rawConfig)
})
.then(response => response.json())
.then(data => console.log(data));
});
});
/**
* The click handler for the Regenerate button.
*/
regenerateButton.addEventListener("click", (event) => {
event.preventDefault();
regenerateButton.style.cursor = "progress";
regenerateButton.disabled = true;
fetch("/api/update?force=true")
.then(response => response.json())
.then(data => {
regenerateButton.style.cursor = "pointer";
regenerateButton.disabled = false;
console.log(data);
});
})
/**
* Adds config elements to the DOM representing the sub-components
* of one of the fields in the raw config file.
* @param {the parent element} element
* @param {the data to be rendered for this element and its children} data
*/
function processChildren(element, data) {
for (let key in data) {
var child = document.createElement("div");
child.id = key;
child.className = "config-element";
child.appendChild(document.createTextNode(key + ": "));
if (data[key] === Object(data[key]) && !Array.isArray(data[key])) {
child.className+=" config-title";
processChildren(child, data[key]);
} else {
child.appendChild(createValueNode(data, key));
}
element.appendChild(child);
}
}
/**
* Takes an element, and replaces it with an editable
* element with the same data in place.
* @param {the original element to be replaced} original
* @param {the source data to be rendered for the new element} data
* @param {the key for this input in the source data} key
*/
function makeElementEditable(original, data, key) {
original.addEventListener("click", () => {
var inputNewText = document.createElement("input");
inputNewText.type = "text";
inputNewText.className = "config-element-edit";
inputNewText.value = (original.textContent == emptyValueDefault) ? "" : original.textContent;
fixInputOnFocusOut(inputNewText, data, key);
original.parentNode.replaceChild(inputNewText, original);
inputNewText.focus();
});
}
/**
* Creates a node corresponding to the value of a config element.
* @param {the source data} data
* @param {the key corresponding to this node's data} key
* @returns A new element which corresponds to the value in some field.
*/
function createValueNode(data, key) {
var valueElement = document.createElement("span");
valueElement.className = "config-element-value";
valueElement.textContent = !data[key] ? emptyValueDefault : data[key];
makeElementEditable(valueElement, data, key);
return valueElement;
}
/**
* Replaces an existing input element with an element with the same data, which is not an input.
* If the input data for this element was changed, update the corresponding data in the raw config.
* @param {the original element to be replaced} original
* @param {the source data} data
* @param {the key corresponding to this node's data} key
*/
function fixInputOnFocusOut(original, data, key) {
original.addEventListener("blur", () => {
data[key] = (original.value != emptyValueDefault) ? original.value : "";
original.parentNode.replaceChild(createValueNode(data, key), original);
})
}

Binary file not shown.

Before

Width:  |  Height:  |  Size: 26 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 159 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 29 KiB

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,261 +0,0 @@
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0 maximum-scale=1.0">
<title>Khoj</title>
<link rel="icon" href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 144 144%22><text y=%22.86em%22 font-size=%22144%22>🦅</text></svg>">
<link rel="icon" type="image/png" sizes="144x144" href="/static/assets/icons/favicon-144x144.png">
<link rel="manifest" href="/static/khoj.webmanifest">
</head>
<script>
function setTypeFieldInUrl(type) {
let url = new URL(window.location.href);
url.searchParams.set("t", type.value);
window.history.pushState({}, "", url.href);
}
function formatDate(date) {
// Format date in HH:MM, DD MMM YYYY format
let time_string = date.toLocaleTimeString('en-IN', { hour: '2-digit', minute: '2-digit', hour12: false });
let date_string = date.toLocaleString('en-IN', { year: 'numeric', month: 'short', day: '2-digit'}).replaceAll('-', ' ');
return `${time_string}, ${date_string}`;
}
function renderMessage(message, by, dt=null) {
let message_time = formatDate(dt ?? new Date());
let by_name = by == "khoj" ? "🦅 Khoj" : "🤔 You";
// Generate HTML for Chat Message and Append to Chat Body
document.getElementById("chat-body").innerHTML += `
<div data-meta="${by_name} at ${message_time}" class="chat-message ${by}">
<div class="chat-message-text ${by}">${message}</div>
</div>
`;
// Scroll to bottom of input-body element
document.getElementById("chat-body").scrollTop = document.getElementById("chat-body").scrollHeight;
}
function chat() {
// Extract required fields for search from form
query = document.getElementById("chat-input").value.trim();
type_ = document.getElementById("chat-type").value;
console.log(`Query: ${query}, Type: ${type_}`);
// Short circuit on empty query
if (query.length === 0)
return;
// Add message by user to chat body
renderMessage(query, "you");
document.getElementById("chat-input").value = "";
// Generate backend API URL to execute query
url = type_ === "chat"
? `/api/beta/chat?q=${encodeURIComponent(query)}`
: `/api/beta/summarize?q=${encodeURIComponent(query)}`;
// Call specified Khoj API
fetch(url)
.then(response => response.json())
.then(data => data.response)
.then(response => {
// Render message by Khoj to chat body
console.log(response);
renderMessage(response, "khoj");
});
}
function incrementalChat(event) {
// Send chat message on 'Enter'
if (event.key === 'Enter') {
chat();
}
}
window.onload = function () {
// Fill type field with value passed in URL query parameters, if any.
var type_via_url = new URLSearchParams(window.location.search).get("t");
if (type_via_url)
document.getElementById("chat-type").value = type_via_url;
fetch('/api/beta/chat')
.then(response => response.json())
.then(data => data.response)
.then(chat_logs => {
// Render conversation history, if any
chat_logs.forEach(chat_log => {
renderMessage(chat_log.message, chat_log.by, new Date(chat_log.created));
});
});
// Set welcome message on load
renderMessage("Hey, what's up?", "khoj");
// Fill query field with value passed in URL query parameters, if any.
var query_via_url = new URLSearchParams(window.location.search).get("q");
if (query_via_url) {
document.getElementById("chat-input").value = query_via_url;
chat();
}
}
</script>
<body>
<!-- Chat Header -->
<h1>Khoj</h1>
<!-- Chat Body -->
<div id="chat-body"></div>
<!-- Chat Footer -->
<div id="chat-footer">
<input type="text" id="chat-input" class="option" onkeyup=incrementalChat(event) autofocus="autofocus" placeholder="What is the meaning of life?">
<!--Select Chat Type from: Chat, Summarize -->
<select id="chat-type" class="option" onchange="setTypeFieldInUrl(this)">
<option value="chat">Chat</option>
<option value="summarize">Summarize</option>
</select>
</div>
</body>
<style>
html, body {
height: 100%;
width: 100%;
padding: 0px;
margin: 0px;
}
body {
display: grid;
background: #f8fafc;
color: #475569;
text-align: center;
font-family: roboto, karma, segoe ui, sans-serif;
font-size: 20px;
font-weight: 300;
line-height: 1.5em;
}
body > * {
padding: 10px;
margin: 10px;
}
h1 {
font-weight: 200;
color: #017eff;
}
#chat-body {
font-size: medium;
margin: 0px;
line-height: 20px;
overflow-y: scroll; /* Make chat body scroll to see history */
}
/* add chat metatdata to bottom of bubble */
.chat-message::after {
content: attr(data-meta);
display: block;
font-size: x-small;
color: #475569;
margin: -12px 7px 0 -5px;
}
/* move message by khoj to left */
.chat-message.khoj {
margin-left: auto;
text-align: left;
}
/* move message by you to right */
.chat-message.you {
margin-right: auto;
text-align: right;
}
/* basic style chat message text */
.chat-message-text {
margin: 10px;
border-radius: 10px;
padding: 10px;
position: relative;
display: inline-block;
max-width: 80%;
text-align: left;
}
/* color chat bubble by khoj blue */
.chat-message-text.khoj {
color: #f8fafc;
background: #017eff;
margin-left: auto;
}
/* add left protrusion to khoj chat bubble */
.chat-message-text.khoj:after {
content: '';
position: absolute;
bottom: -2px;
left: -7px;
border: 10px solid transparent;
border-top-color: #017eff;
border-bottom: 0;
transform: rotate(-60deg);
}
/* color chat bubble by you dark grey */
.chat-message-text.you {
color: #f8fafc;
background: #475569;
margin-right: auto;
}
/* add right protrusion to you chat bubble */
.chat-message-text.you:after {
content: '';
position: absolute;
top: 91%;
right: -2px;
border: 10px solid transparent;
border-left-color: #475569;
border-right: 0;
margin-top: -10px;
transform: rotate(-60deg)
}
#chat-footer {
padding: 0;
display: grid;
grid-template-columns: minmax(70px, 85%) auto;
grid-column-gap: 10px;
grid-row-gap: 10px;
}
#chat-footer > * {
padding: 15px;
border-radius: 5px;
border: 1px solid #475569;
background: #f9fafc
}
.option:hover {
box-shadow: 0 0 11px #aaa;
}
#chat-input {
font-size: medium;
}
@media only screen and (max-width: 600px) {
body {
grid-template-columns: 1fr;
grid-template-rows: auto minmax(80px, 100%) auto;
}
body > * {
grid-column: 1;
}
#chat-footer {
padding: 0;
margin: 4px;
grid-template-columns: auto;
}
}
@media only screen and (min-width: 600px) {
body {
grid-template-columns: auto min(70vw, 100%) auto;
grid-template-rows: auto minmax(80px, 100%) auto;
}
body > * {
grid-column: 2;
}
}
</style>
</html>

View File

@@ -1,14 +0,0 @@
<!DOCTYPE html>
<head>
<meta charset="utf-8">
<link rel="icon" href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 100 100%22><text y=%22.9em%22 font-size=%2290%22>🦅</text></svg>">
<link rel="stylesheet" href="static/assets/config.css">
<title>Khoj - Configure App</title>
</head>
<body>
<form id="config-form">
</form>
<button id="config-regenerate">regenerate</button>
</body>
<script src="static/assets/config.js"></script>
</html>

View File

@@ -1,326 +0,0 @@
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0 maximum-scale=1.0">
<title>Khoj</title>
<link rel="icon" href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 144 144%22><text y=%22.86em%22 font-size=%22144%22>🦅</text></svg>">
<link rel="icon" type="image/png" sizes="144x144" href="/static/assets/icons/favicon-144x144.png">
<link rel="manifest" href="/static/khoj.webmanifest">
</head>
<script type="text/javascript" src="static/assets/org.min.js"></script>
<script type="text/javascript" src="static/assets/markdown-it.min.js"></script>
<script>
function render_image(item) {
return `
<a href="${item.entry}" class="image-link">
<img id=${item.score} src="${item.entry}?${Math.random()}"
title="Effective Score: ${item.score}, Meta: ${item.additional.metadata_score}, Image: ${item.additional.image_score}"
class="image">
</a>`
}
function render_org(query, data, classPrefix="") {
var orgCode = data.map(function (item) {
return `${item.entry}`
}).join("\n")
var orgParser = new Org.Parser();
var orgDocument = orgParser.parse(orgCode);
var orgHTMLDocument = orgDocument.convert(Org.ConverterHTML, { htmlClassPrefix: classPrefix });
return orgHTMLDocument.toString();
}
function render_markdown(query, data) {
var md = window.markdownit();
return md.render(data.map(function (item) {
return `${item.entry}`
}).join("\n"));
}
function render_ledger(query, data) {
return `<div id="results-ledger">` + data.map(function (item) {
return `<p>${item.entry}</p>`
}).join("\n") + `</div>`;
}
function render_json(data, query, type) {
if (type === "markdown") {
return render_markdown(query, data);
} else if (type === "org") {
return render_org(query, data);
} else if (type === "music") {
return render_org(query, data, "music-");
} else if (type === "image") {
return data.map(render_image).join('');
} else if (type === "ledger") {
return render_ledger(query, data);
} else {
return `<pre id="json">${JSON.stringify(data, null, 2)}</pre>`;
}
}
function search(rerank=false) {
// Extract required fields for search from form
query = document.getElementById("query").value.trim();
type = document.getElementById("type").value;
results_count = document.getElementById("results-count").value || 6;
console.log(`Query: ${query}, Type: ${type}`);
// Short circuit on empty query
if (query.length === 0)
return;
// If set query field in url query param on rerank
if (rerank)
setQueryFieldInUrl(query);
// Generate Backend API URL to execute Search
url = type === "image"
? `/api/search?q=${encodeURIComponent(query)}&t=${type}&n=${results_count}`
: `/api/search?q=${encodeURIComponent(query)}&t=${type}&n=${results_count}&r=${rerank}`;
// Execute Search and Render Results
fetch(url)
.then(response => response.json())
.then(data => {
console.log(data);
document.getElementById("results").innerHTML =
`<div id=results-${type}>`
+ render_json(data, query, type)
+ `</div>`;
});
}
function updateIndex() {
type = document.getElementById("type").value;
fetch(`/api/update?t=${type}`)
.then(response => response.json())
.then(data => {
console.log(data);
document.getElementById("results").innerHTML =
render_json(data);
});
}
function incrementalSearch(event) {
type = document.getElementById("type").value;
// Search with reranking on 'Enter'
if (event.key === 'Enter') {
search(rerank=true);
}
// Limit incremental search to text types
else if (type !== "image") {
search(rerank=false);
}
}
function populate_type_dropdown() {
// Populate type dropdown field with enabled search types only
var possible_search_types = ["org", "markdown", "ledger", "music", "image"];
fetch("/api/config/data")
.then(response => response.json())
.then(data => {
document.getElementById("type").innerHTML =
possible_search_types
.filter(type => data["content-type"].hasOwnProperty(type) && data["content-type"][type])
.map(type => `<option value="${type}">${type.slice(0,1).toUpperCase() + type.slice(1)}</option>`)
.join('');
})
.then(() => {
// Set type field to search type passed in URL query parameter, if valid
var type_via_url = new URLSearchParams(window.location.search).get("t");
if (type_via_url && possible_search_types.includes(type_via_url))
document.getElementById("type").value = type_via_url;
});
}
function setTypeFieldInUrl(type) {
var url = new URL(window.location.href);
url.searchParams.set("t", type.value);
window.history.pushState({}, "", url.href);
}
function setCountFieldInUrl(results_count) {
var url = new URL(window.location.href);
url.searchParams.set("n", results_count.value);
window.history.pushState({}, "", url.href);
}
function setQueryFieldInUrl(query) {
var url = new URL(window.location.href);
url.searchParams.set("q", query);
window.history.pushState({}, "", url.href);
}
window.onload = function () {
// Dynamically populate type dropdown based on enabled search types and type passed as URL query parameter
populate_type_dropdown();
// Set results count field with value passed in URL query parameters, if any.
var results_count = new URLSearchParams(window.location.search).get("n");
if (results_count)
document.getElementById("results-count").value = results_count;
// Fill query field with value passed in URL query parameters, if any.
var query_via_url = new URLSearchParams(window.location.search).get("q");
if (query_via_url)
document.getElementById("query").value = query_via_url;
}
</script>
<body>
<h1>Khoj</h1>
<!--Add Text Box To Enter Query, Trigger Incremental Search OnChange -->
<input type="text" id="query" class="option" onkeyup=incrementalSearch(event) autofocus="autofocus" placeholder="What is the meaning of life?">
<div id="options">
<!--Add Dropdown to Select Query Type -->
<select id="type" class="option" onchange="setTypeFieldInUrl(this)"></select>
<!--Add Button To Regenerate -->
<button id="update" class="option" onclick="updateIndex()">Update</button>
<!--Add Results Count Input To Set Results Count -->
<input type="number" id="results-count" min="1" max="100" value="6" placeholder="results count" onchange="setCountFieldInUrl(this)">
</div>
<!-- Section to Render Results -->
<div id="results"></div>
</body>
<style>
@media only screen and (max-width: 600px) {
body {
display: grid;
grid-template-columns: 1fr;
grid-template-rows: 1fr 1fr 1fr minmax(80px, 100%);
}
body > * {
grid-column: 1;
}
}
@media only screen and (min-width: 600px) {
body {
display: grid;
grid-template-columns: 1fr min(70vw, 100%) 1fr;
grid-template-rows: 1fr 1fr 1fr minmax(80px, 100%);
padding-top: 60vw;
}
body > * {
grid-column: 2;
}
}
body {
padding: 0px;
margin: 0px;
background: #f8fafc;
color: #475569;
text-align: center;
font-family: roboto, karma, segoe ui, sans-serif;
font-size: 20px;
font-weight: 300;
line-height: 1.5em;
}
body > * {
padding: 10px;
margin: 10px;
}
h1 {
font-weight: 200;
color: #017eff;
}
#options {
padding: 0;
display: grid;
grid-template-columns: 1fr 1fr minmax(70px, 0.5fr);
}
#options > * {
padding: 15px;
border-radius: 5px;
border: 1px solid #475569;
background: #f9fafc
}
.option:hover {
box-shadow: 0 0 11px #aaa;
}
#options > select {
margin-right: 10px;
}
#options > button {
margin-right: 10px;
}
#query {
font-size: larger;
}
#results {
font-size: medium;
margin: 0px;
line-height: 20px;
}
#results-image {
display: grid;
grid-template-columns: repeat(3, 1fr);
}
.image-link {
place-self: center;
}
.image {
width: 20vw;
border-radius: 10px;
border: 1px solid #475569;
}
#json {
white-space: pre-wrap;
}
#results-ledger {
white-space: pre-line;
text-align: left;
}
#results-markdown {
text-align: left;
}
#results-music,
#results-org {
text-align: left;
white-space: pre-line;
}
#results-music h3,
#results-org h3 {
margin: 20px 0 0 0;
font-size: larger;
}
span.music-task-status,
span.task-status {
color: white;
padding: 3.5px 3.5px 0;
margin-right: 5px;
border-radius: 5px;
background-color: #eab308;
font-size: medium;
}
span.music-task-status.todo,
span.task-status.todo {
background-color: #3b82f6
}
span.music-task-status.done,
span.task-status.done {
background-color: #22c55e;
}
span.music-task-tag,
span.task-tag {
color: white;
padding: 3.5px 3.5px 0;
margin-right: 5px;
border-radius: 5px;
border: 1px solid #475569;
background-color: #ef4444;
font-size: small;
}
</style>
</html>

View File

@@ -1,16 +0,0 @@
{
"name": "Khoj",
"short_name": "Khoj",
"description": "A natural language search engine for your personal notes, transactions and photos",
"icons": [
{
"src": "/static/assets/icons/favicon-144x144.png",
"sizes": "144x144",
"type": "image/png"
}
],
"theme_color": "#ffffff",
"background_color": "#ffffff",
"display": "standalone",
"start_url": "/"
}