Merge branch 'master' of github.com:khoj-ai/khoj into features/big-upgrade-chat-ux

This commit is contained in:
sabaimran
2024-07-27 14:18:05 +05:30
46 changed files with 1598 additions and 1394 deletions

View File

@@ -61,6 +61,14 @@
let city = null;
let countryName = null;
let timezone = null;
let chatMessageState = {
newResponseTextEl: null,
newResponseEl: null,
loadingEllipsis: null,
references: {},
rawResponse: "",
isVoice: false,
}
fetch("https://ipapi.co/json")
.then(response => response.json())
@@ -75,10 +83,9 @@
return;
});
async function chat() {
// Extract required fields for search from form
async function chat(isVoice=false) {
// Extract chat message from chat input form
let query = document.getElementById("chat-input").value.trim();
let resultsCount = localStorage.getItem("khojResultsCount") || 5;
console.log(`Query: ${query}`);
// Short circuit on empty query
@@ -106,9 +113,6 @@
await refreshChatSessionsPanel();
}
// Generate backend API URL to execute query
let chatApi = `${hostURL}/api/chat?q=${encodeURIComponent(query)}&n=${resultsCount}&client=web&stream=true&conversation_id=${conversationID}&region=${region}&city=${city}&country=${countryName}&timezone=${timezone}`;
let newResponseEl = document.createElement("div");
newResponseEl.classList.add("chat-message", "khoj");
newResponseEl.attributes["data-meta"] = "🏮 Khoj at " + formatDate(new Date());
@@ -119,25 +123,7 @@
newResponseEl.appendChild(newResponseTextEl);
// Temporary status message to indicate that Khoj is thinking
let loadingEllipsis = document.createElement("div");
loadingEllipsis.classList.add("lds-ellipsis");
let firstEllipsis = document.createElement("div");
firstEllipsis.classList.add("lds-ellipsis-item");
let secondEllipsis = document.createElement("div");
secondEllipsis.classList.add("lds-ellipsis-item");
let thirdEllipsis = document.createElement("div");
thirdEllipsis.classList.add("lds-ellipsis-item");
let fourthEllipsis = document.createElement("div");
fourthEllipsis.classList.add("lds-ellipsis-item");
loadingEllipsis.appendChild(firstEllipsis);
loadingEllipsis.appendChild(secondEllipsis);
loadingEllipsis.appendChild(thirdEllipsis);
loadingEllipsis.appendChild(fourthEllipsis);
let loadingEllipsis = createLoadingEllipsis();
newResponseTextEl.appendChild(loadingEllipsis);
document.getElementById("chat-body").scrollTop = document.getElementById("chat-body").scrollHeight;
@@ -148,107 +134,36 @@
let chatInput = document.getElementById("chat-input");
chatInput.classList.remove("option-enabled");
// Setup chat message state
chatMessageState = {
newResponseTextEl,
newResponseEl,
loadingEllipsis,
references: {},
rawResponse: "",
rawQuery: query,
isVoice: isVoice,
}
// Call Khoj chat API
let response = await fetch(chatApi, { headers });
let rawResponse = "";
let references = null;
const contentType = response.headers.get("content-type");
let chatApi = `${hostURL}/api/chat?q=${encodeURIComponent(query)}&conversation_id=${conversationID}&stream=true&client=desktop`;
chatApi += (!!region && !!city && !!countryName && !!timezone)
? `&region=${region}&city=${city}&country=${countryName}&timezone=${timezone}`
: '';
if (contentType === "application/json") {
// Handle JSON response
try {
const responseAsJson = await response.json();
if (responseAsJson.image) {
// If response has image field, response is a generated image.
if (responseAsJson.intentType === "text-to-image") {
rawResponse += `![${query}](data:image/png;base64,${responseAsJson.image})`;
} else if (responseAsJson.intentType === "text-to-image2") {
rawResponse += `![${query}](${responseAsJson.image})`;
} else if (responseAsJson.intentType === "text-to-image-v3") {
rawResponse += `![${query}](data:image/webp;base64,${responseAsJson.image})`;
}
const inferredQueries = responseAsJson.inferredQueries?.[0];
if (inferredQueries) {
rawResponse += `\n\n**Inferred Query**:\n\n${inferredQueries}`;
}
}
if (responseAsJson.context) {
const rawReferenceAsJson = responseAsJson.context;
references = createReferenceSection(rawReferenceAsJson);
}
if (responseAsJson.detail) {
// If response has detail field, response is an error message.
rawResponse += responseAsJson.detail;
}
} catch (error) {
// If the chunk is not a JSON object, just display it as is
rawResponse += chunk;
} finally {
newResponseTextEl.innerHTML = "";
newResponseTextEl.appendChild(formatHTMLMessage(rawResponse));
const response = await fetch(chatApi, { headers });
if (references != null) {
newResponseTextEl.appendChild(references);
}
document.getElementById("chat-body").scrollTop = document.getElementById("chat-body").scrollHeight;
document.getElementById("chat-input").removeAttribute("disabled");
}
} else {
// Handle streamed response of type text/event-stream or text/plain
const reader = response.body.getReader();
const decoder = new TextDecoder();
let references = {};
readStream();
function readStream() {
reader.read().then(({ done, value }) => {
if (done) {
// Append any references after all the data has been streamed
if (references != {}) {
newResponseTextEl.appendChild(createReferenceSection(references));
}
document.getElementById("chat-body").scrollTop = document.getElementById("chat-body").scrollHeight;
document.getElementById("chat-input").removeAttribute("disabled");
return;
}
// Decode message chunk from stream
const chunk = decoder.decode(value, { stream: true });
if (chunk.includes("### compiled references:")) {
const additionalResponse = chunk.split("### compiled references:")[0];
rawResponse += additionalResponse;
newResponseTextEl.innerHTML = "";
newResponseTextEl.appendChild(formatHTMLMessage(rawResponse));
const rawReference = chunk.split("### compiled references:")[1];
const rawReferenceAsJson = JSON.parse(rawReference);
if (rawReferenceAsJson instanceof Array) {
references["notes"] = rawReferenceAsJson;
} else if (typeof rawReferenceAsJson === "object" && rawReferenceAsJson !== null) {
references["online"] = rawReferenceAsJson;
}
readStream();
} else {
// Display response from Khoj
if (newResponseTextEl.getElementsByClassName("lds-ellipsis").length > 0) {
newResponseTextEl.removeChild(loadingEllipsis);
}
// If the chunk is not a JSON object, just display it as is
rawResponse += chunk;
newResponseTextEl.innerHTML = "";
newResponseTextEl.appendChild(formatHTMLMessage(rawResponse));
readStream();
}
// Scroll to bottom of chat window as chat response is streamed
document.getElementById("chat-body").scrollTop = document.getElementById("chat-body").scrollHeight;
});
}
try {
if (!response.ok) throw new Error(response.statusText);
if (!response.body) throw new Error("Response body is empty");
// Stream and render chat response
await readChatStream(response);
} catch (err) {
console.error(`Khoj chat response failed with\n${err}`);
if (chatMessageState.newResponseEl.getElementsByClassName("lds-ellipsis").length > 0 && chatMessageState.loadingEllipsis)
chatMessageState.newResponseTextEl.removeChild(chatMessageState.loadingEllipsis);
let errorMsg = "Sorry, unable to get response from Khoj backend ❤️‍🩹. Retry or contact developers for help at <a href=mailto:'team@khoj.dev'>team@khoj.dev</a> or <a href='https://discord.gg/BDgyabRM6e'>on Discord</a>";
newResponseTextEl.textContent = errorMsg;
}
}

View File

@@ -364,3 +364,194 @@ function createReferenceSection(references, createLinkerSection=false) {
return referencesDiv;
}
function createLoadingEllipsis() {
let loadingEllipsis = document.createElement("div");
loadingEllipsis.classList.add("lds-ellipsis");
let firstEllipsis = document.createElement("div");
firstEllipsis.classList.add("lds-ellipsis-item");
let secondEllipsis = document.createElement("div");
secondEllipsis.classList.add("lds-ellipsis-item");
let thirdEllipsis = document.createElement("div");
thirdEllipsis.classList.add("lds-ellipsis-item");
let fourthEllipsis = document.createElement("div");
fourthEllipsis.classList.add("lds-ellipsis-item");
loadingEllipsis.appendChild(firstEllipsis);
loadingEllipsis.appendChild(secondEllipsis);
loadingEllipsis.appendChild(thirdEllipsis);
loadingEllipsis.appendChild(fourthEllipsis);
return loadingEllipsis;
}
function handleStreamResponse(newResponseElement, rawResponse, rawQuery, loadingEllipsis, replace=true) {
if (!newResponseElement) return;
// Remove loading ellipsis if it exists
if (newResponseElement.getElementsByClassName("lds-ellipsis").length > 0 && loadingEllipsis)
newResponseElement.removeChild(loadingEllipsis);
// Clear the response element if replace is true
if (replace) newResponseElement.innerHTML = "";
// Append response to the response element
newResponseElement.appendChild(formatHTMLMessage(rawResponse, false, replace, rawQuery));
// Append loading ellipsis if it exists
if (!replace && loadingEllipsis) newResponseElement.appendChild(loadingEllipsis);
// Scroll to bottom of chat view
document.getElementById("chat-body").scrollTop = document.getElementById("chat-body").scrollHeight;
}
function handleImageResponse(imageJson, rawResponse) {
if (imageJson.image) {
const inferredQuery = imageJson.inferredQueries?.[0] ?? "generated image";
// If response has image field, response is a generated image.
if (imageJson.intentType === "text-to-image") {
rawResponse += `![generated_image](data:image/png;base64,${imageJson.image})`;
} else if (imageJson.intentType === "text-to-image2") {
rawResponse += `![generated_image](${imageJson.image})`;
} else if (imageJson.intentType === "text-to-image-v3") {
rawResponse = `![](data:image/webp;base64,${imageJson.image})`;
}
if (inferredQuery) {
rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
}
}
// If response has detail field, response is an error message.
if (imageJson.detail) rawResponse += imageJson.detail;
return rawResponse;
}
function finalizeChatBodyResponse(references, newResponseElement) {
if (!!newResponseElement && references != null && Object.keys(references).length > 0) {
newResponseElement.appendChild(createReferenceSection(references));
}
document.getElementById("chat-body").scrollTop = document.getElementById("chat-body").scrollHeight;
document.getElementById("chat-input")?.removeAttribute("disabled");
}
function convertMessageChunkToJson(rawChunk) {
// Split the chunk into lines
if (rawChunk?.startsWith("{") && rawChunk?.endsWith("}")) {
try {
let jsonChunk = JSON.parse(rawChunk);
if (!jsonChunk.type)
jsonChunk = {type: 'message', data: jsonChunk};
return jsonChunk;
} catch (e) {
return {type: 'message', data: rawChunk};
}
} else if (rawChunk.length > 0) {
return {type: 'message', data: rawChunk};
}
}
function processMessageChunk(rawChunk) {
const chunk = convertMessageChunkToJson(rawChunk);
console.debug("Chunk:", chunk);
if (!chunk || !chunk.type) return;
if (chunk.type ==='status') {
console.log(`status: ${chunk.data}`);
const statusMessage = chunk.data;
handleStreamResponse(chatMessageState.newResponseTextEl, statusMessage, chatMessageState.rawQuery, chatMessageState.loadingEllipsis, false);
} else if (chunk.type === 'start_llm_response') {
console.log("Started streaming", new Date());
} else if (chunk.type === 'end_llm_response') {
console.log("Stopped streaming", new Date());
// Automatically respond with voice if the subscribed user has sent voice message
if (chatMessageState.isVoice && "{{ is_active }}" == "True")
textToSpeech(chatMessageState.rawResponse);
// Append any references after all the data has been streamed
finalizeChatBodyResponse(chatMessageState.references, chatMessageState.newResponseTextEl);
const liveQuery = chatMessageState.rawQuery;
// Reset variables
chatMessageState = {
newResponseTextEl: null,
newResponseEl: null,
loadingEllipsis: null,
references: {},
rawResponse: "",
rawQuery: liveQuery,
isVoice: false,
}
} else if (chunk.type === "references") {
chatMessageState.references = {"notes": chunk.data.context, "online": chunk.data.onlineContext};
} else if (chunk.type === 'message') {
const chunkData = chunk.data;
if (typeof chunkData === 'object' && chunkData !== null) {
// If chunkData is already a JSON object
handleJsonResponse(chunkData);
} else if (typeof chunkData === 'string' && chunkData.trim()?.startsWith("{") && chunkData.trim()?.endsWith("}")) {
// Try process chunk data as if it is a JSON object
try {
const jsonData = JSON.parse(chunkData.trim());
handleJsonResponse(jsonData);
} catch (e) {
chatMessageState.rawResponse += chunkData;
handleStreamResponse(chatMessageState.newResponseTextEl, chatMessageState.rawResponse, chatMessageState.rawQuery, chatMessageState.loadingEllipsis);
}
} else {
chatMessageState.rawResponse += chunkData;
handleStreamResponse(chatMessageState.newResponseTextEl, chatMessageState.rawResponse, chatMessageState.rawQuery, chatMessageState.loadingEllipsis);
}
}
}
function handleJsonResponse(jsonData) {
if (jsonData.image || jsonData.detail) {
chatMessageState.rawResponse = handleImageResponse(jsonData, chatMessageState.rawResponse);
} else if (jsonData.response) {
chatMessageState.rawResponse = jsonData.response;
}
if (chatMessageState.newResponseTextEl) {
chatMessageState.newResponseTextEl.innerHTML = "";
chatMessageState.newResponseTextEl.appendChild(formatHTMLMessage(chatMessageState.rawResponse));
}
}
async function readChatStream(response) {
if (!response.body) return;
const reader = response.body.getReader();
const decoder = new TextDecoder();
const eventDelimiter = '␃🔚␗';
let buffer = '';
while (true) {
const { value, done } = await reader.read();
// If the stream is done
if (done) {
// Process the last chunk
processMessageChunk(buffer);
buffer = '';
break;
}
// Read chunk from stream and append it to the buffer
const chunk = decoder.decode(value, { stream: true });
console.debug("Raw Chunk:", chunk)
// Start buffering chunks until complete event is received
buffer += chunk;
// Once the buffer contains a complete event
let newEventIndex;
while ((newEventIndex = buffer.indexOf(eventDelimiter)) !== -1) {
// Extract the event from the buffer
const event = buffer.slice(0, newEventIndex);
buffer = buffer.slice(newEventIndex + eventDelimiter.length);
// Process the event
if (event) processMessageChunk(event);
}
}
}

View File

@@ -1,6 +1,6 @@
{
"name": "Khoj",
"version": "1.16.0",
"version": "1.17.0",
"description": "An AI copilot for your Second Brain",
"author": "Saba Imran, Debanjum Singh Solanky <team@khoj.dev>",
"license": "GPL-3.0-or-later",

View File

@@ -346,7 +346,7 @@
inp.focus();
}
async function chat() {
async function chat(isVoice=false) {
//set chat body to empty
let chatBody = document.getElementById("chat-body");
chatBody.innerHTML = "";
@@ -375,9 +375,6 @@
chat_body.dataset.conversationId = conversationID;
}
// Generate backend API URL to execute query
let chatApi = `${hostURL}/api/chat?q=${encodeURIComponent(query)}&n=${resultsCount}&client=web&stream=true&conversation_id=${conversationID}&region=${region}&city=${city}&country=${countryName}&timezone=${timezone}`;
let newResponseEl = document.createElement("div");
newResponseEl.classList.add("chat-message", "khoj");
newResponseEl.attributes["data-meta"] = "🏮 Khoj at " + formatDate(new Date());
@@ -388,128 +385,41 @@
newResponseEl.appendChild(newResponseTextEl);
// Temporary status message to indicate that Khoj is thinking
let loadingEllipsis = document.createElement("div");
loadingEllipsis.classList.add("lds-ellipsis");
let firstEllipsis = document.createElement("div");
firstEllipsis.classList.add("lds-ellipsis-item");
let secondEllipsis = document.createElement("div");
secondEllipsis.classList.add("lds-ellipsis-item");
let thirdEllipsis = document.createElement("div");
thirdEllipsis.classList.add("lds-ellipsis-item");
let fourthEllipsis = document.createElement("div");
fourthEllipsis.classList.add("lds-ellipsis-item");
loadingEllipsis.appendChild(firstEllipsis);
loadingEllipsis.appendChild(secondEllipsis);
loadingEllipsis.appendChild(thirdEllipsis);
loadingEllipsis.appendChild(fourthEllipsis);
newResponseTextEl.appendChild(loadingEllipsis);
let loadingEllipsis = createLoadingEllipsis();
document.body.scrollTop = document.getElementById("chat-body").scrollHeight;
// Call Khoj chat API
let response = await fetch(chatApi, { headers });
let rawResponse = "";
let references = null;
const contentType = response.headers.get("content-type");
toggleLoading();
if (contentType === "application/json") {
// Handle JSON response
try {
const responseAsJson = await response.json();
if (responseAsJson.image) {
// If response has image field, response is a generated image.
if (responseAsJson.intentType === "text-to-image") {
rawResponse += `![${query}](data:image/png;base64,${responseAsJson.image})`;
} else if (responseAsJson.intentType === "text-to-image2") {
rawResponse += `![${query}](${responseAsJson.image})`;
} else if (responseAsJson.intentType === "text-to-image-v3") {
rawResponse += `![${query}](data:image/webp;base64,${responseAsJson.image})`;
}
const inferredQueries = responseAsJson.inferredQueries?.[0];
if (inferredQueries) {
rawResponse += `\n\n**Inferred Query**:\n\n${inferredQueries}`;
}
}
if (responseAsJson.context) {
const rawReferenceAsJson = responseAsJson.context;
references = createReferenceSection(rawReferenceAsJson, createLinkerSection=true);
}
if (responseAsJson.detail) {
// If response has detail field, response is an error message.
rawResponse += responseAsJson.detail;
}
} catch (error) {
// If the chunk is not a JSON object, just display it as is
rawResponse += chunk;
} finally {
newResponseTextEl.innerHTML = "";
newResponseTextEl.appendChild(formatHTMLMessage(rawResponse));
if (references != null) {
newResponseTextEl.appendChild(references);
}
// Setup chat message state
chatMessageState = {
newResponseTextEl,
newResponseEl,
loadingEllipsis,
references: {},
rawResponse: "",
rawQuery: query,
isVoice: isVoice,
}
document.body.scrollTop = document.getElementById("chat-body").scrollHeight;
}
} else {
// Handle streamed response of type text/event-stream or text/plain
const reader = response.body.getReader();
const decoder = new TextDecoder();
let references = {};
// Construct API URL to execute chat query
let chatApi = `${hostURL}/api/chat?q=${encodeURIComponent(query)}&conversation_id=${conversationID}&stream=true&client=desktop`;
chatApi += (!!region && !!city && !!countryName && !!timezone)
? `&region=${region}&city=${city}&country=${countryName}&timezone=${timezone}`
: '';
readStream();
const response = await fetch(chatApi, { headers });
function readStream() {
reader.read().then(({ done, value }) => {
if (done) {
// Append any references after all the data has been streamed
if (references != {}) {
newResponseTextEl.appendChild(createReferenceSection(references, createLinkerSection=true));
}
document.body.scrollTop = document.getElementById("chat-body").scrollHeight;
return;
}
// Decode message chunk from stream
const chunk = decoder.decode(value, { stream: true });
if (chunk.includes("### compiled references:")) {
const additionalResponse = chunk.split("### compiled references:")[0];
rawResponse += additionalResponse;
newResponseTextEl.innerHTML = "";
newResponseTextEl.appendChild(formatHTMLMessage(rawResponse));
const rawReference = chunk.split("### compiled references:")[1];
const rawReferenceAsJson = JSON.parse(rawReference);
if (rawReferenceAsJson instanceof Array) {
references["notes"] = rawReferenceAsJson;
} else if (typeof rawReferenceAsJson === "object" && rawReferenceAsJson !== null) {
references["online"] = rawReferenceAsJson;
}
readStream();
} else {
// Display response from Khoj
if (newResponseTextEl.getElementsByClassName("lds-ellipsis").length > 0) {
newResponseTextEl.removeChild(loadingEllipsis);
}
// If the chunk is not a JSON object, just display it as is
rawResponse += chunk;
newResponseTextEl.innerHTML = "";
newResponseTextEl.appendChild(formatHTMLMessage(rawResponse));
readStream();
}
// Scroll to bottom of chat window as chat response is streamed
document.body.scrollTop = document.getElementById("chat-body").scrollHeight;
});
}
try {
if (!response.ok) throw new Error(response.statusText);
if (!response.body) throw new Error("Response body is empty");
// Stream and render chat response
await readChatStream(response);
} catch (err) {
console.error(`Khoj chat response failed with\n${err}`);
if (chatMessageState.newResponseEl.getElementsByClassName("lds-ellipsis").length > 0 && chatMessageState.loadingEllipsis)
chatMessageState.newResponseTextEl.removeChild(chatMessageState.loadingEllipsis);
let errorMsg = "Sorry, unable to get response from Khoj backend ❤️‍🩹. Retry or contact developers for help at <a href=mailto:'team@khoj.dev'>team@khoj.dev</a> or <a href='https://discord.gg/BDgyabRM6e'>on Discord</a>";
newResponseTextEl.textContent = errorMsg;
}
document.body.scrollTop = document.getElementById("chat-body").scrollHeight;
}

View File

@@ -34,8 +34,8 @@ function toggleNavMenu() {
document.addEventListener('click', function(event) {
let menu = document.getElementById("khoj-nav-menu");
let menuContainer = document.getElementById("khoj-nav-menu-container");
let isClickOnMenu = menuContainer.contains(event.target) || menuContainer === event.target;
if (isClickOnMenu === false && menu.classList.contains("show")) {
let isClickOnMenu = menuContainer?.contains(event.target) || menuContainer === event.target;
if (menu && isClickOnMenu === false && menu.classList.contains("show")) {
menu.classList.remove("show");
}
});

View File

@@ -6,7 +6,7 @@
;; Saba Imran <saba@khoj.dev>
;; Description: An AI copilot for your Second Brain
;; Keywords: search, chat, org-mode, outlines, markdown, pdf, image
;; Version: 1.16.0
;; Version: 1.17.0
;; Package-Requires: ((emacs "27.1") (transient "0.3.0") (dash "2.19.1"))
;; URL: https://github.com/khoj-ai/khoj/tree/master/src/interface/emacs
@@ -283,9 +283,9 @@ Auto invokes setup steps on calling main entrypoint."
(if (/= (apply #'call-process khoj-server-python-command
nil t nil
"-m" "pip" "install" "--upgrade"
'("khoj-assistant"))
'("khoj"))
0)
(message "khoj.el: Failed to install Khoj server. Please install it manually using pip install `khoj-assistant'.\n%s" (buffer-string))
(message "khoj.el: Failed to install Khoj server. Please install it manually using pip install `khoj'.\n%s" (buffer-string))
(message "khoj.el: Installed and upgraded Khoj server version: %s" (khoj--server-get-version)))))
(defun khoj--server-start ()

View File

@@ -1,7 +1,7 @@
{
"id": "khoj",
"name": "Khoj",
"version": "1.16.0",
"version": "1.17.0",
"minAppVersion": "0.15.0",
"description": "An AI copilot for your Second Brain",
"author": "Khoj Inc.",

View File

@@ -1,6 +1,6 @@
{
"name": "Khoj",
"version": "1.16.0",
"version": "1.17.0",
"description": "An AI copilot for your Second Brain",
"author": "Debanjum Singh Solanky, Saba Imran <team@khoj.dev>",
"license": "GPL-3.0-or-later",

View File

@@ -12,6 +12,25 @@ export interface ChatJsonResult {
inferredQueries?: string[];
}
interface ChunkResult {
objects: string[];
remainder: string;
}
interface MessageChunk {
type: string;
data: any;
}
interface ChatMessageState {
newResponseTextEl: HTMLElement | null;
newResponseEl: HTMLElement | null;
loadingEllipsis: HTMLElement | null;
references: any;
rawResponse: string;
rawQuery: string;
isVoice: boolean;
}
interface Location {
region: string;
@@ -26,6 +45,7 @@ export class KhojChatView extends KhojPaneView {
waitingForLocation: boolean;
location: Location;
keyPressTimeout: NodeJS.Timeout | null = null;
chatMessageState: ChatMessageState;
constructor(leaf: WorkspaceLeaf, setting: KhojSetting) {
super(leaf, setting);
@@ -409,16 +429,15 @@ export class KhojChatView extends KhojPaneView {
message = DOMPurify.sanitize(message);
// Convert the message to html, sanitize the message html and render it to the real DOM
let chat_message_body_text_el = this.contentEl.createDiv();
chat_message_body_text_el.className = "chat-message-text-response";
chat_message_body_text_el.innerHTML = this.markdownTextToSanitizedHtml(message, this);
let chatMessageBodyTextEl = this.contentEl.createDiv();
chatMessageBodyTextEl.innerHTML = this.markdownTextToSanitizedHtml(message, this);
// Add a copy button to each chat message, if it doesn't already exist
if (willReplace === true) {
this.renderActionButtons(message, chat_message_body_text_el);
this.renderActionButtons(message, chatMessageBodyTextEl);
}
return chat_message_body_text_el;
return chatMessageBodyTextEl;
}
markdownTextToSanitizedHtml(markdownText: string, component: ItemView): string {
@@ -502,23 +521,23 @@ export class KhojChatView extends KhojPaneView {
class: `khoj-chat-message ${sender}`
},
})
let chat_message_body_el = chatMessageEl.createDiv();
chat_message_body_el.addClasses(["khoj-chat-message-text", sender]);
let chat_message_body_text_el = chat_message_body_el.createDiv();
let chatMessageBodyEl = chatMessageEl.createDiv();
chatMessageBodyEl.addClasses(["khoj-chat-message-text", sender]);
let chatMessageBodyTextEl = chatMessageBodyEl.createDiv();
// Sanitize the markdown to render
message = DOMPurify.sanitize(message);
if (raw) {
chat_message_body_text_el.innerHTML = message;
chatMessageBodyTextEl.innerHTML = message;
} else {
// @ts-ignore
chat_message_body_text_el.innerHTML = this.markdownTextToSanitizedHtml(message, this);
chatMessageBodyTextEl.innerHTML = this.markdownTextToSanitizedHtml(message, this);
}
// Add action buttons to each chat message element
if (willReplace === true) {
this.renderActionButtons(message, chat_message_body_text_el);
this.renderActionButtons(message, chatMessageBodyTextEl);
}
// Remove user-select: none property to make text selectable
@@ -531,42 +550,38 @@ export class KhojChatView extends KhojPaneView {
}
createKhojResponseDiv(dt?: Date): HTMLDivElement {
let message_time = this.formatDate(dt ?? new Date());
let messageTime = this.formatDate(dt ?? new Date());
// Append message to conversation history HTML element.
// The chat logs should display above the message input box to follow standard UI semantics
let chat_body_el = this.contentEl.getElementsByClassName("khoj-chat-body")[0];
let chat_message_el = chat_body_el.createDiv({
let chatBodyEl = this.contentEl.getElementsByClassName("khoj-chat-body")[0];
let chatMessageEl = chatBodyEl.createDiv({
attr: {
"data-meta": `🏮 Khoj at ${message_time}`,
"data-meta": `🏮 Khoj at ${messageTime}`,
class: `khoj-chat-message khoj`
},
}).createDiv({
attr: {
class: `khoj-chat-message-text khoj`
},
}).createDiv();
})
// Scroll to bottom after inserting chat messages
this.scrollChatToBottom();
return chat_message_el;
return chatMessageEl;
}
async renderIncrementalMessage(htmlElement: HTMLDivElement, additionalMessage: string) {
this.result += additionalMessage;
this.chatMessageState.rawResponse += additionalMessage;
htmlElement.innerHTML = "";
// Sanitize the markdown to render
this.result = DOMPurify.sanitize(this.result);
this.chatMessageState.rawResponse = DOMPurify.sanitize(this.chatMessageState.rawResponse);
// @ts-ignore
htmlElement.innerHTML = this.markdownTextToSanitizedHtml(this.result, this);
htmlElement.innerHTML = this.markdownTextToSanitizedHtml(this.chatMessageState.rawResponse, this);
// Render action buttons for the message
this.renderActionButtons(this.result, htmlElement);
this.renderActionButtons(this.chatMessageState.rawResponse, htmlElement);
// Scroll to bottom of modal, till the send message input box
this.scrollChatToBottom();
}
renderActionButtons(message: string, chat_message_body_text_el: HTMLElement) {
renderActionButtons(message: string, chatMessageBodyTextEl: HTMLElement) {
let copyButton = this.contentEl.createEl('button');
copyButton.classList.add("chat-action-button");
copyButton.title = "Copy Message to Clipboard";
@@ -593,10 +608,10 @@ export class KhojChatView extends KhojPaneView {
}
// Append buttons to parent element
chat_message_body_text_el.append(copyButton, pasteToFile);
chatMessageBodyTextEl.append(copyButton, pasteToFile);
if (speechButton) {
chat_message_body_text_el.append(speechButton);
chatMessageBodyTextEl.append(speechButton);
}
}
@@ -854,35 +869,122 @@ export class KhojChatView extends KhojPaneView {
return true;
}
async readChatStream(response: Response, responseElement: HTMLDivElement, isVoice: boolean = false): Promise<void> {
convertMessageChunkToJson(rawChunk: string): MessageChunk {
if (rawChunk?.startsWith("{") && rawChunk?.endsWith("}")) {
try {
let jsonChunk = JSON.parse(rawChunk);
if (!jsonChunk.type)
jsonChunk = {type: 'message', data: jsonChunk};
return jsonChunk;
} catch (e) {
return {type: 'message', data: rawChunk};
}
} else if (rawChunk.length > 0) {
return {type: 'message', data: rawChunk};
}
return {type: '', data: ''};
}
processMessageChunk(rawChunk: string): void {
const chunk = this.convertMessageChunkToJson(rawChunk);
console.debug("Chunk:", chunk);
if (!chunk || !chunk.type) return;
if (chunk.type === 'status') {
console.log(`status: ${chunk.data}`);
const statusMessage = chunk.data;
this.handleStreamResponse(this.chatMessageState.newResponseTextEl, statusMessage, this.chatMessageState.loadingEllipsis, false);
} else if (chunk.type === 'start_llm_response') {
console.log("Started streaming", new Date());
} else if (chunk.type === 'end_llm_response') {
console.log("Stopped streaming", new Date());
// Automatically respond with voice if the subscribed user has sent voice message
if (this.chatMessageState.isVoice && this.setting.userInfo?.is_active)
this.textToSpeech(this.chatMessageState.rawResponse);
// Append any references after all the data has been streamed
this.finalizeChatBodyResponse(this.chatMessageState.references, this.chatMessageState.newResponseTextEl);
const liveQuery = this.chatMessageState.rawQuery;
// Reset variables
this.chatMessageState = {
newResponseTextEl: null,
newResponseEl: null,
loadingEllipsis: null,
references: {},
rawResponse: "",
rawQuery: liveQuery,
isVoice: false,
};
} else if (chunk.type === "references") {
this.chatMessageState.references = {"notes": chunk.data.context, "online": chunk.data.onlineContext};
} else if (chunk.type === 'message') {
const chunkData = chunk.data;
if (typeof chunkData === 'object' && chunkData !== null) {
// If chunkData is already a JSON object
this.handleJsonResponse(chunkData);
} else if (typeof chunkData === 'string' && chunkData.trim()?.startsWith("{") && chunkData.trim()?.endsWith("}")) {
// Try process chunk data as if it is a JSON object
try {
const jsonData = JSON.parse(chunkData.trim());
this.handleJsonResponse(jsonData);
} catch (e) {
this.chatMessageState.rawResponse += chunkData;
this.handleStreamResponse(this.chatMessageState.newResponseTextEl, this.chatMessageState.rawResponse, this.chatMessageState.loadingEllipsis);
}
} else {
this.chatMessageState.rawResponse += chunkData;
this.handleStreamResponse(this.chatMessageState.newResponseTextEl, this.chatMessageState.rawResponse, this.chatMessageState.loadingEllipsis);
}
}
}
handleJsonResponse(jsonData: any): void {
if (jsonData.image || jsonData.detail) {
this.chatMessageState.rawResponse = this.handleImageResponse(jsonData, this.chatMessageState.rawResponse);
} else if (jsonData.response) {
this.chatMessageState.rawResponse = jsonData.response;
}
if (this.chatMessageState.newResponseTextEl) {
this.chatMessageState.newResponseTextEl.innerHTML = "";
this.chatMessageState.newResponseTextEl.appendChild(this.formatHTMLMessage(this.chatMessageState.rawResponse));
}
}
async readChatStream(response: Response): Promise<void> {
// Exit if response body is empty
if (response.body == null) return;
const reader = response.body.getReader();
const decoder = new TextDecoder();
const eventDelimiter = '␃🔚␗';
let buffer = '';
while (true) {
const { value, done } = await reader.read();
if (done) {
// Automatically respond with voice if the subscribed user has sent voice message
if (isVoice && this.setting.userInfo?.is_active) this.textToSpeech(this.result);
this.processMessageChunk(buffer);
buffer = '';
// Break if the stream is done
break;
}
let responseText = decoder.decode(value);
if (responseText.includes("### compiled references:")) {
// Render any references used to generate the response
const [additionalResponse, rawReference] = responseText.split("### compiled references:", 2);
await this.renderIncrementalMessage(responseElement, additionalResponse);
const chunk = decoder.decode(value, { stream: true });
console.debug("Raw Chunk:", chunk)
// Start buffering chunks until complete event is received
buffer += chunk;
const rawReferenceAsJson = JSON.parse(rawReference);
let references = this.extractReferences(rawReferenceAsJson);
responseElement.appendChild(this.createReferenceSection(references));
} else {
// Render incremental chat response
await this.renderIncrementalMessage(responseElement, responseText);
// Once the buffer contains a complete event
let newEventIndex;
while ((newEventIndex = buffer.indexOf(eventDelimiter)) !== -1) {
// Extract the event from the buffer
const event = buffer.slice(0, newEventIndex);
buffer = buffer.slice(newEventIndex + eventDelimiter.length);
// Process the event
if (event) this.processMessageChunk(event);
}
}
}
@@ -895,83 +997,59 @@ export class KhojChatView extends KhojPaneView {
let chatBodyEl = this.contentEl.getElementsByClassName("khoj-chat-body")[0] as HTMLElement;
this.renderMessage(chatBodyEl, query, "you");
let conversationID = chatBodyEl.dataset.conversationId;
if (!conversationID) {
let conversationId = chatBodyEl.dataset.conversationId;
if (!conversationId) {
let chatUrl = `${this.setting.khojUrl}/api/chat/sessions?client=obsidian`;
let response = await fetch(chatUrl, {
method: "POST",
headers: { "Authorization": `Bearer ${this.setting.khojApiKey}` },
});
let data = await response.json();
conversationID = data.conversation_id;
chatBodyEl.dataset.conversationId = conversationID;
conversationId = data.conversation_id;
chatBodyEl.dataset.conversationId = conversationId;
}
// Get chat response from Khoj backend
let encodedQuery = encodeURIComponent(query);
let chatUrl = `${this.setting.khojUrl}/api/chat?q=${encodedQuery}&n=${this.setting.resultsCount}&client=obsidian&stream=true&region=${this.location.region}&city=${this.location.city}&country=${this.location.countryName}&timezone=${this.location.timezone}`;
let responseElement = this.createKhojResponseDiv();
let chatUrl = `${this.setting.khojUrl}/api/chat?q=${encodedQuery}&conversation_id=${conversationId}&n=${this.setting.resultsCount}&stream=true&client=obsidian`;
if (!!this.location) chatUrl += `&region=${this.location.region}&city=${this.location.city}&country=${this.location.countryName}&timezone=${this.location.timezone}`;
let newResponseEl = this.createKhojResponseDiv();
let newResponseTextEl = newResponseEl.createDiv();
newResponseTextEl.classList.add("khoj-chat-message-text", "khoj");
// Temporary status message to indicate that Khoj is thinking
this.result = "";
let loadingEllipsis = this.createLoadingEllipse();
responseElement.appendChild(loadingEllipsis);
newResponseTextEl.appendChild(loadingEllipsis);
// Set chat message state
this.chatMessageState = {
newResponseEl: newResponseEl,
newResponseTextEl: newResponseTextEl,
loadingEllipsis: loadingEllipsis,
references: {},
rawQuery: query,
rawResponse: "",
isVoice: isVoice,
};
let response = await fetch(chatUrl, {
method: "GET",
headers: {
"Content-Type": "text/event-stream",
"Content-Type": "text/plain",
"Authorization": `Bearer ${this.setting.khojApiKey}`,
},
})
try {
if (response.body === null) {
throw new Error("Response body is null");
}
if (response.body === null) throw new Error("Response body is null");
// Clear loading status message
if (responseElement.getElementsByClassName("lds-ellipsis").length > 0 && loadingEllipsis) {
responseElement.removeChild(loadingEllipsis);
}
// Reset collated chat result to empty string
this.result = "";
responseElement.innerHTML = "";
if (response.headers.get("content-type") === "application/json") {
let responseText = ""
try {
const responseAsJson = await response.json() as ChatJsonResult;
if (responseAsJson.image) {
// If response has image field, response is a generated image.
if (responseAsJson.intentType === "text-to-image") {
responseText += `![${query}](data:image/png;base64,${responseAsJson.image})`;
} else if (responseAsJson.intentType === "text-to-image2") {
responseText += `![${query}](${responseAsJson.image})`;
} else if (responseAsJson.intentType === "text-to-image-v3") {
responseText += `![${query}](data:image/webp;base64,${responseAsJson.image})`;
}
const inferredQuery = responseAsJson.inferredQueries?.[0];
if (inferredQuery) {
responseText += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
}
} else if (responseAsJson.detail) {
responseText = responseAsJson.detail;
}
} catch (error) {
// If the chunk is not a JSON object, just display it as is
responseText = await response.text();
} finally {
await this.renderIncrementalMessage(responseElement, responseText);
}
} else {
// Stream and render chat response
await this.readChatStream(response, responseElement, isVoice);
}
// Stream and render chat response
await this.readChatStream(response);
} catch (err) {
console.log(`Khoj chat response failed with\n${err}`);
console.error(`Khoj chat response failed with\n${err}`);
let errorMsg = "Sorry, unable to get response from Khoj backend ❤️‍🩹. Retry or contact developers for help at <a href=mailto:'team@khoj.dev'>team@khoj.dev</a> or <a href='https://discord.gg/BDgyabRM6e'>on Discord</a>";
responseElement.innerHTML = errorMsg
newResponseTextEl.textContent = errorMsg;
}
}
@@ -1196,30 +1274,21 @@ export class KhojChatView extends KhojPaneView {
handleStreamResponse(newResponseElement: HTMLElement | null, rawResponse: string, loadingEllipsis: HTMLElement | null, replace = true) {
if (!newResponseElement) return;
if (newResponseElement.getElementsByClassName("lds-ellipsis").length > 0 && loadingEllipsis) {
// Remove loading ellipsis if it exists
if (newResponseElement.getElementsByClassName("lds-ellipsis").length > 0 && loadingEllipsis)
newResponseElement.removeChild(loadingEllipsis);
}
if (replace) {
newResponseElement.innerHTML = "";
}
// Clear the response element if replace is true
if (replace) newResponseElement.innerHTML = "";
// Append response to the response element
newResponseElement.appendChild(this.formatHTMLMessage(rawResponse, false, replace));
// Append loading ellipsis if it exists
if (!replace && loadingEllipsis) newResponseElement.appendChild(loadingEllipsis);
// Scroll to bottom of chat view
this.scrollChatToBottom();
}
handleCompiledReferences(rawResponseElement: HTMLElement | null, chunk: string, references: any, rawResponse: string) {
if (!rawResponseElement || !chunk) return { rawResponse, references };
const [additionalResponse, rawReference] = chunk.split("### compiled references:", 2);
rawResponse += additionalResponse;
rawResponseElement.innerHTML = "";
rawResponseElement.appendChild(this.formatHTMLMessage(rawResponse));
const rawReferenceAsJson = JSON.parse(rawReference);
references = this.extractReferences(rawReferenceAsJson);
return { rawResponse, references };
}
handleImageResponse(imageJson: any, rawResponse: string) {
if (imageJson.image) {
const inferredQuery = imageJson.inferredQueries?.[0] ?? "generated image";
@@ -1236,33 +1305,10 @@ export class KhojChatView extends KhojPaneView {
rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
}
}
let references = {};
if (imageJson.context && imageJson.context.length > 0) {
references = this.extractReferences(imageJson.context);
}
if (imageJson.detail) {
// If response has detail field, response is an error message.
rawResponse += imageJson.detail;
}
return { rawResponse, references };
}
// If response has detail field, response is an error message.
if (imageJson.detail) rawResponse += imageJson.detail;
extractReferences(rawReferenceAsJson: any): object {
let references: any = {};
if (rawReferenceAsJson instanceof Array) {
references["notes"] = rawReferenceAsJson;
} else if (typeof rawReferenceAsJson === "object" && rawReferenceAsJson !== null) {
references["online"] = rawReferenceAsJson;
}
return references;
}
addMessageToChatBody(rawResponse: string, newResponseElement: HTMLElement | null, references: any) {
if (!newResponseElement) return;
newResponseElement.innerHTML = "";
newResponseElement.appendChild(this.formatHTMLMessage(rawResponse));
this.finalizeChatBodyResponse(references, newResponseElement);
return rawResponse;
}
finalizeChatBodyResponse(references: object, newResponseElement: HTMLElement | null) {

View File

@@ -1,6 +1,6 @@
import { App, SuggestModal, request, MarkdownRenderer, Instruction, Platform } from 'obsidian';
import { KhojSetting } from 'src/settings';
import { createNoteAndCloseModal, getLinkToEntry } from 'src/utils';
import { supportedBinaryFileTypes, createNoteAndCloseModal, getFileFromPath, getLinkToEntry, supportedImageFilesTypes } from 'src/utils';
export interface SearchResult {
entry: string;
@@ -112,28 +112,41 @@ export class KhojSearchModal extends SuggestModal<SearchResult> {
let os_path_separator = result.file.includes('\\') ? '\\' : '/';
let filename = result.file.split(os_path_separator).pop();
// Remove YAML frontmatter when rendering string
result.entry = result.entry.replace(/---[\n\r][\s\S]*---[\n\r]/, '');
// Truncate search results to lines_to_render
let entry_snipped_indicator = result.entry.split('\n').length > lines_to_render ? ' **...**' : '';
let snipped_entry = result.entry.split('\n').slice(0, lines_to_render).join('\n');
// Show filename of each search result for context
el.createEl("div",{ cls: 'khoj-result-file' }).setText(filename ?? "");
let result_el = el.createEl("div", { cls: 'khoj-result-entry' })
let resultToRender = "";
let fileExtension = filename?.split(".").pop() ?? "";
if (supportedImageFilesTypes.includes(fileExtension) && filename) {
let linkToEntry: string = filename;
let imageFiles = this.app.vault.getFiles().filter(file => supportedImageFilesTypes.includes(fileExtension));
// Find vault file of chosen search result
let fileInVault = getFileFromPath(imageFiles, result.file);
if (fileInVault)
linkToEntry = this.app.vault.getResourcePath(fileInVault);
resultToRender = `![](${linkToEntry})`;
} else {
// Remove YAML frontmatter when rendering string
result.entry = result.entry.replace(/---[\n\r][\s\S]*---[\n\r]/, '');
// Truncate search results to lines_to_render
let entry_snipped_indicator = result.entry.split('\n').length > lines_to_render ? ' **...**' : '';
let snipped_entry = result.entry.split('\n').slice(0, lines_to_render).join('\n');
resultToRender = `${snipped_entry}${entry_snipped_indicator}`;
}
// @ts-ignore
MarkdownRenderer.renderMarkdown(snipped_entry + entry_snipped_indicator, result_el, result.file, null);
MarkdownRenderer.renderMarkdown(resultToRender, result_el, result.file, null);
}
async onChooseSuggestion(result: SearchResult, _: MouseEvent | KeyboardEvent) {
// Get all markdown and PDF files in vault
// Get all markdown, pdf and image files in vault
const mdFiles = this.app.vault.getMarkdownFiles();
const pdfFiles = this.app.vault.getFiles().filter(file => file.extension === 'pdf');
const binaryFiles = this.app.vault.getFiles().filter(file => supportedBinaryFileTypes.includes(file.extension));
// Find, Open vault file at heading of chosen search result
let linkToEntry = getLinkToEntry(mdFiles.concat(pdfFiles), result.file, result.entry);
let linkToEntry = getLinkToEntry(mdFiles.concat(binaryFiles), result.file, result.entry);
if (linkToEntry) this.app.workspace.openLinkText(linkToEntry, '');
}
}

View File

@@ -10,7 +10,6 @@ export interface UserInfo {
email?: string;
}
export interface KhojSetting {
resultsCount: number;
khojUrl: string;

View File

@@ -48,11 +48,14 @@ function filenameToMimeType (filename: TFile): string {
}
}
export const supportedImageFilesTypes = ['png', 'jpg', 'jpeg'];
export const supportedBinaryFileTypes = ['pdf'].concat(supportedImageFilesTypes);
export const supportedFileTypes = ['md', 'markdown'].concat(supportedBinaryFileTypes);
export async function updateContentIndex(vault: Vault, setting: KhojSetting, lastSync: Map<TFile, number>, regenerate: boolean = false): Promise<Map<TFile, number>> {
// Get all markdown, pdf files in the vault
console.log(`Khoj: Updating Khoj content index...`)
const files = vault.getFiles().filter(file => file.extension === 'md' || file.extension === 'markdown' || file.extension === 'pdf');
const binaryFileTypes = ['pdf']
const files = vault.getFiles().filter(file => supportedFileTypes.includes(file.extension));
let countOfFilesToIndex = 0;
let countOfFilesToDelete = 0;
lastSync = lastSync.size > 0 ? lastSync : new Map<TFile, number>();
@@ -66,7 +69,7 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las
}
countOfFilesToIndex++;
const encoding = binaryFileTypes.includes(file.extension) ? "binary" : "utf8";
const encoding = supportedBinaryFileTypes.includes(file.extension) ? "binary" : "utf8";
const mimeType = fileExtensionToMimeType(file.extension) + (encoding === "utf8" ? "; charset=UTF-8" : "");
const fileContent = encoding == 'binary' ? await vault.readBinary(file) : await vault.read(file);
fileData.push({blob: new Blob([fileContent], { type: mimeType }), path: file.path});
@@ -354,7 +357,7 @@ export function pasteTextAtCursor(text: string | undefined) {
}
}
export function getLinkToEntry(sourceFiles: TFile[], chosenFile: string, chosenEntry: string): string | undefined {
export function getFileFromPath(sourceFiles: TFile[], chosenFile: string): TFile | undefined {
// Find the vault file matching file of chosen file, entry
let fileMatch = sourceFiles
// Sort by descending length of path
@@ -363,6 +366,12 @@ export function getLinkToEntry(sourceFiles: TFile[], chosenFile: string, chosenE
// The first match is the best file match across OS
// e.g. Khoj server on Linux, Obsidian vault on Android
.find(file => chosenFile.replace(/\\/g, "/").endsWith(file.path))
return fileMatch;
}
export function getLinkToEntry(sourceFiles: TFile[], chosenFile: string, chosenEntry: string): string | undefined {
// Find the vault file matching file of chosen file, entry
let fileMatch = getFileFromPath(sourceFiles, chosenFile);
// Return link to vault file at heading of chosen search result
if (fileMatch) {

View File

@@ -85,6 +85,12 @@ If your plugin does not need CSS, delete this file.
margin-left: auto;
white-space: pre-line;
}
/* Override white-space for ul, ol, li under khoj-chat-message-text.khoj */
.khoj-chat-message-text.khoj ul,
.khoj-chat-message-text.khoj ol,
.khoj-chat-message-text.khoj li {
white-space: normal;
}
/* add left protrusion to khoj chat bubble */
.khoj-chat-message-text.khoj:after {
content: '';

View File

@@ -53,5 +53,6 @@
"1.13.0": "0.15.0",
"1.14.0": "0.15.0",
"1.15.0": "0.15.0",
"1.16.0": "0.15.0"
"1.16.0": "0.15.0",
"1.17.0": "0.15.0"
}