From 9b61f0b5f7d7b39b07b6115303d741ea2ba646a1 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Sun, 25 Feb 2024 00:11:11 +0530 Subject: [PATCH 1/3] Fix rendering images from Khoj response in Obsidian client --- src/interface/obsidian/src/chat_modal.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/interface/obsidian/src/chat_modal.ts b/src/interface/obsidian/src/chat_modal.ts index dbdfa667..d7688d35 100644 --- a/src/interface/obsidian/src/chat_modal.ts +++ b/src/interface/obsidian/src/chat_modal.ts @@ -398,7 +398,7 @@ export class KhojChatModal extends Modal { // Reset collated chat result to empty string this.result = ""; responseElement.innerHTML = ""; - if (response.headers.get("content-type") == "application/json") { + if (response.headers.get("content-type") === "application/json") { let responseText = "" try { const responseAsJson = await response.json() as ChatJsonResult; @@ -413,10 +413,10 @@ export class KhojChatModal extends Modal { } finally { await this.renderIncrementalMessage(responseElement, responseText); } + } else { + // Stream and render chat response + await this.readChatStream(response, responseElement); } - - // Stream and render chat response - await this.readChatStream(response, responseElement); } catch (err) { console.log(`Khoj chat response failed with\n${err}`); let errorMsg = "Sorry, unable to get response from Khoj backend ❤️‍🩹. Retry or contact developers for help at team@khoj.dev or on Discord"; From a2e53d5e4116f1170f9af487e3fc531fe8fb573b Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Sun, 25 Feb 2024 00:24:58 +0530 Subject: [PATCH 2/3] Add inferred queries to image chat responses in Obsidian client --- src/interface/obsidian/src/chat_modal.ts | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/interface/obsidian/src/chat_modal.ts b/src/interface/obsidian/src/chat_modal.ts index d7688d35..765a43a8 100644 --- a/src/interface/obsidian/src/chat_modal.ts +++ b/src/interface/obsidian/src/chat_modal.ts @@ -4,6 +4,8 @@ import { KhojSetting } from 'src/settings'; export interface ChatJsonResult { image?: string; detail?: string; + intentType?: string; + inferredQueries?: string[]; } @@ -145,11 +147,17 @@ export class KhojChatModal extends Modal { return referenceButton; } - renderMessageWithReferences(chatEl: Element, message: string, sender: string, context?: string[], dt?: Date, intentType?: string) { + renderMessageWithReferences(chatEl: Element, message: string, sender: string, context?: string[], dt?: Date, intentType?: string, inferredQueries?: string) { if (!message) { return; } else if (intentType === "text-to-image") { let imageMarkdown = `![](data:image/png;base64,${message})`; + if (inferredQueries) { + imageMarkdown += "\n\n**Inferred Query**:"; + for (let inferredQuery of inferredQueries) { + imageMarkdown += `\n\n${inferredQuery}`; + } + } this.renderMessage(chatEl, imageMarkdown, sender, dt); return; } else if (!context) { @@ -287,7 +295,15 @@ export class KhojChatModal extends Modal { } else if (responseJson.response) { let chatLogs = responseJson.response?.conversation_id ? responseJson.response.chat ?? [] : responseJson.response; chatLogs.forEach((chatLog: any) => { - this.renderMessageWithReferences(chatBodyEl, chatLog.message, chatLog.by, chatLog.context, new Date(chatLog.created), chatLog.intent?.type); + this.renderMessageWithReferences( + chatBodyEl, + chatLog.message, + chatLog.by, + chatLog.context, + new Date(chatLog.created), + chatLog.intent?.type, + chatLog.intent?.["inferred-queries"], + ); }); } } catch (err) { @@ -404,6 +420,10 @@ export class KhojChatModal extends Modal { const responseAsJson = await response.json() as ChatJsonResult; if (responseAsJson.image) { responseText = `![${query}](data:image/png;base64,${responseAsJson.image})`; + const inferredQuery = responseAsJson.inferredQueries?.[0]; + if (inferredQuery) { + responseText += `\n\n**Inferred Query**:\n\n${inferredQuery}`; + } } else if (responseAsJson.detail) { responseText = responseAsJson.detail; } From f84606325ce7294cd5489c17f4948b29b36dffd6 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Sun, 25 Feb 2024 00:39:58 +0530 Subject: [PATCH 3/3] Improve render of inferred query in image chat messages in Web, Desktop apps --- src/interface/desktop/chat.html | 17 +++++++---------- src/khoj/interface/web/chat.html | 14 ++++++-------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/src/interface/desktop/chat.html b/src/interface/desktop/chat.html index b3083168..3d281dfb 100644 --- a/src/interface/desktop/chat.html +++ b/src/interface/desktop/chat.html @@ -199,10 +199,9 @@ function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null, intentType=null, inferredQueries=null) { if (intentType === "text-to-image") { let imageMarkdown = `![](data:image/png;base64,${message})`; - imageMarkdown += "\n\n"; - if (inferredQueries) { - const inferredQuery = inferredQueries?.[0]; - imageMarkdown += `**Inferred Query**: ${inferredQuery}`; + const inferredQuery = inferredQueries?.[0]; + if (inferredQuery) { + imageMarkdown += `\n\n**Inferred Query**:\n\n${inferredQuery}`; } renderMessage(imageMarkdown, by, dt); return; @@ -392,10 +391,9 @@ if (responseAsJson.image) { // If response has image field, response is a generated image. rawResponse += `![${query}](data:image/png;base64,${responseAsJson.image})`; - rawResponse += "\n\n"; const inferredQueries = responseAsJson.inferredQueries?.[0]; if (inferredQueries) { - rawResponse += `**Inferred Query**: ${inferredQueries}`; + rawResponse += `\n\n**Inferred Query**:\n\n${inferredQueries}`; } } if (responseAsJson.detail) { @@ -496,10 +494,9 @@ if (responseAsJson.image) { // If response has image field, response is a generated image. rawResponse += `![${query}](data:image/png;base64,${responseAsJson.image})`; - rawResponse += "\n\n"; - const inferredQueries = responseAsJson.inferredQueries?.[0]; - if (inferredQueries) { - rawResponse += `**Inferred Query**: ${inferredQueries}`; + const inferredQuery = responseAsJson.inferredQueries?.[0]; + if (inferredQuery) { + rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`; } } if (responseAsJson.detail) { diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html index a2229376..af4f498e 100644 --- a/src/khoj/interface/web/chat.html +++ b/src/khoj/interface/web/chat.html @@ -211,10 +211,9 @@ To get started, just start typing below. You can also type / to see a list of co function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null, intentType=null, inferredQueries=null) { if (intentType === "text-to-image") { let imageMarkdown = `![](data:image/png;base64,${message})`; - imageMarkdown += "\n\n"; - if (inferredQueries) { - const inferredQuery = inferredQueries?.[0]; - imageMarkdown += `**Inferred Query**: ${inferredQuery}`; + const inferredQuery = inferredQueries?.[0]; + if (inferredQuery) { + imageMarkdown += `\n\n**Inferred Query**:\n\n${inferredQuery}`; } renderMessage(imageMarkdown, by, dt); return; @@ -400,10 +399,9 @@ To get started, just start typing below. You can also type / to see a list of co if (responseAsJson.image) { // If response has image field, response is a generated image. rawResponse += `![${query}](data:image/png;base64,${responseAsJson.image})`; - rawResponse += "\n\n"; - const inferredQueries = responseAsJson.inferredQueries?.[0]; - if (inferredQueries) { - rawResponse += `**Inferred Query**: ${inferredQueries}`; + const inferredQuery = responseAsJson.inferredQueries?.[0]; + if (inferredQuery) { + rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`; } } if (responseAsJson.detail) {