Render Chat Responses as Markdown in Desktop, Obsidian Client (#571)

- Show temporary status message when copied to clipboard
- Render chat responses as markdown in Desktop client
- Render chat responses as markdown in chat modal of Obsidian client
- Render references of new responses in chat modal on Obsidian client. Use new style for references
- Properly stop `mediaRecorder` stream to clear microphone in-use state
- Render newlines when references expanded in Web, Desktop and Obsidian clients

src/interface/desktop/chat.html

@@ -8,17 +8,26 @@
         <link rel="manifest" href="/static/khoj_chat.webmanifest">
         <link rel="stylesheet" href="./assets/khoj.css">
     </head>
+    <script type="text/javascript" src="./assets/markdown-it.min.js"></script>
     <script src="./utils.js"></script>
 
     <script>
         let chatOptions = [];
         function copyProgrammaticOutput(event) {
             // Remove the first 4 characters which are the "Copy" button
+            const originalCopyText = event.target.parentNode.textContent.trim().slice(0, 4);
             const programmaticOutput = event.target.parentNode.textContent.trim().slice(4);
             navigator.clipboard.writeText(programmaticOutput).then(() => {
-                console.log("Programmatic output copied to clipboard");
+                event.target.textContent = "✅ Copied to clipboard!";
+                setTimeout(() => {
+                    event.target.textContent = originalCopyText;
+                }, 1000);
             }).catch((error) => {
                 console.error("Error copying programmatic output to clipboard:", error);
+                event.target.textContent = "⛔️ Failed to copy!";
+                setTimeout(() => {
+                    event.target.textContent = originalCopyText;
+                }, 1000);
             });
         }
 
@@ -120,8 +129,7 @@
             // Create a new div for the chat message text and append it to the chat message
             let chatMessageText = document.createElement('div');
             chatMessageText.className = `chat-message-text ${by}`;
-            let textNode = document.createTextNode(formattedMessage);
-            chatMessageText.appendChild(textNode);
+            chatMessageText.appendChild(formattedMessage);
             chatMessage.appendChild(chatMessageText);
 
             // Append annotations div to the chat message
@@ -230,16 +238,49 @@
         }
 
         function formatHTMLMessage(htmlMessage) {
-            // Replace any ``` with <div class="programmatic-output">
-            let newHTML = htmlMessage.replace(/```([\s\S]*?)```/g, '<div class="programmatic-output"><button class="copy-button" onclick="copyProgrammaticOutput(event)">Copy</button>$1</div>');
-            // Replace any ** with <b> and __ with <u>
-            newHTML = newHTML.replace(/\*\*([\s\S]*?)\*\*/g, '<b>$1</b>');
-            newHTML = newHTML.replace(/__([\s\S]*?)__/g, '<u>$1</u>');
+            var md = window.markdownit();
+            let newHTML = htmlMessage;
+
             // Remove any text between <s>[INST] and </s> tags. These are spurious instructions for the AI chat model.
             newHTML = newHTML.replace(/<s>\[INST\].+(<\/s>)?/g, '');
-            // For any text that has single backticks, replace them with <code> tags
-            newHTML = newHTML.replace(/`([^`]+)`/g, '<code class="chat-response">$1</code>');
-            return newHTML;
+
+            // Render markdown
+            newHTML = md.render(newHTML);
+            // Get any elements with a class that starts with "language"
+            let element = document.createElement('div');
+            element.innerHTML = newHTML;
+            let codeBlockElements = element.querySelectorAll('[class^="language-"]');
+            // For each element, add a parent div with the class "programmatic-output"
+            codeBlockElements.forEach((codeElement) => {
+                // Create the parent div
+                let parentDiv = document.createElement('div');
+                parentDiv.classList.add("programmatic-output");
+                // Add the parent div before the code element
+                codeElement.parentNode.insertBefore(parentDiv, codeElement);
+                // Move the code element into the parent div
+                parentDiv.appendChild(codeElement);
+                // Add a copy button to each element
+                let copyButton = document.createElement('button');
+                copyButton.classList.add("copy-button");
+                copyButton.innerHTML = "Copy";
+                copyButton.addEventListener('click', copyProgrammaticOutput);
+                codeElement.prepend(copyButton);
+            });
+
+            // Get all code elements that have no class.
+            let codeElements = element.querySelectorAll('code:not([class])');
+            codeElements.forEach((codeElement) => {
+                // Add the class "chat-response" to each element
+                codeElement.classList.add("chat-response");
+            });
+
+            let anchorElements = element.querySelectorAll('a');
+            anchorElements.forEach((anchorElement) => {
+                // Add the class "inline-chat-link" to each element
+                anchorElement.classList.add("inline-chat-link");
+            });
+
+            return element
         }
 
         async function chat() {
@@ -299,8 +340,11 @@
                         reader.read().then(({ done, value }) => {
                             if (done) {
                                 // Append any references after all the data has been streamed
-                                newResponseText.appendChild(references);
+                                if (references != null) {
+                                    newResponseText.appendChild(references);
+                                }
                                 document.getElementById("chat-body").scrollTop = document.getElementById("chat-body").scrollHeight;
+                                document.getElementById("chat-input").removeAttribute("disabled");
                                 return;
                             }
 
@@ -321,7 +365,6 @@
                                 let referenceExpandButton = document.createElement('button');
                                 referenceExpandButton.classList.add("reference-expand-button");
 
-
                                 let referenceSection = document.createElement('div');
                                 referenceSection.classList.add("reference-section");
                                 referenceSection.classList.add("collapsed");
@@ -387,7 +430,6 @@
                         });
                     }
                     readStream();
-                    document.getElementById("chat-input").removeAttribute("disabled");
                 });
         }
 
@@ -618,6 +660,8 @@
                 });
             } else if (mediaRecorder.state === 'recording') {
                 mediaRecorder.stop();
+                mediaRecorder.stream.getTracks().forEach(track => track.stop());
+                mediaRecorder = null;
                 speakButtonImg.src = './assets/icons/microphone-solid.svg';
                 speakButtonImg.alt = 'Transcribe';
             }
@@ -901,6 +945,7 @@
         }
         button.reference-button.expanded {
             max-height: none;
+            white-space: pre-wrap;
         }
 
         button.reference-button::before {
@@ -933,7 +978,6 @@
             background: var(--primary-hover);
         }
 
-
         .option-enabled:focus {
             outline: none !important;
             border:1px solid #475569;
@@ -946,6 +990,25 @@
             border-bottom: 1px dotted #475569;
         }
 
+        a.reference-link {
+            color: var(--main-text-color);
+            border-bottom: 1px dotted var(--main-text-color);
+        }
+
+        button.copy-button {
+            display: block;
+            border-radius: 4px;
+            background-color: var(--background-color);
+        }
+        button.copy-button:hover {
+            background: #f5f5f5;
+            cursor: pointer;
+        }
+
+        pre {
+            text-wrap: unset;
+        }
+
         div.khoj-empty-container {
             padding: 0;
             margin: 0;
@@ -1019,6 +1082,10 @@
             text-align: center;
         }
 
+        p {
+            margin: 0;
+        }
+
         div.programmatic-output {
             background-color: #f5f5f5;
             border: 1px solid #ddd;

src/interface/obsidian/src/chat_modal.ts

@@ -1,4 +1,4 @@
-import { App, Modal, RequestUrlParam, request, requestUrl, setIcon } from 'obsidian';
+import { App, MarkdownRenderer, Modal, request, requestUrl, setIcon } from 'obsidian';
 import { KhojSetting } from 'src/settings';
 import fetch from "node-fetch";
 
@@ -32,10 +32,10 @@ export class KhojChatModal extends Modal {
         contentEl.createEl("h1", ({ attr: { id: "khoj-chat-title" }, text: "Khoj Chat" }));
 
         // Create area for chat logs
-        contentEl.createDiv({ attr: { id: "khoj-chat-body", class: "khoj-chat-body" } });
+        let chatBodyEl = contentEl.createDiv({ attr: { id: "khoj-chat-body", class: "khoj-chat-body" } });
 
         // Get chat history from Khoj backend
-        await this.getChatHistory();
+        await this.getChatHistory(chatBodyEl);
 
         // Add chat input field
         let inputRow = contentEl.createDiv("khoj-input-row");
@@ -49,7 +49,6 @@ export class KhojChatModal extends Modal {
                     class: "khoj-chat-input option"
                 }
             })
-        chatInput.addEventListener('change', (event) => { this.result = (<HTMLInputElement>event.target).value });
 
         let transcribe = inputRow.createEl("button", {
             text: "Transcribe",
@@ -75,52 +74,108 @@ export class KhojChatModal extends Modal {
         chatInput.focus();
     }
 
-    generateReference(messageEl: any, reference: string, index: number) {
-        // Generate HTML for Chat Reference
-        // `<sup><abbr title="${escaped_ref}" tabindex="0">${index}</abbr></sup>`;
+    generateReference(messageEl: Element, reference: string, index: number) {
+        // Escape reference for HTML rendering
         let escaped_ref = reference.replace(/"/g, "&quot;")
-        return messageEl.createEl("sup").createEl("abbr", {
-            attr: {
-                title: escaped_ref,
-                tabindex: "0",
-            },
-            text: `[${index}] `,
+
+        // Generate HTML for Chat Reference
+        let short_ref = escaped_ref.slice(0, 100);
+        short_ref = short_ref.length < escaped_ref.length ? short_ref + "..." : short_ref;
+        let referenceButton = messageEl.createEl('button');
+        referenceButton.innerHTML = short_ref;
+        referenceButton.id = `ref-${index}`;
+        referenceButton.classList.add("reference-button");
+        referenceButton.classList.add("collapsed");
+        referenceButton.tabIndex = 0;
+
+        // Add event listener to toggle full reference on click
+        referenceButton.addEventListener('click', function() {
+            console.log(`Toggling ref-${index}`)
+            if (this.classList.contains("collapsed")) {
+                this.classList.remove("collapsed");
+                this.classList.add("expanded");
+                this.innerHTML = escaped_ref;
+            } else {
+                this.classList.add("collapsed");
+                this.classList.remove("expanded");
+                this.innerHTML = short_ref;
+            }
         });
+
+        return referenceButton;
     }
 
-    renderMessageWithReferences(message: string, sender: string, context?: [string], dt?: Date) {
-        let messageEl = this.renderMessage(message, sender, dt);
-        if (context && !!messageEl) {
-            context.map((reference, index) => this.generateReference(messageEl, reference, index + 1));
+    renderMessageWithReferences(chatEl: Element, message: string, sender: string, context?: string[], dt?: Date) {
+        if (!message) {
+            return;
+        } else if (!context) {
+            this.renderMessage(chatEl, message, sender, dt);
+            return
+        } else if (!!context && context?.length === 0) {
+            this.renderMessage(chatEl, message, sender, dt);
+            return
         }
+        let chatMessageEl = this.renderMessage(chatEl, message, sender, dt);
+        let chatMessageBodyEl = chatMessageEl.getElementsByClassName("khoj-chat-message-text")[0]
+        let references = chatMessageBodyEl.createDiv();
+
+        let referenceExpandButton = references.createEl('button');
+        referenceExpandButton.classList.add("reference-expand-button");
+        let numReferences = 0;
+
+        if (context) {
+            numReferences += context.length;
+        }
+
+        let referenceSection = references.createEl('div');
+        referenceSection.classList.add("reference-section");
+        referenceSection.classList.add("collapsed");
+
+        referenceExpandButton.addEventListener('click', function() {
+            if (referenceSection.classList.contains("collapsed")) {
+                referenceSection.classList.remove("collapsed");
+                referenceSection.classList.add("expanded");
+            } else {
+                referenceSection.classList.add("collapsed");
+                referenceSection.classList.remove("expanded");
+            }
+        });
+
+        references.classList.add("references");
+        if (context) {
+            context.map((reference, index) => {
+                this.generateReference(referenceSection, reference, index + 1);
+            });
+        }
+
+        let expandButtonText = numReferences == 1 ? "1 reference" : `${numReferences} references`;
+        referenceExpandButton.innerHTML = expandButtonText;
     }
 
-    renderMessage(message: string, sender: string, dt?: Date): Element | null {
+    renderMessage(chatEl: Element, message: string, sender: string, dt?: Date): Element {
         let message_time = this.formatDate(dt ?? new Date());
         let emojified_sender = sender == "khoj" ? "🏮 Khoj" : "🤔 You";
 
         // Append message to conversation history HTML element.
         // The chat logs should display above the message input box to follow standard UI semantics
-        let chat_body_el = this.contentEl.getElementsByClassName("khoj-chat-body")[0];
-        let chat_message_el = chat_body_el.createDiv({
+        let chatMessageEl = chatEl.createDiv({
             attr: {
                 "data-meta": `${emojified_sender} at ${message_time}`,
                 class: `khoj-chat-message ${sender}`
             },
-        }).createDiv({
-            attr: {
-                class: `khoj-chat-message-text ${sender}`
-            },
-            text: `${message}`
         })
+        let chat_message_body_el = chatMessageEl.createDiv();
+        chat_message_body_el.addClasses(["khoj-chat-message-text", sender]);
+        let chat_message_body_text_el = chat_message_body_el.createDiv();
+        MarkdownRenderer.renderMarkdown(message, chat_message_body_text_el, '', null);
 
         // Remove user-select: none property to make text selectable
-        chat_message_el.style.userSelect = "text";
+        chatMessageEl.style.userSelect = "text";
 
         // Scroll to bottom after inserting chat messages
         this.modalEl.scrollTop = this.modalEl.scrollHeight;
 
-        return chat_message_el
+        return chatMessageEl
     }
 
     createKhojResponseDiv(dt?: Date): HTMLDivElement {
@@ -147,7 +202,9 @@ export class KhojChatModal extends Modal {
     }
 
     renderIncrementalMessage(htmlElement: HTMLDivElement, additionalMessage: string) {
-        htmlElement.innerHTML += additionalMessage;
+        this.result += additionalMessage;
+        htmlElement.innerHTML = "";
+        MarkdownRenderer.renderMarkdown(this.result, htmlElement, '', null);
         // Scroll to bottom of modal, till the send message input box
         this.modalEl.scrollTop = this.modalEl.scrollHeight;
     }
@@ -159,14 +216,14 @@ export class KhojChatModal extends Modal {
         return `${time_string}, ${date_string}`;
     }
 
-    async getChatHistory(): Promise<void> {
+    async getChatHistory(chatBodyEl: Element): Promise<void> {
         // Get chat history from Khoj backend
         let chatUrl = `${this.setting.khojUrl}/api/chat/history?client=obsidian`;
         let headers = { "Authorization": `Bearer ${this.setting.khojApiKey}` };
         let response = await request({ url: chatUrl, headers: headers });
         let chatLogs = JSON.parse(response).response;
         chatLogs.forEach((chatLog: any) => {
-            this.renderMessageWithReferences(chatLog.message, chatLog.by, chatLog.context, new Date(chatLog.created));
+            this.renderMessageWithReferences(chatBodyEl, chatLog.message, chatLog.by, chatLog.context, new Date(chatLog.created));
         });
     }
 
@@ -175,7 +232,8 @@ export class KhojChatModal extends Modal {
         if (!query || query === "") return;
 
         // Render user query as chat message
-        this.renderMessage(query, "you");
+        let chatBodyEl = this.contentEl.getElementsByClassName("khoj-chat-body")[0];
+        this.renderMessage(chatBodyEl, query, "you");
 
         // Get chat response from Khoj backend
         let encodedQuery = encodeURIComponent(query);
@@ -203,12 +261,54 @@ export class KhojChatModal extends Modal {
                 responseElement.innerHTML = "";
             }
 
+            this.result = "";
+            responseElement.innerHTML = "";
             for await (const chunk of response.body) {
                 const responseText = chunk.toString();
-                if (responseText.startsWith("### compiled references:")) {
-                    return;
+                if (responseText.includes("### compiled references:")) {
+                    const additionalResponse = responseText.split("### compiled references:")[0];
+                    this.renderIncrementalMessage(responseElement, additionalResponse);
+
+                    const rawReference = responseText.split("### compiled references:")[1];
+                    const rawReferenceAsJson = JSON.parse(rawReference);
+                    let references = responseElement.createDiv();
+                    references.classList.add("references");
+
+                    let referenceExpandButton = references.createEl('button');
+                    referenceExpandButton.classList.add("reference-expand-button");
+
+                    let referenceSection = references.createDiv();
+                    referenceSection.classList.add("reference-section");
+                    referenceSection.classList.add("collapsed");
+
+                    let numReferences = 0;
+
+                    // If rawReferenceAsJson is a list, then count the length
+                    if (Array.isArray(rawReferenceAsJson)) {
+                        numReferences = rawReferenceAsJson.length;
+
+                        rawReferenceAsJson.forEach((reference, index) => {
+                            this.generateReference(referenceSection, reference, index);
+                        });
+                    }
+                    references.appendChild(referenceExpandButton);
+
+                    referenceExpandButton.addEventListener('click', function() {
+                        if (referenceSection.classList.contains("collapsed")) {
+                            referenceSection.classList.remove("collapsed");
+                            referenceSection.classList.add("expanded");
+                        } else {
+                            referenceSection.classList.add("collapsed");
+                            referenceSection.classList.remove("expanded");
+                        }
+                    });
+
+                    let expandButtonText = numReferences == 1 ? "1 reference" : `${numReferences} references`;
+                    referenceExpandButton.innerHTML = expandButtonText;
+                    references.appendChild(referenceSection);
+                } else {
+                    this.renderIncrementalMessage(responseElement, responseText);
                 }
-                this.renderIncrementalMessage(responseElement, responseText);
             }
         } catch (err) {
             this.renderIncrementalMessage(responseElement, "Sorry, unable to get response from Khoj backend ❤️‍🩹. Contact developer for help at team@khoj.dev or <a href='https://discord.gg/BDgyabRM6e'>in Discord</a>")
@@ -243,7 +343,7 @@ export class KhojChatModal extends Modal {
             } else {
                 // If conversation history is cleared successfully, clear chat logs from modal
                 chatBody.innerHTML = "";
-                await this.getChatHistory();
+                await this.getChatHistory(chatBody);
                 this.flashStatusInChatInput(result.message);
             }
         } catch (err) {
@@ -321,6 +421,8 @@ export class KhojChatModal extends Modal {
                 });
         } else if (this.mediaRecorder.state === 'recording') {
             this.mediaRecorder.stop();
+            this.mediaRecorder.stream.getTracks().forEach(track => track.stop());
+            this.mediaRecorder = undefined;
             setIcon(transcribeButton, "mic");
         }
     }

src/interface/obsidian/styles.css

@@ -13,6 +13,13 @@ If your plugin does not need CSS, delete this file.
     --khoj-storm-grey: #475569;
  }
 
+.khoj-chat p {
+    margin: 0;
+}
+.khoj-chat pre {
+    text-wrap: unset;
+}
+
 .khoj-chat {
     display: grid;
     background: var(--background-primary);
@@ -104,6 +111,114 @@ If your plugin does not need CSS, delete this file.
     transform: rotate(-60deg)
 }
 
+.option-enabled {
+    box-shadow: 0 0 12px rgb(119, 156, 46);
+}
+
+code.chat-response {
+    background: var(--khoj-sun);
+    color: var(--khoj-storm-grey);
+    border-radius: 5px;
+    padding: 5px;
+    font-size: 14px;
+    font-weight: 300;
+    line-height: 1.5em;
+}
+
+div.collapsed {
+    display: none;
+}
+div.expanded {
+    display: block;
+}
+div.reference {
+    display: grid;
+    grid-template-rows: auto;
+    grid-auto-flow: row;
+    grid-column-gap: 10px;
+    grid-row-gap: 10px;
+    margin: 10px;
+}
+div.expanded.reference-section {
+    display: grid;
+    grid-template-rows: auto;
+    grid-auto-flow: row;
+    grid-column-gap: 10px;
+    grid-row-gap: 10px;
+    margin: 10px;
+}
+button.reference-button {
+    background: var(--color-base-00);
+    color: var(--color-base-100);
+    border: 1px solid var(--khoj-storm-grey);
+    border-radius: 5px;
+    padding: 5px;
+    font-size: 14px;
+    font-weight: 300;
+    line-height: 1.5em;
+    cursor: pointer;
+    transition: background 0.2s ease-in-out;
+    text-align: left;
+    max-height: 75px;
+    height: auto;
+    transition: max-height 0.3s ease-in-out;
+    overflow: hidden;
+    display: inline-block;
+    text-wrap: inherit;
+}
+button.reference-button.expanded {
+    height: auto;
+    max-height: none;
+    white-space: pre-wrap;
+}
+button.reference-button::before {
+    content: "▶";
+    margin-right: 5px;
+    display: inline-block;
+    transition: transform 0.3s ease-in-out;
+}
+button.reference-button:active:before,
+button.reference-button[aria-expanded="true"]::before {
+    transform: rotate(90deg);
+}
+button.reference-expand-button {
+    background: var(--color-base-00);
+    color: var(--color-base-100);
+    border: 1px solid var(--khoj-storm-grey);
+    border-radius: 5px;
+    padding: 5px;
+    font-size: 14px;
+    font-weight: 300;
+    line-height: 1.5em;
+    cursor: pointer;
+    transition: background 0.2s ease-in-out;
+    text-align: left;
+}
+button.reference-expand-button:hover {
+    background: var(--khoj-sun);
+    color: var(--color-base-00);
+}
+a.inline-chat-link {
+    color: #475569;
+    text-decoration: none;
+    border-bottom: 1px dotted #475569;
+}
+a.reference-link {
+    color: var(--khoj-storm-grey);
+    border-bottom: 1px dotted var(--khoj-storm-grey);
+}
+
+button.copy-button {
+    display: block;
+    border-radius: 4px;
+    background-color: var(--color-base-00);
+}
+button.copy-button:hover {
+    background: #f5f5f5;
+    cursor: pointer;
+}
+
+
 #khoj-chat-footer {
     padding: 0;
     display: grid;

src/khoj/interface/web/chat.html

@@ -24,11 +24,19 @@ To get started, just start typing below. You can also type / to see a list of co
         let chatOptions = [];
         function copyProgrammaticOutput(event) {
             // Remove the first 4 characters which are the "Copy" button
+            const originalCopyText = event.target.parentNode.textContent.trim().slice(0, 4);
             const programmaticOutput = event.target.parentNode.textContent.trim().slice(4);
             navigator.clipboard.writeText(programmaticOutput).then(() => {
-                console.log("Programmatic output copied to clipboard");
+                event.target.textContent = "✅ Copied to clipboard!";
+                setTimeout(() => {
+                    event.target.textContent = originalCopyText;
+                }, 1000);
             }).catch((error) => {
                 console.error("Error copying programmatic output to clipboard:", error);
+                event.target.textContent = "⛔️ Failed to copy!";
+                setTimeout(() => {
+                    event.target.textContent = originalCopyText;
+                }, 1000);
             });
         }
 
@@ -407,11 +415,14 @@ To get started, just start typing below. You can also type / to see a list of co
                                     try {
                                         const responseAsJson = JSON.parse(chunk);
                                         if (responseAsJson.detail) {
-                                            newResponseText.innerHTML += responseAsJson.detail;
+                                            rawResponse += responseAsJson.detail;
                                         }
                                     } catch (error) {
                                         // If the chunk is not a JSON object, just display it as is
-                                        newResponseText.innerHTML += chunk;
+                                        rawResponse += chunk;
+                                    } finally {
+                                        newResponseText.innerHTML = "";
+                                        newResponseText.appendChild(formatHTMLMessage(rawResponse));
                                     }
                                 } else {
                                     // If the chunk is not a JSON object, just display it as is
@@ -635,6 +646,8 @@ To get started, just start typing below. You can also type / to see a list of co
                 });
             } else if (mediaRecorder.state === 'recording') {
                 mediaRecorder.stop();
+                mediaRecorder.stream.getTracks().forEach(track => track.stop());
+                mediaRecorder = null;
                 speakButtonImg.src = '/static/assets/icons/microphone-solid.svg';
                 speakButtonImg.alt = 'Transcribe';
             }
@@ -762,6 +775,7 @@ To get started, just start typing below. You can also type / to see a list of co
         }
         button.reference-button.expanded {
             max-height: none;
+            white-space: pre-wrap;
         }
 
         button.reference-button::before {
@@ -970,12 +984,15 @@ To get started, just start typing below. You can also type / to see a list of co
             border-radius: 4px;
             background-color: var(--background-color);
         }
-
         button.copy-button:hover {
             background: #f5f5f5;
             cursor: pointer;
         }
 
+        pre {
+            text-wrap: unset;
+        }
+
         @media (pointer: coarse), (hover: none) {
             abbr[title] {
                 position: relative;