From afbeee9e82fbed74562e829423f320c810394553 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Wed, 26 Jun 2024 15:25:27 +0530 Subject: [PATCH 1/5] Rename copy-button to more general chat-action-button in Obsidian client - Use 4 space indent of activateView function in pane_view component --- src/interface/obsidian/src/chat_view.ts | 4 ++-- src/interface/obsidian/src/main.ts | 14 +++++++------- src/interface/obsidian/src/pane_view.ts | 14 +++++++------- src/interface/obsidian/styles.css | 14 +++++++------- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/interface/obsidian/src/chat_view.ts b/src/interface/obsidian/src/chat_view.ts index 4a4bf0ae..e4e9fd8d 100644 --- a/src/interface/obsidian/src/chat_view.ts +++ b/src/interface/obsidian/src/chat_view.ts @@ -461,7 +461,7 @@ export class KhojChatView extends KhojPaneView { renderActionButtons(message: string, chat_message_body_text_el: HTMLElement) { let copyButton = this.contentEl.createEl('button'); - copyButton.classList.add("copy-button"); + copyButton.classList.add("chat-action-button"); copyButton.title = "Copy Message to Clipboard"; setIcon(copyButton, "copy-plus"); copyButton.addEventListener('click', createCopyParentText(message)); @@ -469,7 +469,7 @@ export class KhojChatView extends KhojPaneView { // Add button to paste into current buffer let pasteToFile = this.contentEl.createEl('button'); - pasteToFile.classList.add("copy-button"); + pasteToFile.classList.add("chat-action-button"); pasteToFile.title = "Paste Message to File"; setIcon(pasteToFile, "clipboard-paste"); pasteToFile.addEventListener('click', (event) => { pasteTextAtCursor(createCopyParentText(message, 'clipboard-paste')(event)); }); diff --git a/src/interface/obsidian/src/main.ts b/src/interface/obsidian/src/main.ts index a97c1466..d7178f78 100644 --- a/src/interface/obsidian/src/main.ts +++ b/src/interface/obsidian/src/main.ts @@ -79,16 +79,16 @@ export default class Khoj extends Plugin { const leaves = workspace.getLeavesOfType(viewType); if (leaves.length > 0) { - // A leaf with our view already exists, use that - leaf = leaves[0]; + // A leaf with our view already exists, use that + leaf = leaves[0]; } else { - // Our view could not be found in the workspace, create a new leaf - // in the right sidebar for it - leaf = workspace.getRightLeaf(false); - await leaf?.setViewState({ type: viewType, active: true }); + // Our view could not be found in the workspace, create a new leaf + // in the right sidebar for it + leaf = workspace.getRightLeaf(false); + await leaf?.setViewState({ type: viewType, active: true }); } // "Reveal" the leaf in case it is in a collapsed sidebar if (leaf) workspace.revealLeaf(leaf); - } + } } diff --git a/src/interface/obsidian/src/pane_view.ts b/src/interface/obsidian/src/pane_view.ts index 3912ce4d..74afacab 100644 --- a/src/interface/obsidian/src/pane_view.ts +++ b/src/interface/obsidian/src/pane_view.ts @@ -38,16 +38,16 @@ export abstract class KhojPaneView extends ItemView { const leaves = workspace.getLeavesOfType(viewType); if (leaves.length > 0) { - // A leaf with our view already exists, use that - leaf = leaves[0]; + // A leaf with our view already exists, use that + leaf = leaves[0]; } else { - // Our view could not be found in the workspace, create a new leaf - // in the right sidebar for it - leaf = workspace.getRightLeaf(false); - await leaf?.setViewState({ type: viewType, active: true }); + // Our view could not be found in the workspace, create a new leaf + // in the right sidebar for it + leaf = workspace.getRightLeaf(false); + await leaf?.setViewState({ type: viewType, active: true }); } // "Reveal" the leaf in case it is in a collapsed sidebar if (leaf) workspace.revealLeaf(leaf); - } + } } diff --git a/src/interface/obsidian/styles.css b/src/interface/obsidian/styles.css index a79e0116..cc0bfb30 100644 --- a/src/interface/obsidian/styles.css +++ b/src/interface/obsidian/styles.css @@ -477,7 +477,7 @@ span.khoj-nav-item-text { } /* Copy button */ -button.copy-button { +button.chat-action-button { display: block; border-radius: 4px; color: var(--text-muted); @@ -491,22 +491,22 @@ button.copy-button { margin-top: 8px; float: right; } -button.copy-button span { +button.chat-action-button span { cursor: pointer; display: inline-block; position: relative; transition: 0.5s; } +button.chat-action-button:hover { + background-color: var(--background-modifier-active-hover); + color: var(--text-normal); +} + img.copy-icon { width: 16px; height: 16px; } -button.copy-button:hover { - background-color: var(--background-modifier-active-hover); - color: var(--text-normal); -} - /* Loading Spinner */ .lds-ellipsis { display: inline-block; From 093e276908071a71fc0d1b86ea1da52176fd803f Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Wed, 26 Jun 2024 15:29:09 +0530 Subject: [PATCH 2/5] Enable Voice chat in Khoj Obsidian plugin - Automatically carry out voice chats with Khoj from within Obsidian When send voice message, Khoj will auto respond with voice as well - Listen to past Khoj messages as speech - Add circular loading spinner to use while message is being converted to speech --- src/interface/obsidian/src/chat_view.ts | 95 ++++++++++++++++++++++--- src/interface/obsidian/styles.css | 34 +++++++++ 2 files changed, 119 insertions(+), 10 deletions(-) diff --git a/src/interface/obsidian/src/chat_view.ts b/src/interface/obsidian/src/chat_view.ts index e4e9fd8d..8e0d4d6d 100644 --- a/src/interface/obsidian/src/chat_view.ts +++ b/src/interface/obsidian/src/chat_view.ts @@ -61,7 +61,7 @@ export class KhojChatView extends KhojPaneView { return "message-circle"; } - async chat() { + async chat(isVoice: boolean = false) { // Get text in chat input element let input_el = this.contentEl.getElementsByClassName("khoj-chat-input")[0]; @@ -72,7 +72,7 @@ export class KhojChatView extends KhojPaneView { this.autoResize(); // Get and render chat response to user message - await this.getChatResponse(user_message); + await this.getChatResponse(user_message, isVoice); } async onOpen() { @@ -294,6 +294,60 @@ export class KhojChatView extends KhojPaneView { return referenceButton; } + textToSpeech(message: string, event: MouseEvent | null = null): void { + // Replace the speaker with a loading icon. + let loader = document.createElement("span"); + loader.classList.add("loader"); + + let speechButton: HTMLButtonElement; + let speechIcon: Element; + + if (event === null) { + // Pick the last speech button if none is provided + let speechButtons = document.getElementsByClassName("speech-button"); + speechButton = speechButtons[speechButtons.length - 1] as HTMLButtonElement; + + let speechIcons = document.getElementsByClassName("speech-icon"); + speechIcon = speechIcons[speechIcons.length - 1]; + } else { + speechButton = event.currentTarget as HTMLButtonElement; + speechIcon = event.target as Element; + } + + speechButton.innerHTML = ""; + speechButton.appendChild(loader); + speechButton.disabled = true; + + const context = new AudioContext(); + let textToSpeechApi = `${this.setting.khojUrl}/api/chat/speech?text=${encodeURIComponent(message)}`; + fetch(textToSpeechApi, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + "Authorization": `Bearer ${this.setting.khojApiKey}`, + }, + }) + .then(response => response.arrayBuffer()) + .then(arrayBuffer => context.decodeAudioData(arrayBuffer)) + .then(audioBuffer => { + const source = context.createBufferSource(); + source.buffer = audioBuffer; + source.connect(context.destination); + source.start(0); + source.onended = function() { + speechButton.innerHTML = ""; + speechButton.appendChild(speechIcon); + speechButton.disabled = false; + }; + }) + .catch(err => { + console.error("Error playing speech:", err); + speechButton.innerHTML = ""; + speechButton.appendChild(speechIcon); + speechButton.disabled = false; // Consider enabling the button again to allow retrying + }); + } + formatHTMLMessage(message: string, raw = false, willReplace = true) { // Remove any text between [INST] and tags. These are spurious instructions for some AI chat model. message = message.replace(/\[INST\].+(<\/s>)?/g, ''); @@ -465,7 +519,6 @@ export class KhojChatView extends KhojPaneView { copyButton.title = "Copy Message to Clipboard"; setIcon(copyButton, "copy-plus"); copyButton.addEventListener('click', createCopyParentText(message)); - chat_message_body_text_el.append(copyButton); // Add button to paste into current buffer let pasteToFile = this.contentEl.createEl('button'); @@ -473,7 +526,25 @@ export class KhojChatView extends KhojPaneView { pasteToFile.title = "Paste Message to File"; setIcon(pasteToFile, "clipboard-paste"); pasteToFile.addEventListener('click', (event) => { pasteTextAtCursor(createCopyParentText(message, 'clipboard-paste')(event)); }); - chat_message_body_text_el.append(pasteToFile); + + // Only enable the speech feature if the user is subscribed + let speechButton = null; + + if (this.setting.userInfo?.is_active) { + // Create a speech button icon to play the message out loud + speechButton = this.contentEl.createEl('button'); + speechButton.classList.add("chat-action-button", "speech-button"); + speechButton.title = "Listen to Message"; + setIcon(speechButton, "speech") + speechButton.addEventListener('click', (event) => this.textToSpeech(message, event)); + } + + // Append buttons to parent element + chat_message_body_text_el.append(copyButton, pasteToFile); + + if (speechButton) { + chat_message_body_text_el.append(speechButton); + } } formatDate(date: Date): string { @@ -727,7 +798,7 @@ export class KhojChatView extends KhojPaneView { return true; } - async readChatStream(response: Response, responseElement: HTMLDivElement): Promise { + async readChatStream(response: Response, responseElement: HTMLDivElement, isVoice: boolean = false): Promise { // Exit if response body is empty if (response.body == null) return; @@ -737,8 +808,12 @@ export class KhojChatView extends KhojPaneView { while (true) { const { value, done } = await reader.read(); - // Break if the stream is done - if (done) break; + if (done) { + // Automatically respond with voice if the subscribed user has sent voice message + if (isVoice && this.setting.userInfo?.is_active) this.textToSpeech(this.result); + // Break if the stream is done + break; + } let responseText = decoder.decode(value); if (responseText.includes("### compiled references:")) { @@ -756,7 +831,7 @@ export class KhojChatView extends KhojPaneView { } } - async getChatResponse(query: string | undefined | null): Promise { + async getChatResponse(query: string | undefined | null, isVoice: boolean = false): Promise { // Exit if query is empty if (!query || query === "") return; @@ -835,7 +910,7 @@ export class KhojChatView extends KhojPaneView { } } else { // Stream and render chat response - await this.readChatStream(response, responseElement); + await this.readChatStream(response, responseElement, isVoice); } } catch (err) { console.log(`Khoj chat response failed with\n${err}`); @@ -947,7 +1022,7 @@ export class KhojChatView extends KhojPaneView { sendImg.addEventListener('click', async (_) => { await this.chat() }); // Send message - this.chat(); + this.chat(true); }, 3000); }; diff --git a/src/interface/obsidian/styles.css b/src/interface/obsidian/styles.css index cc0bfb30..8e3d2c6b 100644 --- a/src/interface/obsidian/styles.css +++ b/src/interface/obsidian/styles.css @@ -507,6 +507,40 @@ img.copy-icon { height: 16px; } +/* Circular Loading Spinner */ +.loader { + width: 18px; + height: 18px; + border: 3px solid #FFF; + border-radius: 50%; + display: inline-block; + position: relative; + box-sizing: border-box; + animation: rotation 1s linear infinite; +} +.loader::after { + content: ''; + box-sizing: border-box; + position: absolute; + left: 50%; + top: 50%; + transform: translate(-50%, -50%); + width: 18px; + height: 18px; + border-radius: 50%; + border: 3px solid transparent; + border-bottom-color: var(--flower); +} + +@keyframes rotation { + 0% { + transform: rotate(0deg); + } + 100% { + transform: rotate(360deg); + } +} + /* Loading Spinner */ .lds-ellipsis { display: inline-block; From fbb95ca342803352c7db0f98b7934df86f99092d Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Wed, 26 Jun 2024 18:10:43 +0530 Subject: [PATCH 3/5] Put cursor on chat input when focus on chat view in Obsidian This should improve fluidity of keyboard interactions with Khoj on Obsidian. Open Khoj chat view via keybinding or command pallete and ask question using only the keyboard, with no mouse clicks required --- src/interface/obsidian/src/main.ts | 13 +++++++++++-- src/interface/obsidian/src/pane_view.ts | 12 ++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/interface/obsidian/src/main.ts b/src/interface/obsidian/src/main.ts index d7178f78..e0e09c24 100644 --- a/src/interface/obsidian/src/main.ts +++ b/src/interface/obsidian/src/main.ts @@ -88,7 +88,16 @@ export default class Khoj extends Plugin { await leaf?.setViewState({ type: viewType, active: true }); } - // "Reveal" the leaf in case it is in a collapsed sidebar - if (leaf) workspace.revealLeaf(leaf); + if (leaf) { + if (viewType === KhojView.CHAT) { + // focus on the chat input when the chat view is opened + let chatView = leaf.view as KhojChatView; + let chatInput = chatView.contentEl.getElementsByClassName("khoj-chat-input")[0]; + if (chatInput) chatInput.focus(); + } + + // "Reveal" the leaf in case it is in a collapsed sidebar + workspace.revealLeaf(leaf); + } } } diff --git a/src/interface/obsidian/src/pane_view.ts b/src/interface/obsidian/src/pane_view.ts index 74afacab..64a167dd 100644 --- a/src/interface/obsidian/src/pane_view.ts +++ b/src/interface/obsidian/src/pane_view.ts @@ -47,7 +47,15 @@ export abstract class KhojPaneView extends ItemView { await leaf?.setViewState({ type: viewType, active: true }); } - // "Reveal" the leaf in case it is in a collapsed sidebar - if (leaf) workspace.revealLeaf(leaf); + if (leaf) { + if (viewType === KhojView.CHAT) { + // focus on the chat input when the chat view is opened + let chatInput = this.contentEl.getElementsByClassName("khoj-chat-input")[0]; + if (chatInput) chatInput.focus(); + } + + // "Reveal" the leaf in case it is in a collapsed sidebar + workspace.revealLeaf(leaf); + } } } From 37239045127661d49d89bccd13e92931d1d78aa4 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Thu, 27 Jun 2024 08:40:24 +0530 Subject: [PATCH 4/5] Toggle jump between Khoj side pane & previous editor via cmd, kbd shortcut Improve quick navigation to, from Khoj side pane using Keyboard shortcut or Obsidian command --- src/interface/obsidian/src/main.ts | 24 +++++++++++++++--------- src/interface/obsidian/src/utils.ts | 6 ++++++ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/interface/obsidian/src/main.ts b/src/interface/obsidian/src/main.ts index e0e09c24..bf7cad54 100644 --- a/src/interface/obsidian/src/main.ts +++ b/src/interface/obsidian/src/main.ts @@ -2,7 +2,8 @@ import { Plugin, WorkspaceLeaf } from 'obsidian'; import { KhojSetting, KhojSettingTab, DEFAULT_SETTINGS } from 'src/settings' import { KhojSearchModal } from 'src/search_modal' import { KhojChatView } from 'src/chat_view' -import { updateContentIndex, canConnectToBackend, KhojView } from './utils'; +import { updateContentIndex, canConnectToBackend, KhojView, jumpToPreviousView } from './utils'; +import { KhojPaneView } from './pane_view'; export default class Khoj extends Plugin { @@ -89,15 +90,20 @@ export default class Khoj extends Plugin { } if (leaf) { - if (viewType === KhojView.CHAT) { - // focus on the chat input when the chat view is opened - let chatView = leaf.view as KhojChatView; - let chatInput = chatView.contentEl.getElementsByClassName("khoj-chat-input")[0]; - if (chatInput) chatInput.focus(); - } + const activeKhojLeaf = workspace.getActiveViewOfType(KhojPaneView)?.leaf; + // Jump to the previous view if the current view is Khoj Side Pane + if (activeKhojLeaf === leaf) jumpToPreviousView(); + // Else Reveal the leaf in case it is in a collapsed sidebar + else { + workspace.revealLeaf(leaf); - // "Reveal" the leaf in case it is in a collapsed sidebar - workspace.revealLeaf(leaf); + if (viewType === KhojView.CHAT) { + // focus on the chat input when the chat view is opened + let chatView = leaf.view as KhojChatView; + let chatInput = chatView.contentEl.getElementsByClassName("khoj-chat-input")[0]; + if (chatInput) chatInput.focus(); + } + } } } } diff --git a/src/interface/obsidian/src/utils.ts b/src/interface/obsidian/src/utils.ts index 5f3e5acb..4a969793 100644 --- a/src/interface/obsidian/src/utils.ts +++ b/src/interface/obsidian/src/utils.ts @@ -333,6 +333,12 @@ export function createCopyParentText(message: string, originalButton: string = ' } } +export function jumpToPreviousView() { + const editor: Editor = this.app.workspace.getActiveFileView()?.editor + if (!editor) return; + editor.focus(); +} + export function pasteTextAtCursor(text: string | undefined) { // Get the current active file's editor const editor: Editor = this.app.workspace.getActiveFileView()?.editor From cffc14a46abc68b132f7e9ff3ccdb1e71387db59 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Mon, 1 Jul 2024 18:06:09 +0530 Subject: [PATCH 5/5] Trigger voice chat via keyboard shortcut in Khoj side pane Quickly trigger voice chat from Khoj side pane using Keyboard shortcuts --- src/interface/obsidian/src/chat_view.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/interface/obsidian/src/chat_view.ts b/src/interface/obsidian/src/chat_view.ts index 8e0d4d6d..ca0c95e8 100644 --- a/src/interface/obsidian/src/chat_view.ts +++ b/src/interface/obsidian/src/chat_view.ts @@ -1,4 +1,4 @@ -import { ItemView, MarkdownRenderer, WorkspaceLeaf, request, requestUrl, setIcon } from 'obsidian'; +import { ItemView, MarkdownRenderer, Scope, WorkspaceLeaf, request, requestUrl, setIcon } from 'obsidian'; import * as DOMPurify from 'dompurify'; import { KhojSetting } from 'src/settings'; import { KhojPaneView } from 'src/pane_view'; @@ -28,6 +28,10 @@ export class KhojChatView extends KhojPaneView { constructor(leaf: WorkspaceLeaf, setting: KhojSetting) { super(leaf, setting); + // Register Modal Keybindings to send voice message + this.scope = new Scope(this.app.scope); + this.scope.register(["Mod"], 's', async (event) => { await this.speechToText(event); }); + this.waitingForLocation = true; fetch("https://ipapi.co/json") @@ -958,7 +962,7 @@ export class KhojChatView extends KhojPaneView { sendMessageTimeout: NodeJS.Timeout | undefined; mediaRecorder: MediaRecorder | undefined; - async speechToText(event: MouseEvent | TouchEvent) { + async speechToText(event: MouseEvent | TouchEvent | KeyboardEvent) { event.preventDefault(); const transcribeButton = this.contentEl.getElementsByClassName("khoj-transcribe")[0]; const chatInput = this.contentEl.getElementsByClassName("khoj-chat-input")[0];