From fa7b40ab862b2495c7807acbf7769e4c98e41e14 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Fri, 21 Jun 2024 15:53:01 +0530 Subject: [PATCH] Automatically respond with Voice if subscribed user sent Voice message --- src/khoj/interface/web/chat.html | 112 ++++++++++++++++++------------- 1 file changed, 67 insertions(+), 45 deletions(-) diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html index 62c91f7d..72f1681c 100644 --- a/src/khoj/interface/web/chat.html +++ b/src/khoj/interface/web/chat.html @@ -87,6 +87,7 @@ To get started, just start typing below. You can also type / to see a list of co loadingEllipsis: null, references: {}, rawResponse: "", + isVoice: false, } fetch("https://ipapi.co/json") @@ -348,6 +349,57 @@ To get started, just start typing below. You can also type / to see a list of co .then(response => response.json()) } + function textToSpeech(message, event=null) { + // Replace the speaker with a loading icon. + let loader = document.createElement("span"); + loader.classList.add("loader"); + + let speechButton; + let speechIcon; + if (event === null) { + // Pick the last speech button if none is provided + let speechButtons = document.getElementsByClassName("speech-button"); + speechButton = speechButtons[speechButtons.length - 1]; + + let speechIcons = document.getElementsByClassName("speech-icon"); + speechIcon = speechIcons[speechIcons.length - 1]; + } else { + speechButton = event.currentTarget; + speechIcon = event.target; + } + + speechButton.innerHTML = ""; + speechButton.appendChild(loader); + speechButton.disabled = true; + + const context = new (window.AudioContext || window.webkitAudioContext)(); + fetch(`/api/chat/speech?text=${encodeURIComponent(message)}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + }) + .then(response => response.arrayBuffer()) + .then(arrayBuffer => context.decodeAudioData(arrayBuffer)) + .then(audioBuffer => { + const source = context.createBufferSource(); + source.buffer = audioBuffer; + source.connect(context.destination); + source.start(0); + source.onended = function() { + speechButton.innerHTML = ""; + speechButton.appendChild(speechIcon); + speechButton.disabled = false; + }; + }) + .catch(err => { + console.error("Error playing speech:", err); + speechButton.innerHTML = ""; + speechButton.appendChild(speechIcon); + speechButton.disabled = true; + }); + } + function formatHTMLMessage(message, raw=false, willReplace=true, userQuery) { var md = window.markdownit(); let newHTML = message; @@ -434,47 +486,9 @@ To get started, just start typing below. You can also type / to see a list of co speechIcon.src = "/static/assets/icons/speaker.svg"; speechIcon.classList.add("speech-icon"); speechButton.appendChild(speechIcon); - speechButton.addEventListener('click', function() { - // Replace the speaker with a loading icon. - let loader = document.createElement("span"); - loader.classList.add("loader"); - - speechButton.innerHTML = ""; - speechButton.appendChild(loader); - speechButton.disabled = true; - - const context = new (window.AudioContext || window.webkitAudioContext)(); - fetch(`/api/chat/speech?text=${encodeURIComponent(message)}`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json' - }, - }) - .then(response => response.arrayBuffer()) - .then(arrayBuffer => { - return context.decodeAudioData(arrayBuffer); - }) - .then(audioBuffer => { - const source = context.createBufferSource(); - source.buffer = audioBuffer; - source.connect(context.destination); - source.start(0); - source.onended = function() { - speechButton.innerHTML = ""; - speechButton.appendChild(speechIcon); - speechButton.disabled = false; - }; - }) - .catch(err => { - console.error("Error playing speech:", err); - speechButton.innerHTML = ""; - speechButton.appendChild(speechIcon); - speechButton.disabled = true; - }); - }); + speechButton.addEventListener('click', (event) => textToSpeech(message, event)); } - // Append buttons to parent element element.append(copyButton, thumbsDownButton, thumbsUpButton); @@ -584,9 +598,9 @@ To get started, just start typing below. You can also type / to see a list of co return referencesDiv; } - async function chat() { + async function chat(isVoice=false) { if (websocket) { - sendMessageViaWebSocket(); + sendMessageViaWebSocket(isVoice); return; } @@ -1050,7 +1064,7 @@ To get started, just start typing below. You can also type / to see a list of co window.onload = loadChat; - function setupWebSocket() { + function setupWebSocket(isVoice=false) { let chatBody = document.getElementById("chat-body"); let wsProtocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; let webSocketUrl = `${wsProtocol}//${window.location.host}/api/chat/ws`; @@ -1067,6 +1081,7 @@ To get started, just start typing below. You can also type / to see a list of co references: {}, rawResponse: "", rawQuery: "", + isVoice: isVoice, } if (chatBody.dataset.conversationId) { @@ -1080,8 +1095,13 @@ To get started, just start typing below. You can also type / to see a list of co let chunk = event.data; if (chunk == "start_llm_response") { console.log("Started streaming", new Date()); - } else if(chunk == "end_llm_response") { + } else if (chunk == "end_llm_response") { console.log("Stopped streaming", new Date()); + + // Automatically respond with voice if the subscribed user has sent voice message + if (websocketState.isVoice && "{{ is_active }}" == "True") + textToSpeech(websocketState.rawResponse); + // Append any references after all the data has been streamed finalizeChatBodyResponse(websocketState.references, websocketState.newResponseTextEl); @@ -1094,6 +1114,7 @@ To get started, just start typing below. You can also type / to see a list of co references: {}, rawResponse: "", rawQuery: liveQuery, + isVoice: false, } } else { try { @@ -1172,7 +1193,7 @@ To get started, just start typing below. You can also type / to see a list of co } } - function sendMessageViaWebSocket() { + function sendMessageViaWebSocket(isVoice=false) { let chatBody = document.getElementById("chat-body"); var query = document.getElementById("chat-input").value.trim(); @@ -1223,6 +1244,7 @@ To get started, just start typing below. You can also type / to see a list of co references, rawResponse, rawQuery: query, + isVoice: isVoice, } } var userMessages = []; @@ -1851,7 +1873,7 @@ To get started, just start typing below. You can also type / to see a list of co document.getElementById('countdown-circle').style.animation = "none"; // Send message - chat(); + chat(true); }, 3000); }) .catch(err => {