From cd85a519806223afacd5366f9e5c8fa6e3900a58 Mon Sep 17 00:00:00 2001 From: sabaimran Date: Thu, 1 Aug 2024 12:50:43 +0530 Subject: [PATCH 01/16] Ingest new format for server sent events within the HTTP streamed response - Note that the SSR for next doesn't support rendering on the client-side, so it'll only update it one big chunk - Fix unique key error in the chatmessage history for incoming messages - Remove websocket value usage in the chat history side panel - Remove other websocket code from the chat page --- src/interface/web/app/chat/page.tsx | 220 +++++++++--------- src/interface/web/app/common/chatFunctions.ts | 48 +++- .../components/chatHistory/chatHistory.tsx | 11 +- .../chatMessage/chatMessage.module.css | 5 + .../components/chatMessage/chatMessage.tsx | 10 +- .../sidePanel/chatHistorySidePanel.tsx | 10 - 6 files changed, 173 insertions(+), 131 deletions(-) diff --git a/src/interface/web/app/chat/page.tsx b/src/interface/web/app/chat/page.tsx index f81287fd..43533ab8 100644 --- a/src/interface/web/app/chat/page.tsx +++ b/src/interface/web/app/chat/page.tsx @@ -9,12 +9,12 @@ import NavMenu from '../components/navMenu/navMenu'; import { useSearchParams } from 'next/navigation' import Loading from '../components/loading/loading'; -import { handleCompiledReferences, handleImageResponse, setupWebSocket } from '../common/chatFunctions'; +import { convertMessageChunkToJson, handleImageResponse, RawReferenceData } from '../common/chatFunctions'; import 'katex/dist/katex.min.css'; import { StreamMessage } from '../components/chatMessage/chatMessage'; -import { welcomeConsole } from '../common/utils'; +import { useIPLocationData, welcomeConsole } from '../common/utils'; import ChatInputArea, { ChatOptions } from '../components/chatInputArea/chatInputArea'; import { useAuthenticatedData } from '../common/auth'; import { AgentData } from '../agents/page'; @@ -97,83 +97,22 @@ function ChatBodyData(props: ChatBodyDataProps) { ); } + export default function Chat() { const [chatOptionsData, setChatOptionsData] = useState(null); const [isLoading, setLoading] = useState(true); const [title, setTitle] = useState('Khoj AI - Chat'); const [conversationId, setConversationID] = useState(null); - const [chatWS, setChatWS] = useState(null); const [messages, setMessages] = useState([]); const [queryToProcess, setQueryToProcess] = useState(''); const [processQuerySignal, setProcessQuerySignal] = useState(false); const [uploadedFiles, setUploadedFiles] = useState([]); const [isMobileWidth, setIsMobileWidth] = useState(false); + const locationData = useIPLocationData(); const authenticatedData = useAuthenticatedData(); welcomeConsole(); - const handleWebSocketMessage = (event: MessageEvent) => { - let chunk = event.data; - let currentMessage = messages.find(message => !message.completed); - if (!currentMessage) { - console.error("No current message found"); - return; - } - - // Process WebSocket streamed data - if (chunk === "start_llm_response") { - console.log("Started streaming", new Date()); - } else if (chunk === "end_llm_response") { - currentMessage.completed = true; - } else { - // Get the current message - // Process and update state with the new message - if (chunk.includes("application/json")) { - chunk = JSON.parse(chunk); - } - - const contentType = chunk["content-type"]; - if (contentType === "application/json") { - try { - if (chunk.image || chunk.detail) { - let responseWithReference = handleImageResponse(chunk); - console.log("Image response", responseWithReference); - if (responseWithReference.response) currentMessage.rawResponse = responseWithReference.response; - if (responseWithReference.online) currentMessage.onlineContext = responseWithReference.online; - if (responseWithReference.context) currentMessage.context = responseWithReference.context; - } else if (chunk.type == "status") { - currentMessage.trainOfThought.push(chunk.message); - } else if (chunk.type == "rate_limit") { - console.log("Rate limit message", chunk); - currentMessage.rawResponse = chunk.message; - } else { - console.log("any message", chunk); - } - } catch (error) { - console.error("Error processing message", error); - currentMessage.completed = true; - } finally { - // no-op - } - } else { - // Update the current message with the new chunk - if (chunk && chunk.includes("### compiled references:")) { - let responseWithReference = handleCompiledReferences(chunk, ""); - currentMessage.rawResponse += responseWithReference.response; - - if (responseWithReference.response) currentMessage.rawResponse = responseWithReference.response; - if (responseWithReference.online) currentMessage.onlineContext = responseWithReference.online; - if (responseWithReference.context) currentMessage.context = responseWithReference.context; - } else { - // If the chunk is not a JSON object, just display it as is - currentMessage.rawResponse += chunk; - } - } - }; - // Update the state with the new message, currentMessage - setMessages([...messages]); - } - useEffect(() => { fetch('/api/chat/options') .then(response => response.json()) @@ -198,19 +137,7 @@ export default function Chat() { }, []); useEffect(() => { - if (chatWS) { - chatWS.onmessage = handleWebSocketMessage; - } - }, [chatWS, messages]); - - //same as ChatBodyData for local storage message - useEffect(() => { - const storedMessage = localStorage.getItem("message"); - setQueryToProcess(storedMessage || ''); - }, []); - - useEffect(() => { - if (chatWS && queryToProcess) { + if (queryToProcess) { const newStreamMessage: StreamMessage = { rawResponse: "", trainOfThought: [], @@ -221,44 +148,118 @@ export default function Chat() { rawQuery: queryToProcess || "", }; setMessages(prevMessages => [...prevMessages, newStreamMessage]); - - if (chatWS.readyState === WebSocket.OPEN) { - chatWS.send(queryToProcess); - setProcessQuerySignal(true); - } - else { - console.error("WebSocket is not open. ReadyState:", chatWS.readyState); - } - - setQueryToProcess(''); + setProcessQuerySignal(true); } - }, [queryToProcess, chatWS]); + }, [queryToProcess]); useEffect(() => { - if (processQuerySignal && chatWS && chatWS.readyState === WebSocket.OPEN) { - setProcessQuerySignal(false); - chatWS.onmessage = handleWebSocketMessage; - chatWS.send(queryToProcess); - localStorage.removeItem("message"); + if (processQuerySignal) { + chat(); } - }, [processQuerySignal, chatWS]); + }, [processQuerySignal]); - useEffect(() => { - const setupWebSocketConnection = async () => { - if (conversationId && (!chatWS || chatWS.readyState === WebSocket.CLOSED)) { - if (queryToProcess) { - const newWS = await setupWebSocket(conversationId, queryToProcess); - localStorage.removeItem("message"); - setChatWS(newWS); - } - else { - const newWS = await setupWebSocket(conversationId); - setChatWS(newWS); + async function readChatStream(response: Response) { + if (!response.ok) throw new Error(response.statusText); + if (!response.body) throw new Error("Response body is null"); + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + const eventDelimiter = '␃🔚␗'; + let buffer = ""; + + while (true) { + + const { done, value } = await reader.read(); + if (done) { + setQueryToProcess(''); + setProcessQuerySignal(false); + break; + } + + const chunk = decoder.decode(value, { stream: true }); + + buffer += chunk; + + let newEventIndex; + while ((newEventIndex = buffer.indexOf(eventDelimiter)) !== -1) { + const event = buffer.slice(0, newEventIndex); + buffer = buffer.slice(newEventIndex + eventDelimiter.length); + if (event) { + processMessageChunk(event); } } - }; - setupWebSocketConnection(); - }, [conversationId]); + + } + } + + async function chat() { + localStorage.removeItem("message"); + let chatAPI = `/api/chat?q=${encodeURIComponent(queryToProcess)}&conversation_id=${conversationId}&stream=true&client=web`; + if (locationData) { + chatAPI += `®ion=${locationData.region}&country=${locationData.country}&city=${locationData.city}&timezone=${locationData.timezone}`; + } + + const response = await fetch(chatAPI); + try { + await readChatStream(response); + } catch (err) { + console.log(err); + } + } + + function processMessageChunk(rawChunk: string) { + const chunk = convertMessageChunkToJson(rawChunk); + const currentMessage = messages.find(message => !message.completed); + + if (!currentMessage) { + return; + } + + if (!chunk || !chunk.type) { + return; + } + + if (chunk.type === "status") { + const statusMessage = chunk.data as string; + currentMessage.trainOfThought.push(statusMessage); + } else if (chunk.type === "references") { + const references = chunk.data as RawReferenceData; + + if (references.context) { + currentMessage.context = references.context; + } + + if (references.onlineContext) { + currentMessage.onlineContext = references.onlineContext; + } + } else if (chunk.type === "message") { + const chunkData = chunk.data; + + if (chunkData !== null && typeof chunkData === 'object') { + try { + const jsonData = chunkData as any; + if (jsonData.image || jsonData.detail) { + let responseWithReference = handleImageResponse(chunk.data, true); + if (responseWithReference.response) currentMessage.rawResponse = responseWithReference.response; + if (responseWithReference.online) currentMessage.onlineContext = responseWithReference.online; + if (responseWithReference.context) currentMessage.context = responseWithReference.context; + } else if (jsonData.response) { + currentMessage.rawResponse = jsonData.response; + } + else { + console.log("any message", chunk); + } + } catch (e) { + currentMessage.rawResponse += chunkData; + } + } else { + currentMessage.rawResponse += chunkData; + } + } else if (chunk.type === "end_llm_response") { + currentMessage.completed = true; + } + setMessages([...messages]); + } const handleConversationIdChange = (newConversationId: string) => { setConversationID(newConversationId); @@ -276,7 +277,6 @@ export default function Chat() {
0) { + return { + type: "message", + data: chunk + }; + } else { + return { + type: "message", + data: "" + }; + } +} + +export function handleImageResponse(imageJson: any, liveStream: boolean): ResponseWithReferences { let rawResponse = ""; @@ -123,7 +165,7 @@ export function handleImageResponse(imageJson: any) { } else if (imageJson.intentType === "text-to-image-v3") { rawResponse = `![](data:image/webp;base64,${imageJson.image})`; } - if (inferredQuery) { + if (inferredQuery && !liveStream) { rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`; } } diff --git a/src/interface/web/app/components/chatHistory/chatHistory.tsx b/src/interface/web/app/components/chatHistory/chatHistory.tsx index a6fde5c9..2505dbf7 100644 --- a/src/interface/web/app/components/chatHistory/chatHistory.tsx +++ b/src/interface/web/app/components/chatHistory/chatHistory.tsx @@ -17,6 +17,7 @@ import { Lightbulb } from "@phosphor-icons/react"; import ProfileCard from '../profileCard/profileCard'; import { getIconFromIconName } from '@/app/common/iconUtils'; import { AgentData } from '@/app/agents/page'; +import React from 'react'; interface ChatResponse { status: string; @@ -120,7 +121,6 @@ export default function ChatHistory(props: ChatHistoryProps) { }, [props.conversationId]); useEffect(() => { - console.log(props.incomingMessages); if (props.incomingMessages) { const lastMessage = props.incomingMessages[props.incomingMessages.length - 1]; if (lastMessage && !lastMessage.completed) { @@ -195,7 +195,7 @@ export default function ChatHistory(props: ChatHistoryProps) { setFetchingData(false); } else { if (chatData.response.agent && chatData.response.conversation_id) { - const chatMetadata ={ + const chatMetadata = { chat: [], agent: chatData.response.agent, conversation_id: chatData.response.conversation_id, @@ -256,7 +256,7 @@ export default function ChatHistory(props: ChatHistoryProps) {
- {fetchingData && } + {fetchingData && }
{(data && data.chat) && data.chat.map((chatMessage, index) => ( { return ( - <> + + - + ) }) } diff --git a/src/interface/web/app/components/chatMessage/chatMessage.module.css b/src/interface/web/app/components/chatMessage/chatMessage.module.css index 592e30af..809acc2e 100644 --- a/src/interface/web/app/components/chatMessage/chatMessage.module.css +++ b/src/interface/web/app/components/chatMessage/chatMessage.module.css @@ -123,6 +123,11 @@ div.trainOfThought.primary p { color: inherit; } +div.trainOfThoughtElement { + display: grid; + grid-template-columns: auto 1fr; +} + @media screen and (max-width: 768px) { div.youfullHistory { max-width: 90%; diff --git a/src/interface/web/app/components/chatMessage/chatMessage.tsx b/src/interface/web/app/components/chatMessage/chatMessage.tsx index 6b05fdf9..a926afb9 100644 --- a/src/interface/web/app/components/chatMessage/chatMessage.tsx +++ b/src/interface/web/app/components/chatMessage/chatMessage.tsx @@ -10,7 +10,7 @@ import 'katex/dist/katex.min.css'; import { TeaserReferencesSection, constructAllReferences } from '../referencePanel/referencePanel'; -import { ThumbsUp, ThumbsDown, Copy, Brain, Cloud, Folder, Book, Aperture, SpeakerHigh, MagnifyingGlass, Pause } from '@phosphor-icons/react'; +import { ThumbsUp, ThumbsDown, Copy, Brain, Cloud, Folder, Book, Aperture, SpeakerHigh, MagnifyingGlass, Pause, Palette } from '@phosphor-icons/react'; import * as DomPurify from 'dompurify'; import { InlineLoading } from '../loading/loading'; @@ -180,10 +180,14 @@ function chooseIconFromHeader(header: string, iconColor: string) { return ; } - if (compareHeader.includes("summary") || compareHeader.includes("summarize")) { + if (compareHeader.includes("summary") || compareHeader.includes("summarize") || compareHeader.includes("enhanc")) { return ; } + if (compareHeader.includes("paint")) { + return ; + } + return ; } @@ -195,7 +199,7 @@ export function TrainOfThought(props: TrainOfThoughtProps) { const icon = chooseIconFromHeader(header, iconColor); let markdownRendered = DomPurify.sanitize(md.render(props.message)); return ( -
+
{icon}
diff --git a/src/interface/web/app/components/sidePanel/chatHistorySidePanel.tsx b/src/interface/web/app/components/sidePanel/chatHistorySidePanel.tsx index cd0f8bea..b9209b49 100644 --- a/src/interface/web/app/components/sidePanel/chatHistorySidePanel.tsx +++ b/src/interface/web/app/components/sidePanel/chatHistorySidePanel.tsx @@ -320,7 +320,6 @@ function FilesMenu(props: FilesMenuProps) { } interface SessionsAndFilesProps { - webSocketConnected?: boolean; setEnabled: (enabled: boolean) => void; subsetOrganizedData: GroupedChatHistory | null; organizedData: GroupedChatHistory | null; @@ -591,12 +590,6 @@ function ChatSessionsModal({ data, showSidePanel }: ChatSessionsModalProps) { ); } -interface UserProfileProps { - userProfile: UserProfile; - webSocketConnected?: boolean; - collapsed: boolean; -} - const fetchChatHistory = async (url: string) => { const response = await fetch(url, { method: 'GET', @@ -618,7 +611,6 @@ export const useChatSessionsFetchRequest = (url: string) => { }; interface SidePanelProps { - webSocketConnected?: boolean; conversationId: string | null; uploadedFiles: string[]; isMobileWidth: boolean; @@ -691,7 +683,6 @@ export default function SidePanel(props: SidePanelProps) {
Date: Thu, 1 Aug 2024 12:52:05 +0530 Subject: [PATCH 02/16] Remove the usage of emojis in the incremental status updates --- src/khoj/processor/tools/online_search.py | 8 ++++---- src/khoj/routers/helpers.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/khoj/processor/tools/online_search.py b/src/khoj/processor/tools/online_search.py index c087de70..caf800b0 100644 --- a/src/khoj/processor/tools/online_search.py +++ b/src/khoj/processor/tools/online_search.py @@ -68,7 +68,7 @@ async def search_online( logger.info(f"🌐 Searching the Internet for {list(subqueries)}") if send_status_func: subqueries_str = "\n- " + "\n- ".join(list(subqueries)) - async for event in send_status_func(f"**🌐 Searching the Internet for**: {subqueries_str}"): + async for event in send_status_func(f"**Searching the Internet for**: {subqueries_str}"): yield {ChatEvent.STATUS: event} with timer(f"Internet searches for {list(subqueries)} took", logger): @@ -92,7 +92,7 @@ async def search_online( logger.info(f"🌐👀 Reading web pages at: {list(webpage_links)}") if send_status_func: webpage_links_str = "\n- " + "\n- ".join(list(webpage_links)) - async for event in send_status_func(f"**📖 Reading web pages**: {webpage_links_str}"): + async for event in send_status_func(f"**Reading web pages**: {webpage_links_str}"): yield {ChatEvent.STATUS: event} tasks = [read_webpage_and_extract_content(subquery, link, content) for link, subquery, content in webpages] results = await asyncio.gather(*tasks) @@ -131,14 +131,14 @@ async def read_webpages( "Infer web pages to read from the query and extract relevant information from them" logger.info(f"Inferring web pages to read") if send_status_func: - async for event in send_status_func(f"**🧐 Inferring web pages to read**"): + async for event in send_status_func(f"**Inferring web pages to read**"): yield {ChatEvent.STATUS: event} urls = await infer_webpage_urls(query, conversation_history, location) logger.info(f"Reading web pages at: {urls}") if send_status_func: webpage_links_str = "\n- " + "\n- ".join(list(urls)) - async for event in send_status_func(f"**📖 Reading web pages**: {webpage_links_str}"): + async for event in send_status_func(f"**Reading web pages**: {webpage_links_str}"): yield {ChatEvent.STATUS: event} tasks = [read_webpage_and_extract_content(query, url) for url in urls] results = await asyncio.gather(*tasks) diff --git a/src/khoj/routers/helpers.py b/src/khoj/routers/helpers.py index 9db7a608..b0702448 100644 --- a/src/khoj/routers/helpers.py +++ b/src/khoj/routers/helpers.py @@ -815,7 +815,7 @@ async def text_to_image( ) if send_status_func: - async for event in send_status_func(f"**🖼️ Painting using Enhanced Prompt**:\n{improved_image_prompt}"): + async for event in send_status_func(f"**Painting to Imagine**:\n{improved_image_prompt}"): yield {ChatEvent.STATUS: event} if text_to_image_config.model_type == TextToImageModelConfig.ModelType.OPENAI: From 833553c3a38a52772654042e8f5a1be9987efe94 Mon Sep 17 00:00:00 2001 From: sabaimran Date: Thu, 1 Aug 2024 12:52:41 +0530 Subject: [PATCH 03/16] Move conversation commands selection earlier to include in telemetry collected --- src/khoj/routers/api_chat.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/khoj/routers/api_chat.py b/src/khoj/routers/api_chat.py index fa37e1f5..8b7bada4 100644 --- a/src/khoj/routers/api_chat.py +++ b/src/khoj/routers/api_chat.py @@ -603,6 +603,8 @@ async def chat( metadata=chat_metadata, ) + conversation_commands = [get_conversation_command(query=q, any_references=True)] + conversation = await ConversationAdapters.aget_conversation_by_user( user, client_application=request.user.client_app, conversation_id=conversation_id, title=title ) @@ -624,7 +626,6 @@ async def chat( return user_message_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - conversation_commands = [get_conversation_command(query=q, any_references=True)] async for result in send_event(ChatEvent.STATUS, f"**Understanding Query**: {q}"): yield result From bfeb64b48f2302f88a5b0d5c18a7cb96a2289509 Mon Sep 17 00:00:00 2001 From: sabaimran Date: Thu, 1 Aug 2024 13:14:09 +0530 Subject: [PATCH 04/16] Migrate the shared chat page to also use the new SSE streaming format --- src/interface/web/app/share/chat/page.tsx | 203 ++++++++++++---------- 1 file changed, 110 insertions(+), 93 deletions(-) diff --git a/src/interface/web/app/share/chat/page.tsx b/src/interface/web/app/share/chat/page.tsx index d270c1f0..f02ed1a1 100644 --- a/src/interface/web/app/share/chat/page.tsx +++ b/src/interface/web/app/share/chat/page.tsx @@ -10,12 +10,12 @@ import Loading from '../../components/loading/loading'; import 'katex/dist/katex.min.css'; -import { welcomeConsole } from '../../common/utils'; +import { useIPLocationData, welcomeConsole } from '../../common/utils'; import { useAuthenticatedData } from '@/app/common/auth'; import ChatInputArea, { ChatOptions } from '@/app/components/chatInputArea/chatInputArea'; import { StreamMessage } from '@/app/components/chatMessage/chatMessage'; -import { handleCompiledReferences, handleImageResponse, setupWebSocket } from '@/app/common/chatFunctions'; +import { convertMessageChunkToJson, handleCompiledReferences, handleImageResponse, RawReferenceData, setupWebSocket } from '@/app/common/chatFunctions'; import { AgentData } from '@/app/agents/page'; @@ -95,7 +95,6 @@ export default function SharedChat() { const [isLoading, setLoading] = useState(true); const [title, setTitle] = useState('Khoj AI - Chat'); const [conversationId, setConversationID] = useState(undefined); - const [chatWS, setChatWS] = useState(null); const [messages, setMessages] = useState([]); const [queryToProcess, setQueryToProcess] = useState(''); const [processQuerySignal, setProcessQuerySignal] = useState(false); @@ -103,76 +102,11 @@ export default function SharedChat() { const [isMobileWidth, setIsMobileWidth] = useState(false); const [paramSlug, setParamSlug] = useState(undefined); + const locationData = useIPLocationData(); const authenticatedData = useAuthenticatedData(); welcomeConsole(); - const handleWebSocketMessage = (event: MessageEvent) => { - let chunk = event.data; - - let currentMessage = messages.find(message => !message.completed); - - if (!currentMessage) { - console.error("No current message found"); - return; - } - - // Process WebSocket streamed data - if (chunk === "start_llm_response") { - console.log("Started streaming", new Date()); - } else if (chunk === "end_llm_response") { - currentMessage.completed = true; - } else { - // Get the current message - // Process and update state with the new message - if (chunk.includes("application/json")) { - chunk = JSON.parse(chunk); - } - - const contentType = chunk["content-type"]; - if (contentType === "application/json") { - try { - if (chunk.image || chunk.detail) { - let responseWithReference = handleImageResponse(chunk); - console.log("Image response", responseWithReference); - if (responseWithReference.response) currentMessage.rawResponse = responseWithReference.response; - if (responseWithReference.online) currentMessage.onlineContext = responseWithReference.online; - if (responseWithReference.context) currentMessage.context = responseWithReference.context; - } else if (chunk.type == "status") { - currentMessage.trainOfThought.push(chunk.message); - } else if (chunk.type == "rate_limit") { - console.log("Rate limit message", chunk); - currentMessage.rawResponse = chunk.message; - } else { - console.log("any message", chunk); - } - } catch (error) { - console.error("Error processing message", error); - currentMessage.completed = true; - } finally { - // no-op - } - - } else { - // Update the current message with the new chunk - if (chunk && chunk.includes("### compiled references:")) { - let responseWithReference = handleCompiledReferences(chunk, ""); - currentMessage.rawResponse += responseWithReference.response; - - if (responseWithReference.response) currentMessage.rawResponse = responseWithReference.response; - if (responseWithReference.online) currentMessage.onlineContext = responseWithReference.online; - if (responseWithReference.context) currentMessage.context = responseWithReference.context; - } else { - // If the chunk is not a JSON object, just display it as is - currentMessage.rawResponse += chunk; - } - - } - }; - // Update the state with the new message, currentMessage - setMessages([...messages]); - } - useEffect(() => { fetch('/api/chat/options') @@ -201,6 +135,7 @@ export default function SharedChat() { useEffect(() => { if (queryToProcess && !conversationId) { + // If the user has not yet started conversing in the chat, create a new conversation fetch(`/api/chat/share/fork?public_conversation_slug=${paramSlug}`, { method: 'POST', headers: { @@ -219,7 +154,7 @@ export default function SharedChat() { } - if (chatWS && queryToProcess) { + if (queryToProcess) { // Add a new object to the state const newStreamMessage: StreamMessage = { rawResponse: "", @@ -232,40 +167,68 @@ export default function SharedChat() { } setMessages(prevMessages => [...prevMessages, newStreamMessage]); setProcessQuerySignal(true); - } else { - if (!chatWS) { - console.error("No WebSocket connection available"); - } - if (!queryToProcess) { - console.error("No query to process"); - } } }, [queryToProcess]); useEffect(() => { - if (processQuerySignal && chatWS) { - setProcessQuerySignal(false); - chatWS.onmessage = handleWebSocketMessage; - chatWS?.send(queryToProcess); + if (processQuerySignal) { + chat(); } }, [processQuerySignal]); - useEffect(() => { - if (chatWS) { - chatWS.onmessage = handleWebSocketMessage; + + async function readChatStream(response: Response) { + if (!response.ok) throw new Error(response.statusText); + if (!response.body) throw new Error("Response body is null"); + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + const eventDelimiter = '␃🔚␗'; + let buffer = ""; + + while (true) { + + const { done, value } = await reader.read(); + if (done) { + setQueryToProcess(''); + setProcessQuerySignal(false); + break; + } + + const chunk = decoder.decode(value, { stream: true }); + + buffer += chunk; + + let newEventIndex; + while ((newEventIndex = buffer.indexOf(eventDelimiter)) !== -1) { + const event = buffer.slice(0, newEventIndex); + buffer = buffer.slice(newEventIndex + eventDelimiter.length); + if (event) { + processMessageChunk(event); + } + } + } - }, [chatWS]); + } + + async function chat() { + if (!queryToProcess || !conversationId) return; + let chatAPI = `/api/chat?q=${encodeURIComponent(queryToProcess)}&conversation_id=${conversationId}&stream=true&client=web`; + if (locationData) { + chatAPI += `®ion=${locationData.region}&country=${locationData.country}&city=${locationData.city}&timezone=${locationData.timezone}`; + } + + const response = await fetch(chatAPI); + try { + await readChatStream(response); + } catch (err) { + console.log(err); + } + } useEffect(() => { (async () => { if (conversationId) { - const newWS = await setupWebSocket(conversationId, queryToProcess); - if (!newWS) { - console.error("No WebSocket connection available"); - return; - } - setChatWS(newWS); - // Add a new object to the state const newStreamMessage: StreamMessage = { rawResponse: "", @@ -276,11 +239,66 @@ export default function SharedChat() { timestamp: (new Date()).toISOString(), rawQuery: queryToProcess || "", } + setProcessQuerySignal(true); setMessages(prevMessages => [...prevMessages, newStreamMessage]); } })(); }, [conversationId]); + function processMessageChunk(rawChunk: string) { + const chunk = convertMessageChunkToJson(rawChunk); + const currentMessage = messages.find(message => !message.completed); + + if (!currentMessage) { + return; + } + + if (!chunk || !chunk.type) { + return; + } + + if (chunk.type === "status") { + const statusMessage = chunk.data as string; + currentMessage.trainOfThought.push(statusMessage); + } else if (chunk.type === "references") { + const references = chunk.data as RawReferenceData; + + if (references.context) { + currentMessage.context = references.context; + } + + if (references.onlineContext) { + currentMessage.onlineContext = references.onlineContext; + } + } else if (chunk.type === "message") { + const chunkData = chunk.data; + + if (chunkData !== null && typeof chunkData === 'object') { + try { + const jsonData = chunkData as any; + if (jsonData.image || jsonData.detail) { + let responseWithReference = handleImageResponse(chunk.data, true); + if (responseWithReference.response) currentMessage.rawResponse = responseWithReference.response; + if (responseWithReference.online) currentMessage.onlineContext = responseWithReference.online; + if (responseWithReference.context) currentMessage.context = responseWithReference.context; + } else if (jsonData.response) { + currentMessage.rawResponse = jsonData.response; + } + else { + console.log("any message", chunk); + } + } catch (e) { + currentMessage.rawResponse += chunkData; + } + } else { + currentMessage.rawResponse += chunkData; + } + } else if (chunk.type === "end_llm_response") { + currentMessage.completed = true; + } + setMessages([...messages]); + } + if (isLoading) { return ; } @@ -301,7 +319,6 @@ export default function SharedChat() {
Date: Thu, 1 Aug 2024 13:14:23 +0530 Subject: [PATCH 05/16] Remove usages of the websocketconnected variable --- src/interface/web/app/agents/page.tsx | 1 - src/interface/web/app/automations/page.tsx | 1 - src/interface/web/app/chat/page.tsx | 1 + src/interface/web/app/page.tsx | 1 - src/interface/web/app/search/page.tsx | 1 - 5 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/interface/web/app/agents/page.tsx b/src/interface/web/app/agents/page.tsx index e2e16071..5e68a0b9 100644 --- a/src/interface/web/app/agents/page.tsx +++ b/src/interface/web/app/agents/page.tsx @@ -278,7 +278,6 @@ export default function Agents() {
Date: Thu, 1 Aug 2024 13:53:16 +0530 Subject: [PATCH 06/16] Fix logic for setting and sending the initial chat message from the home page - Load agents only once when the page loads, rather than triggering constant re-renders --- src/interface/web/app/automations/page.tsx | 2 - src/interface/web/app/chat/page.tsx | 3 +- src/interface/web/app/page.tsx | 50 +++++++++++----------- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/src/interface/web/app/automations/page.tsx b/src/interface/web/app/automations/page.tsx index 283f9c63..9416cee7 100644 --- a/src/interface/web/app/automations/page.tsx +++ b/src/interface/web/app/automations/page.tsx @@ -85,7 +85,6 @@ function getEveryBlahFromCron(cron: string) { function getDayOfWeekFromCron(cron: string) { const cronParts = cron.split(' '); - console.log(cronParts); if (cronParts[3] === '*' && cronParts[4] !== '*') { return Number(cronParts[4]); } @@ -248,7 +247,6 @@ function AutomationsCard(props: AutomationsCardProps) { setTimeRecurrence(getTimeRecurrenceFromCron(automationData.crontime)); const frequency = getEveryBlahFromCron(automationData.crontime); - console.log('frequency', frequency); if (frequency === 'Day') { setIntervalString('Daily'); } else if (frequency === 'Week') { diff --git a/src/interface/web/app/chat/page.tsx b/src/interface/web/app/chat/page.tsx index 1bdc1a7a..9bbd4c6c 100644 --- a/src/interface/web/app/chat/page.tsx +++ b/src/interface/web/app/chat/page.tsx @@ -40,7 +40,8 @@ function ChatBodyData(props: ChatBodyDataProps) { useEffect(() => { const storedMessage = localStorage.getItem("message"); if (storedMessage) { - setMessage(storedMessage); + setProcessingMessage(true); + props.setQueryToProcess(storedMessage); } }, []); diff --git a/src/interface/web/app/page.tsx b/src/interface/web/app/page.tsx index 33c566d6..14f2ab4f 100644 --- a/src/interface/web/app/page.tsx +++ b/src/interface/web/app/page.tsx @@ -16,7 +16,7 @@ import 'katex/dist/katex.min.css'; import ChatInputArea, { ChatOptions } from './components/chatInputArea/chatInputArea'; import { useAuthenticatedData } from './common/auth'; import { Card, CardTitle } from '@/components/ui/card'; -import { converColorToBgGradient, colorMap, convertColorToBorderClass } from './common/colorUtils'; +import { colorMap, convertColorToBorderClass } from './common/colorUtils'; import { getIconFromIconName } from './common/iconUtils'; import { ClockCounterClockwise } from '@phosphor-icons/react'; import { AgentData } from './agents/page'; @@ -67,9 +67,11 @@ function ChatBodyData(props: ChatBodyDataProps) { const [processingMessage, setProcessingMessage] = useState(false); const [shuffledOptions, setShuffledOptions] = useState([]); const [selectedAgent, setSelectedAgent] = useState("khoj"); + const [agentIcons, setAgentIcons] = useState([]); + const [agents, setAgents] = useState([]); const agentsFetcher = () => window.fetch('/api/agents').then(res => res.json()).catch(err => console.log(err)); - const { data, error } = useSWR('agents', agentsFetcher, { revalidateOnFocus: false }); + const { data: agentsData, error } = useSWR('agents', agentsFetcher, { revalidateOnFocus: false }); function shuffleAndSetOptions() { const shuffled = [...suggestionsData].sort(() => 0.5 - Math.random()); @@ -82,6 +84,28 @@ function ChatBodyData(props: ChatBodyDataProps) { } }, [props.chatOptionsData]); + useEffect(() => { + const nSlice = props.isMobileWidth ? 3 : 4; + + const shuffledAgents = agentsData ? [...agentsData].sort(() => 0.5 - Math.random()) : []; + + const agents = agentsData ? [agentsData[0]] : []; // Always add the first/default agent. + + shuffledAgents.slice(0, nSlice - 1).forEach(agent => { + if (!agents.find(a => a.slug === agent.slug)) { + agents.push(agent); + } + }); + + setAgents(agents); + + //generate colored icons for the selected agents + const agentIcons = agents.map( + agent => getIconFromIconName(agent.icon, agent.color) || {agent.name} + ); + setAgentIcons(agentIcons); + }, [agentsData]); + function shuffleSuggestionsCards() { shuffleAndSetOptions(); } @@ -109,24 +133,6 @@ function ChatBodyData(props: ChatBodyDataProps) { }; }, [selectedAgent, message]); - const nSlice = props.isMobileWidth ? 3 : 4; - - const shuffledAgents = data ? [...data].sort(() => 0.5 - Math.random()) : []; - - const agents = data ? [data[0]] : []; // Always add the first/default agent. - - shuffledAgents.slice(0, nSlice - 1).forEach(agent => { - if (!agents.find(a => a.slug === agent.slug)) { - agents.push(agent); - } - }); - - - //generate colored icons for the selected agents - const agentIcons = agents.map( - agent => getIconFromIconName(agent.icon, agent.color) || {agent.name} - ); - function fillArea(link: string, type: string, prompt: string) { if (!link) { let message_str = ""; @@ -150,10 +156,6 @@ function ChatBodyData(props: ChatBodyDataProps) { } } - function getTailwindBorderClass(color: string): string { - return colorMap[color] || 'border-black'; // Default to black if color not found - } - return (
From 7941f4d54d89a200775f17ecbf289d2d123fcd63 Mon Sep 17 00:00:00 2001 From: sabaimran Date: Thu, 1 Aug 2024 14:43:17 +0530 Subject: [PATCH 07/16] Remove references to deprecated setupwebsocket function --- src/interface/web/app/common/chatFunctions.ts | 82 ------------------- src/interface/web/app/share/chat/page.tsx | 2 +- 2 files changed, 1 insertion(+), 83 deletions(-) diff --git a/src/interface/web/app/common/chatFunctions.ts b/src/interface/web/app/common/chatFunctions.ts index e1a5f5a5..a1657ad5 100644 --- a/src/interface/web/app/common/chatFunctions.ts +++ b/src/interface/web/app/common/chatFunctions.ts @@ -33,88 +33,6 @@ export function handleCompiledReferences(chunk: string, currentResponse: string) return references; } -async function sendChatStream( - message: string, - conversationId: string, - setIsLoading: (loading: boolean) => void, - setInitialResponse: (response: string) => void, - setInitialReferences: (references: ResponseWithReferences) => void) { - setIsLoading(true); - // Send a message to the chat server to verify the fact - const chatURL = "/api/chat"; - const apiURL = `${chatURL}?q=${encodeURIComponent(message)}&client=web&stream=true&conversation_id=${conversationId}`; - try { - const response = await fetch(apiURL); - if (!response.body) throw new Error("No response body found"); - - const reader = response.body?.getReader(); - let decoder = new TextDecoder(); - let result = ""; - - while (true) { - const { done, value } = await reader.read(); - if (done) break; - - let chunk = decoder.decode(value, { stream: true }); - - if (chunk.includes("### compiled references:")) { - const references = handleCompiledReferences(chunk, result); - if (references.response) { - result = references.response; - setInitialResponse(references.response); - setInitialReferences(references); - } - } else { - result += chunk; - setInitialResponse(result); - } - } - } catch (error) { - console.error("Error verifying statement: ", error); - } finally { - setIsLoading(false); - } -} - -export const setupWebSocket = async (conversationId: string, initialMessage?: string) => { - const wsProtocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; - - const host = process.env.NODE_ENV === 'production' ? window.location.host : 'localhost:42110'; - - let webSocketUrl = `${wsProtocol}//${host}/api/chat/ws`; - - if (conversationId === null) { - return null; - } - - if (conversationId) { - webSocketUrl += `?conversation_id=${conversationId}`; - } - - const chatWS = new WebSocket(webSocketUrl); - - chatWS.onopen = () => { - console.log('WebSocket connection established'); - if (initialMessage) { - chatWS.send(initialMessage); - } - }; - - chatWS.onmessage = (event) => { - console.log(event.data); - }; - - chatWS.onerror = (error) => { - console.error('WebSocket error: ', error); - }; - - chatWS.onclose = () => { - console.log('WebSocket connection closed'); - }; - - return chatWS; -}; - interface MessageChunk { type: string; data: string | object; diff --git a/src/interface/web/app/share/chat/page.tsx b/src/interface/web/app/share/chat/page.tsx index f02ed1a1..56d27bbc 100644 --- a/src/interface/web/app/share/chat/page.tsx +++ b/src/interface/web/app/share/chat/page.tsx @@ -15,7 +15,7 @@ import { useAuthenticatedData } from '@/app/common/auth'; import ChatInputArea, { ChatOptions } from '@/app/components/chatInputArea/chatInputArea'; import { StreamMessage } from '@/app/components/chatMessage/chatMessage'; -import { convertMessageChunkToJson, handleCompiledReferences, handleImageResponse, RawReferenceData, setupWebSocket } from '@/app/common/chatFunctions'; +import { convertMessageChunkToJson, handleCompiledReferences, handleImageResponse, RawReferenceData } from '@/app/common/chatFunctions'; import { AgentData } from '@/app/agents/page'; From ed16914ac353126caa6661fb80de51814763c089 Mon Sep 17 00:00:00 2001 From: sabaimran Date: Thu, 1 Aug 2024 14:45:54 +0530 Subject: [PATCH 08/16] Remove deprecated fields and fix erroneous export in settings page --- src/interface/web/app/settings/page.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/interface/web/app/settings/page.tsx b/src/interface/web/app/settings/page.tsx index 1297d05d..eed448b4 100644 --- a/src/interface/web/app/settings/page.tsx +++ b/src/interface/web/app/settings/page.tsx @@ -268,7 +268,7 @@ interface TokenObject { name: string; } -export const useApiKeys = () => { +const useApiKeys = () => { const [apiKeys, setApiKeys] = useState([]); const { toast } = useToast(); @@ -621,7 +621,6 @@ export default function SettingsView() {
Date: Thu, 1 Aug 2024 19:22:21 +0530 Subject: [PATCH 09/16] Remove status update for understanding query --- src/khoj/routers/api_chat.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/khoj/routers/api_chat.py b/src/khoj/routers/api_chat.py index 8b7bada4..edb1c99a 100644 --- a/src/khoj/routers/api_chat.py +++ b/src/khoj/routers/api_chat.py @@ -627,9 +627,6 @@ async def chat( user_message_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - async for result in send_event(ChatEvent.STATUS, f"**Understanding Query**: {q}"): - yield result - meta_log = conversation.conversation_log is_automated_task = conversation_commands == [ConversationCommand.AutomatedTask] From 4492017b96db26a5c6432d2a9c232237f055d581 Mon Sep 17 00:00:00 2001 From: sabaimran Date: Fri, 2 Aug 2024 12:31:43 +0530 Subject: [PATCH 10/16] Move processmessagechunk file into a common chat function --- src/interface/web/app/chat/page.tsx | 68 +++---------------- src/interface/web/app/common/chatFunctions.ts | 55 ++++++++++++++- src/interface/web/app/share/chat/page.tsx | 67 +++--------------- 3 files changed, 76 insertions(+), 114 deletions(-) diff --git a/src/interface/web/app/chat/page.tsx b/src/interface/web/app/chat/page.tsx index 9bbd4c6c..fe2f9a23 100644 --- a/src/interface/web/app/chat/page.tsx +++ b/src/interface/web/app/chat/page.tsx @@ -9,7 +9,7 @@ import NavMenu from '../components/navMenu/navMenu'; import { useSearchParams } from 'next/navigation' import Loading from '../components/loading/loading'; -import { convertMessageChunkToJson, handleImageResponse, RawReferenceData } from '../common/chatFunctions'; +import { convertMessageChunkToJson, handleImageResponse, processMessageChunk, RawReferenceData } from '../common/chatFunctions'; import 'katex/dist/katex.min.css'; @@ -186,10 +186,18 @@ export default function Chat() { const event = buffer.slice(0, newEventIndex); buffer = buffer.slice(newEventIndex + eventDelimiter.length); if (event) { - processMessageChunk(event); + const currentMessage = messages.find(message => !message.completed); + + if (!currentMessage) { + console.error("No current message found"); + return; + } + + processMessageChunk(event, currentMessage); + + setMessages([...messages]); } } - } } @@ -209,60 +217,6 @@ export default function Chat() { } } - function processMessageChunk(rawChunk: string) { - const chunk = convertMessageChunkToJson(rawChunk); - const currentMessage = messages.find(message => !message.completed); - - if (!currentMessage) { - return; - } - - if (!chunk || !chunk.type) { - return; - } - - if (chunk.type === "status") { - const statusMessage = chunk.data as string; - currentMessage.trainOfThought.push(statusMessage); - } else if (chunk.type === "references") { - const references = chunk.data as RawReferenceData; - - if (references.context) { - currentMessage.context = references.context; - } - - if (references.onlineContext) { - currentMessage.onlineContext = references.onlineContext; - } - } else if (chunk.type === "message") { - const chunkData = chunk.data; - - if (chunkData !== null && typeof chunkData === 'object') { - try { - const jsonData = chunkData as any; - if (jsonData.image || jsonData.detail) { - let responseWithReference = handleImageResponse(chunk.data, true); - if (responseWithReference.response) currentMessage.rawResponse = responseWithReference.response; - if (responseWithReference.online) currentMessage.onlineContext = responseWithReference.online; - if (responseWithReference.context) currentMessage.context = responseWithReference.context; - } else if (jsonData.response) { - currentMessage.rawResponse = jsonData.response; - } - else { - console.log("any message", chunk); - } - } catch (e) { - currentMessage.rawResponse += chunkData; - } - } else { - currentMessage.rawResponse += chunkData; - } - } else if (chunk.type === "end_llm_response") { - currentMessage.completed = true; - } - setMessages([...messages]); - } - const handleConversationIdChange = (newConversationId: string) => { setConversationID(newConversationId); }; diff --git a/src/interface/web/app/common/chatFunctions.ts b/src/interface/web/app/common/chatFunctions.ts index a1657ad5..51a21115 100644 --- a/src/interface/web/app/common/chatFunctions.ts +++ b/src/interface/web/app/common/chatFunctions.ts @@ -1,4 +1,4 @@ -import { Context, OnlineContextData } from "../components/chatMessage/chatMessage"; +import { Context, OnlineContextData, StreamMessage } from "../components/chatMessage/chatMessage"; export interface RawReferenceData { context?: Context[]; @@ -68,6 +68,59 @@ export function convertMessageChunkToJson(chunk: string): MessageChunk { } } + +export function processMessageChunk(rawChunk: string, currentMessage: StreamMessage) { + const chunk = convertMessageChunkToJson(rawChunk); + + if (!currentMessage) { + return; + } + + if (!chunk || !chunk.type) { + return; + } + + if (chunk.type === "status") { + const statusMessage = chunk.data as string; + currentMessage.trainOfThought.push(statusMessage); + } else if (chunk.type === "references") { + const references = chunk.data as RawReferenceData; + + if (references.context) { + currentMessage.context = references.context; + } + + if (references.onlineContext) { + currentMessage.onlineContext = references.onlineContext; + } + } else if (chunk.type === "message") { + const chunkData = chunk.data; + + if (chunkData !== null && typeof chunkData === 'object') { + try { + const jsonData = chunkData as any; + if (jsonData.image || jsonData.detail) { + let responseWithReference = handleImageResponse(chunk.data, true); + if (responseWithReference.response) currentMessage.rawResponse = responseWithReference.response; + if (responseWithReference.online) currentMessage.onlineContext = responseWithReference.online; + if (responseWithReference.context) currentMessage.context = responseWithReference.context; + } else if (jsonData.response) { + currentMessage.rawResponse = jsonData.response; + } + else { + console.debug("any message", chunk); + } + } catch (e) { + currentMessage.rawResponse += chunkData; + } + } else { + currentMessage.rawResponse += chunkData; + } + } else if (chunk.type === "end_llm_response") { + currentMessage.completed = true; + } +} + export function handleImageResponse(imageJson: any, liveStream: boolean): ResponseWithReferences { let rawResponse = ""; diff --git a/src/interface/web/app/share/chat/page.tsx b/src/interface/web/app/share/chat/page.tsx index 56d27bbc..4a08f0a2 100644 --- a/src/interface/web/app/share/chat/page.tsx +++ b/src/interface/web/app/share/chat/page.tsx @@ -15,7 +15,7 @@ import { useAuthenticatedData } from '@/app/common/auth'; import ChatInputArea, { ChatOptions } from '@/app/components/chatInputArea/chatInputArea'; import { StreamMessage } from '@/app/components/chatMessage/chatMessage'; -import { convertMessageChunkToJson, handleCompiledReferences, handleImageResponse, RawReferenceData } from '@/app/common/chatFunctions'; +import { convertMessageChunkToJson, handleCompiledReferences, handleImageResponse, processMessageChunk, RawReferenceData } from '@/app/common/chatFunctions'; import { AgentData } from '@/app/agents/page'; @@ -204,7 +204,16 @@ export default function SharedChat() { const event = buffer.slice(0, newEventIndex); buffer = buffer.slice(newEventIndex + eventDelimiter.length); if (event) { - processMessageChunk(event); + const currentMessage = messages.find(message => !message.completed); + + if (!currentMessage) { + console.error("No current message found"); + return; + } + + processMessageChunk(event, currentMessage); + + setMessages([...messages]); } } @@ -245,60 +254,6 @@ export default function SharedChat() { })(); }, [conversationId]); - function processMessageChunk(rawChunk: string) { - const chunk = convertMessageChunkToJson(rawChunk); - const currentMessage = messages.find(message => !message.completed); - - if (!currentMessage) { - return; - } - - if (!chunk || !chunk.type) { - return; - } - - if (chunk.type === "status") { - const statusMessage = chunk.data as string; - currentMessage.trainOfThought.push(statusMessage); - } else if (chunk.type === "references") { - const references = chunk.data as RawReferenceData; - - if (references.context) { - currentMessage.context = references.context; - } - - if (references.onlineContext) { - currentMessage.onlineContext = references.onlineContext; - } - } else if (chunk.type === "message") { - const chunkData = chunk.data; - - if (chunkData !== null && typeof chunkData === 'object') { - try { - const jsonData = chunkData as any; - if (jsonData.image || jsonData.detail) { - let responseWithReference = handleImageResponse(chunk.data, true); - if (responseWithReference.response) currentMessage.rawResponse = responseWithReference.response; - if (responseWithReference.online) currentMessage.onlineContext = responseWithReference.online; - if (responseWithReference.context) currentMessage.context = responseWithReference.context; - } else if (jsonData.response) { - currentMessage.rawResponse = jsonData.response; - } - else { - console.log("any message", chunk); - } - } catch (e) { - currentMessage.rawResponse += chunkData; - } - } else { - currentMessage.rawResponse += chunkData; - } - } else if (chunk.type === "end_llm_response") { - currentMessage.completed = true; - } - setMessages([...messages]); - } - if (isLoading) { return ; } From 4f783b911c9904db486f3f7e5e109d09994e68e9 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Fri, 2 Aug 2024 00:44:29 +0530 Subject: [PATCH 11/16] Update DOMPurify imports correctly to resolve compilation warnings --- .../web/app/components/chatMessage/chatMessage.tsx | 6 +++--- .../web/app/components/referencePanel/referencePanel.tsx | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/interface/web/app/components/chatMessage/chatMessage.tsx b/src/interface/web/app/components/chatMessage/chatMessage.tsx index a926afb9..b8187f4c 100644 --- a/src/interface/web/app/components/chatMessage/chatMessage.tsx +++ b/src/interface/web/app/components/chatMessage/chatMessage.tsx @@ -12,7 +12,7 @@ import { TeaserReferencesSection, constructAllReferences } from '../referencePan import { ThumbsUp, ThumbsDown, Copy, Brain, Cloud, Folder, Book, Aperture, SpeakerHigh, MagnifyingGlass, Pause, Palette } from '@phosphor-icons/react'; -import * as DomPurify from 'dompurify'; +import DOMPurify from 'dompurify'; import { InlineLoading } from '../loading/loading'; import { convertColorToTextClass } from '@/app/common/colorUtils'; import { AgentData } from '@/app/agents/page'; @@ -197,7 +197,7 @@ export function TrainOfThought(props: TrainOfThoughtProps) { let header = extractedHeader ? extractedHeader[1] : ""; const iconColor = props.primary ? convertColorToTextClass(props.agentColor) : 'text-gray-500'; const icon = chooseIconFromHeader(header, iconColor); - let markdownRendered = DomPurify.sanitize(md.render(props.message)); + let markdownRendered = DOMPurify.sanitize(md.render(props.message)); return (
{icon} @@ -245,7 +245,7 @@ export default function ChatMessage(props: ChatMessageProps) { .replace(/LEFTBRACKET/g, '\\[').replace(/RIGHTBRACKET/g, '\\]'); // Sanitize and set the rendered markdown - setMarkdownRendered(DomPurify.sanitize(markdownRendered)); + setMarkdownRendered(DOMPurify.sanitize(markdownRendered)); }, [props.chatMessage.message]); useEffect(() => { diff --git a/src/interface/web/app/components/referencePanel/referencePanel.tsx b/src/interface/web/app/components/referencePanel/referencePanel.tsx index 17632bea..75345904 100644 --- a/src/interface/web/app/components/referencePanel/referencePanel.tsx +++ b/src/interface/web/app/components/referencePanel/referencePanel.tsx @@ -23,7 +23,7 @@ import { SheetTrigger, } from "@/components/ui/sheet"; import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; -import * as DomPurify from 'dompurify'; +import DOMPurify from 'dompurify'; interface NotesContextReferenceData { title: string; @@ -36,7 +36,7 @@ interface NotesContextReferenceCardProps extends NotesContextReferenceData { function NotesContextReferenceCard(props: NotesContextReferenceCardProps) { - const snippet = props.showFullContent ? DomPurify.sanitize(md.render(props.content)) : DomPurify.sanitize(props.content); + const snippet = props.showFullContent ? DOMPurify.sanitize(md.render(props.content)) : DOMPurify.sanitize(props.content); const [isHovering, setIsHovering] = useState(false); return ( From 3f607b397824ee9d41c622f792eeb9e918e0dc9f Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Fri, 2 Aug 2024 01:48:04 +0530 Subject: [PATCH 12/16] Add icons, improve description of home, chat & search page metadata --- src/interface/web/app/chat/layout.tsx | 7 +++++-- src/interface/web/app/layout.tsx | 7 +++++-- src/interface/web/app/search/layout.tsx | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/interface/web/app/chat/layout.tsx b/src/interface/web/app/chat/layout.tsx index 341960f7..0cdc1053 100644 --- a/src/interface/web/app/chat/layout.tsx +++ b/src/interface/web/app/chat/layout.tsx @@ -5,8 +5,11 @@ import "../globals.css"; const inter = Noto_Sans({ subsets: ["latin"] }); export const metadata: Metadata = { - title: "Khoj AI - Chat", - description: "Use this page to chat with Khoj AI.", + title: "Khoj AI - Chat", + description: "Ask anything. Khoj will use the internet and your docs to answer, paint and even automate stuff for you.", + icons: { + icon: '/static/favicon.ico', + }, }; export default function RootLayout({ diff --git a/src/interface/web/app/layout.tsx b/src/interface/web/app/layout.tsx index 1acd4a3f..282cd18b 100644 --- a/src/interface/web/app/layout.tsx +++ b/src/interface/web/app/layout.tsx @@ -5,8 +5,11 @@ import "./globals.css"; const inter = Noto_Sans({ subsets: ["latin"] }); export const metadata: Metadata = { - title: "Khoj AI - Chat", - description: "Use this page to chat with Khoj AI.", + title: "Khoj AI - Home", + description: "Your open, personal AI.", + icons: { + icon: '/static/favicon.ico', + }, }; export default function RootLayout({ diff --git a/src/interface/web/app/search/layout.tsx b/src/interface/web/app/search/layout.tsx index ed06f884..bbaa37a9 100644 --- a/src/interface/web/app/search/layout.tsx +++ b/src/interface/web/app/search/layout.tsx @@ -5,7 +5,7 @@ import "../globals.css"; export const metadata: Metadata = { title: "Khoj AI - Search", - description: "Search through all the documents you've shared with Khoj AI using natural language queries.", + description: "Find anything in documents you've shared with Khoj using natural language queries.", icons: { icon: '/static/favicon.ico', }, From cab0957fd3e6d05908c8b6e09b967cc09fded6aa Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Fri, 2 Aug 2024 12:17:11 +0530 Subject: [PATCH 13/16] Just show Khoj logo on title bar on small screens Continue to show logo + text on larger screens --- .../web/app/components/logo/khogLogo.tsx | 30 ++++++++++++++++++- .../web/app/components/navMenu/navMenu.tsx | 4 +-- .../sidePanel/chatHistorySidePanel.tsx | 9 ++---- src/interface/web/public/logo.svg | 24 +++++++++++++++ 4 files changed, 58 insertions(+), 9 deletions(-) create mode 100644 src/interface/web/public/logo.svg diff --git a/src/interface/web/app/components/logo/khogLogo.tsx b/src/interface/web/app/components/logo/khogLogo.tsx index a53e122a..c2cdc8c1 100644 --- a/src/interface/web/app/components/logo/khogLogo.tsx +++ b/src/interface/web/app/components/logo/khogLogo.tsx @@ -1,4 +1,4 @@ -export function KhojLogo() { +export function KhojLogoType() { return ( @@ -29,3 +29,31 @@ export function KhojLogo() { ); } + +export function KhojLogo() { + return ( + + + + + + + + + + + + + + + + + + + + + + + + ); +} diff --git a/src/interface/web/app/components/navMenu/navMenu.tsx b/src/interface/web/app/components/navMenu/navMenu.tsx index 0f8d78a8..386a3afc 100644 --- a/src/interface/web/app/components/navMenu/navMenu.tsx +++ b/src/interface/web/app/components/navMenu/navMenu.tsx @@ -23,7 +23,7 @@ import { DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { Moon, Sun, UserCircle, User, Robot, MagnifyingGlass, Question, GearFine, ArrowRight } from '@phosphor-icons/react'; -import { KhojLogo } from '../logo/khogLogo'; +import { KhojLogoType } from '../logo/khogLogo'; interface NavMenuProps { @@ -99,7 +99,7 @@ export default function NavMenu(props: NavMenuProps) { { !displayTitle && props.showLogo && - + }
diff --git a/src/interface/web/app/components/sidePanel/chatHistorySidePanel.tsx b/src/interface/web/app/components/sidePanel/chatHistorySidePanel.tsx index b9209b49..b02ea754 100644 --- a/src/interface/web/app/components/sidePanel/chatHistorySidePanel.tsx +++ b/src/interface/web/app/components/sidePanel/chatHistorySidePanel.tsx @@ -5,10 +5,8 @@ import styles from "./sidePanel.module.css"; import { useEffect, useState } from "react"; import { UserProfile, useAuthenticatedData } from "@/app/common/auth"; -import { Avatar, AvatarImage, AvatarFallback } from "@/components/ui/avatar"; import Link from "next/link"; import useSWR from "swr"; -import Image from "next/image"; import { Command, @@ -72,13 +70,12 @@ import { import { Pencil, Trash, Share } from "@phosphor-icons/react"; -import { Button, buttonVariants } from "@/components/ui/button"; +import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; -import { Label } from "@/components/ui/label"; import { AlertDialog, AlertDialogAction, AlertDialogCancel, AlertDialogContent, AlertDialogDescription, AlertDialogFooter, AlertDialogHeader, AlertDialogTitle, AlertDialogTrigger } from "@/components/ui/alert-dialog"; import { modifyFileFilterForConversation } from "@/app/common/chatFunctions"; import { ScrollAreaScrollbar } from "@radix-ui/react-scroll-area"; -import { KhojLogo } from "../logo/khogLogo"; +import { KhojLogo, KhojLogoType } from "@/app/components/logo/khogLogo"; // Define a fetcher function const fetcher = (url: string) => fetch(url).then((res) => res.json()); @@ -666,7 +663,7 @@ export default function SidePanel(props: SidePanelProps) {
- + {props.isMobileWidth && || } { authenticatedData && props.isMobileWidth ? diff --git a/src/interface/web/public/logo.svg b/src/interface/web/public/logo.svg new file mode 100644 index 00000000..8c6f62b2 --- /dev/null +++ b/src/interface/web/public/logo.svg @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + From 7858aff2e2ace3fb6b661d94fe0fed5243fee697 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Fri, 2 Aug 2024 01:57:30 +0530 Subject: [PATCH 14/16] Trigger welcomeConsole only once on chat, shared chat page load --- src/interface/web/app/chat/page.tsx | 3 ++- src/interface/web/app/share/chat/page.tsx | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/interface/web/app/chat/page.tsx b/src/interface/web/app/chat/page.tsx index fe2f9a23..9a8ee838 100644 --- a/src/interface/web/app/chat/page.tsx +++ b/src/interface/web/app/chat/page.tsx @@ -112,7 +112,6 @@ export default function Chat() { const locationData = useIPLocationData(); const authenticatedData = useAuthenticatedData(); - welcomeConsole(); useEffect(() => { fetch('/api/chat/options') @@ -129,6 +128,8 @@ export default function Chat() { return; }); + welcomeConsole(); + setIsMobileWidth(window.innerWidth < 786); window.addEventListener('resize', () => { diff --git a/src/interface/web/app/share/chat/page.tsx b/src/interface/web/app/share/chat/page.tsx index 4a08f0a2..1d41fb1d 100644 --- a/src/interface/web/app/share/chat/page.tsx +++ b/src/interface/web/app/share/chat/page.tsx @@ -105,9 +105,6 @@ export default function SharedChat() { const locationData = useIPLocationData(); const authenticatedData = useAuthenticatedData(); - welcomeConsole(); - - useEffect(() => { fetch('/api/chat/options') .then(response => response.json()) @@ -123,6 +120,8 @@ export default function SharedChat() { return; }); + welcomeConsole(); + setIsMobileWidth(window.innerWidth < 786); window.addEventListener('resize', () => { From a733e5c1d458e7b99eeed54c58baea1219a79568 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Fri, 2 Aug 2024 01:57:56 +0530 Subject: [PATCH 15/16] Remove unused handleCompiledReferences chat functions --- src/interface/web/app/common/chatFunctions.ts | 18 ------------------ src/interface/web/app/share/chat/page.tsx | 2 +- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/src/interface/web/app/common/chatFunctions.ts b/src/interface/web/app/common/chatFunctions.ts index 51a21115..17d6aec9 100644 --- a/src/interface/web/app/common/chatFunctions.ts +++ b/src/interface/web/app/common/chatFunctions.ts @@ -15,24 +15,6 @@ export interface ResponseWithReferences { response?: string; } -export function handleCompiledReferences(chunk: string, currentResponse: string) { - const rawReference = chunk.split("### compiled references:")[1]; - const rawResponse = chunk.split("### compiled references:")[0]; - let references: ResponseWithReferences = {}; - - // Set the initial response - references.response = currentResponse + rawResponse; - - const rawReferenceAsJson = JSON.parse(rawReference); - if (rawReferenceAsJson instanceof Array) { - references.context = rawReferenceAsJson; - } else if (typeof rawReferenceAsJson === "object" && rawReferenceAsJson !== null) { - references.online = rawReferenceAsJson; - } - - return references; -} - interface MessageChunk { type: string; data: string | object; diff --git a/src/interface/web/app/share/chat/page.tsx b/src/interface/web/app/share/chat/page.tsx index 1d41fb1d..c763248b 100644 --- a/src/interface/web/app/share/chat/page.tsx +++ b/src/interface/web/app/share/chat/page.tsx @@ -15,7 +15,7 @@ import { useAuthenticatedData } from '@/app/common/auth'; import ChatInputArea, { ChatOptions } from '@/app/components/chatInputArea/chatInputArea'; import { StreamMessage } from '@/app/components/chatMessage/chatMessage'; -import { convertMessageChunkToJson, handleCompiledReferences, handleImageResponse, processMessageChunk, RawReferenceData } from '@/app/common/chatFunctions'; +import { processMessageChunk } from '@/app/common/chatFunctions'; import { AgentData } from '@/app/agents/page'; From 02b46a1784ec74c447d5237e8b759f30442f58b3 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Fri, 2 Aug 2024 02:33:04 +0530 Subject: [PATCH 16/16] Render references after chat response is streamed for smoother render Otherwise the Khoj's chat response is filling up in between the streamed message and already rendered references section at the bottom of the message Define OnlineContext type to simplify typing online context param across other interfaces and functions --- src/interface/web/app/chat/page.tsx | 19 +++-- src/interface/web/app/common/chatFunctions.ts | 76 ++++++++++--------- .../components/chatMessage/chatMessage.tsx | 14 ++-- .../referencePanel/referencePanel.tsx | 4 +- src/interface/web/app/factchecker/page.tsx | 6 +- 5 files changed, 60 insertions(+), 59 deletions(-) diff --git a/src/interface/web/app/chat/page.tsx b/src/interface/web/app/chat/page.tsx index 9a8ee838..1768a228 100644 --- a/src/interface/web/app/chat/page.tsx +++ b/src/interface/web/app/chat/page.tsx @@ -9,11 +9,11 @@ import NavMenu from '../components/navMenu/navMenu'; import { useSearchParams } from 'next/navigation' import Loading from '../components/loading/loading'; -import { convertMessageChunkToJson, handleImageResponse, processMessageChunk, RawReferenceData } from '../common/chatFunctions'; +import { processMessageChunk } from '../common/chatFunctions'; import 'katex/dist/katex.min.css'; -import { StreamMessage } from '../components/chatMessage/chatMessage'; +import { Context, OnlineContext, StreamMessage } from '../components/chatMessage/chatMessage'; import { useIPLocationData, welcomeConsole } from '../common/utils'; import ChatInputArea, { ChatOptions } from '../components/chatInputArea/chatInputArea'; import { useAuthenticatedData } from '../common/auth'; @@ -110,7 +110,6 @@ export default function Chat() { const [uploadedFiles, setUploadedFiles] = useState([]); const [isMobileWidth, setIsMobileWidth] = useState(false); const locationData = useIPLocationData(); - const authenticatedData = useAuthenticatedData(); useEffect(() => { @@ -169,8 +168,11 @@ export default function Chat() { const eventDelimiter = '␃🔚␗'; let buffer = ""; - while (true) { + // Track context used for chat response + let context: Context[] = []; + let onlineContext: OnlineContext = {}; + while (true) { const { done, value } = await reader.read(); if (done) { setQueryToProcess(''); @@ -179,7 +181,6 @@ export default function Chat() { } const chunk = decoder.decode(value, { stream: true }); - buffer += chunk; let newEventIndex; @@ -194,7 +195,8 @@ export default function Chat() { return; } - processMessageChunk(event, currentMessage); + // Track context used for chat response. References are rendered at the end of the chat + ({context, onlineContext} = processMessageChunk(event, currentMessage, context, onlineContext)); setMessages([...messages]); } @@ -222,10 +224,7 @@ export default function Chat() { setConversationID(newConversationId); }; - if (isLoading) { - return ; - } - + if (isLoading) return ; return (
diff --git a/src/interface/web/app/common/chatFunctions.ts b/src/interface/web/app/common/chatFunctions.ts index 17d6aec9..25742eec 100644 --- a/src/interface/web/app/common/chatFunctions.ts +++ b/src/interface/web/app/common/chatFunctions.ts @@ -1,17 +1,13 @@ -import { Context, OnlineContextData, StreamMessage } from "../components/chatMessage/chatMessage"; +import { Context, OnlineContext, StreamMessage } from "../components/chatMessage/chatMessage"; export interface RawReferenceData { context?: Context[]; - onlineContext?: { - [key: string]: OnlineContextData - } + onlineContext?: OnlineContext; } export interface ResponseWithReferences { context?: Context[]; - online?: { - [key: string]: OnlineContextData - } + online?: OnlineContext; response?: string; } @@ -50,57 +46,65 @@ export function convertMessageChunkToJson(chunk: string): MessageChunk { } } +function handleJsonResponse(chunkData: any) { + const jsonData = chunkData as any; + if (jsonData.image || jsonData.detail) { + let responseWithReference = handleImageResponse(chunkData, true); + if (responseWithReference.response) return responseWithReference.response; + } else if (jsonData.response) { + return jsonData.response; + } else { + throw new Error("Invalid JSON response"); + } +} + +export function processMessageChunk( + rawChunk: string, + currentMessage: StreamMessage, + context: Context[] = [], + onlineContext: OnlineContext = {}): { context: Context[], onlineContext: OnlineContext } { -export function processMessageChunk(rawChunk: string, currentMessage: StreamMessage) { const chunk = convertMessageChunkToJson(rawChunk); - if (!currentMessage) { - return; - } - - if (!chunk || !chunk.type) { - return; - } + if (!currentMessage || !chunk || !chunk.type) return {context, onlineContext}; if (chunk.type === "status") { + console.log(`status: ${chunk.data}`); const statusMessage = chunk.data as string; currentMessage.trainOfThought.push(statusMessage); } else if (chunk.type === "references") { const references = chunk.data as RawReferenceData; - if (references.context) { - currentMessage.context = references.context; - } - - if (references.onlineContext) { - currentMessage.onlineContext = references.onlineContext; - } + if (references.context) context = references.context; + if (references.onlineContext) onlineContext = references.onlineContext; + return {context, onlineContext} } else if (chunk.type === "message") { const chunkData = chunk.data; - if (chunkData !== null && typeof chunkData === 'object') { + currentMessage.rawResponse += handleJsonResponse(chunkData); + } else if (typeof chunkData === 'string' && chunkData.trim()?.startsWith("{") && chunkData.trim()?.endsWith("}")) { try { - const jsonData = chunkData as any; - if (jsonData.image || jsonData.detail) { - let responseWithReference = handleImageResponse(chunk.data, true); - if (responseWithReference.response) currentMessage.rawResponse = responseWithReference.response; - if (responseWithReference.online) currentMessage.onlineContext = responseWithReference.online; - if (responseWithReference.context) currentMessage.context = responseWithReference.context; - } else if (jsonData.response) { - currentMessage.rawResponse = jsonData.response; - } - else { - console.debug("any message", chunk); - } + const jsonData = JSON.parse(chunkData.trim()); + currentMessage.rawResponse += handleJsonResponse(jsonData); } catch (e) { - currentMessage.rawResponse += chunkData; + currentMessage.rawResponse += JSON.stringify(chunkData); } } else { currentMessage.rawResponse += chunkData; } + } else if (chunk.type === "start_llm_response") { + console.log(`Started streaming: ${new Date()}`); } else if (chunk.type === "end_llm_response") { + console.log(`Completed streaming: ${new Date()}`); + + // Append any references after all the data has been streamed + if (onlineContext) currentMessage.onlineContext = onlineContext; + if (context) currentMessage.context = context; + + // Mark current message streaming as completed currentMessage.completed = true; } + return {context, onlineContext}; } export function handleImageResponse(imageJson: any, liveStream: boolean): ResponseWithReferences { diff --git a/src/interface/web/app/components/chatMessage/chatMessage.tsx b/src/interface/web/app/components/chatMessage/chatMessage.tsx index b8187f4c..71a5b29e 100644 --- a/src/interface/web/app/components/chatMessage/chatMessage.tsx +++ b/src/interface/web/app/components/chatMessage/chatMessage.tsx @@ -33,6 +33,10 @@ export interface Context { file: string; } +export interface OnlineContext { + [key: string]: OnlineContextData; +} + export interface WebPage { link: string; query: string; @@ -85,11 +89,9 @@ export interface SingleChatMessage { automationId: string; by: string; message: string; - context: Context[]; created: string; - onlineContext: { - [key: string]: OnlineContextData - } + context: Context[]; + onlineContext: OnlineContext; rawQuery?: string; intent?: Intent; agent?: AgentData; @@ -99,9 +101,7 @@ export interface StreamMessage { rawResponse: string; trainOfThought: string[]; context: Context[]; - onlineContext: { - [key: string]: OnlineContextData - } + onlineContext: OnlineContext; completed: boolean; rawQuery: string; timestamp: string; diff --git a/src/interface/web/app/components/referencePanel/referencePanel.tsx b/src/interface/web/app/components/referencePanel/referencePanel.tsx index 75345904..5fc70406 100644 --- a/src/interface/web/app/components/referencePanel/referencePanel.tsx +++ b/src/interface/web/app/components/referencePanel/referencePanel.tsx @@ -11,7 +11,7 @@ const md = new markdownIt({ typographer: true }); -import { Context, WebPage, OnlineContextData } from "../chatMessage/chatMessage"; +import { Context, WebPage, OnlineContext } from "../chatMessage/chatMessage"; import { Card } from "@/components/ui/card"; import { @@ -161,7 +161,7 @@ function GenericOnlineReferenceCard(props: OnlineReferenceCardProps) { ) } -export function constructAllReferences(contextData: Context[], onlineData: { [key: string]: OnlineContextData }) { +export function constructAllReferences(contextData: Context[], onlineData: OnlineContext) { const onlineReferences: OnlineReferenceData[] = []; const contextReferences: NotesContextReferenceData[] = []; diff --git a/src/interface/web/app/factchecker/page.tsx b/src/interface/web/app/factchecker/page.tsx index 5f2a9845..bb6e5af2 100644 --- a/src/interface/web/app/factchecker/page.tsx +++ b/src/interface/web/app/factchecker/page.tsx @@ -4,7 +4,7 @@ import styles from './factChecker.module.css'; import { useAuthenticatedData } from '@/app/common/auth'; import { useState, useEffect } from 'react'; -import ChatMessage, { Context, OnlineContextData, WebPage } from '../components/chatMessage/chatMessage'; +import ChatMessage, { Context, OnlineContext, OnlineContextData, WebPage } from '../components/chatMessage/chatMessage'; import { ModelPicker, Model } from '../components/modelPicker/modelPicker'; import ShareLink from '../components/shareLink/shareLink'; @@ -47,9 +47,7 @@ interface SupplementReferences { interface ResponseWithReferences { context?: Context[]; - online?: { - [key: string]: OnlineContextData - } + online?: OnlineContext; response?: string; }