klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-04 05:39:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into multi-image-chat-and-vision-for-gemini

Browse Source
This commit is contained in:
Debanjum Singh Solanky
2024-10-22 18:29:44 -07:00
parent e8fb79a369 0dad4212fa
commit 6c393800cc
43 changed files with 1080 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/interface/desktop/chat.html
View File

@@ -326,7 +326,7 @@
entries.forEach(entry => {
// If the element is in the viewport, fetch the remaining message and unobserve the element
if (entry.isIntersecting) {
fetchRemainingChatMessages(chatHistoryUrl, headers);
fetchRemainingChatMessages(chatHistoryUrl, headers, chatBody.dataset.conversation_id, hostURL);
observer.unobserve(entry.target);
}
});
@@ -342,7 +342,11 @@
new Date(chat_log.created),
chat_log.onlineContext,
chat_log.intent?.type,
chat_log.intent?.["inferred-queries"]);
chat_log.intent?.["inferred-queries"],
chatBody.dataset.conversationId ?? "",
hostURL,
);
chatBody.appendChild(messageElement);
// When the 4th oldest message is within viewing distance (~60% scrolled up)
@@ -421,7 +425,7 @@
}
}
function fetchRemainingChatMessages(chatHistoryUrl, headers) {
function fetchRemainingChatMessages(chatHistoryUrl, headers, conversationId, hostURL) {
// Create a new IntersectionObserver
let observer = new IntersectionObserver((entries, observer) => {
entries.forEach(entry => {
@@ -435,7 +439,9 @@
new Date(chat_log.created),
chat_log.onlineContext,
chat_log.intent?.type,
chat_log.intent?.["inferred-queries"]
chat_log.intent?.["inferred-queries"],
chatBody.dataset.conversationId ?? "",
hostURL,
);
entry.target.replaceWith(messageElement);

15
src/interface/desktop/chatutils.js
View File

@@ -189,11 +189,19 @@ function processOnlineReferences(referenceSection, onlineContext) { //same
return numOnlineReferences;
}
function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null, intentType=null, inferredQueries=null) { //same
function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null, intentType=null, inferredQueries=null, conversationId=null, hostURL=null) {
let chatEl;
if (intentType?.includes("text-to-image")) {
let imageMarkdown = generateImageMarkdown(message, intentType, inferredQueries);
chatEl = renderMessage(imageMarkdown, by, dt, null, false, "return");
} else if (intentType === "excalidraw") {
let domain = hostURL ?? "https://app.khoj.dev/";
if (!domain.endsWith("/")) domain += "/";
let excalidrawMessage = `Hey, I'm not ready to show you diagrams yet here. But you can view it in the web app at ${domain}chat?conversationId=${conversationId}`;
chatEl = renderMessage(excalidrawMessage, by, dt, null, false, "return");
} else {
chatEl = renderMessage(message, by, dt, null, false, "return");
}
@@ -312,7 +320,6 @@ function formatHTMLMessage(message, raw=false, willReplace=true) { //same
}
function createReferenceSection(references, createLinkerSection=false) {
console.log("linker data: ", createLinkerSection);
let referenceSection = document.createElement('div');
referenceSection.classList.add("reference-section");
referenceSection.classList.add("collapsed");
@@ -417,7 +424,11 @@ function handleImageResponse(imageJson, rawResponse) {
rawResponse += `![generated_image](${imageJson.image})`;
} else if (imageJson.intentType === "text-to-image-v3") {
rawResponse = `![](data:image/webp;base64,${imageJson.image})`;
} else if (imageJson.intentType === "excalidraw") {
const redirectMessage = `Hey, I'm not ready to show you diagrams yet here. But you can view it in the web app`;
rawResponse += redirectMessage;
}
if (inferredQuery) {
rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
}

2
src/interface/desktop/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "Khoj",
"version": "1.26.0",
"version": "1.26.4",
"description": "Your Second Brain",
"author": "Khoj Inc. <team@khoj.dev>",
"license": "GPL-3.0-or-later",

85
src/interface/emacs/khoj.el
View File

@@ -6,7 +6,7 @@
;; Saba Imran <saba@khoj.dev>
;; Description: Your Second Brain
;; Keywords: search, chat, ai, org-mode, outlines, markdown, pdf, image
;; Version: 1.26.0
;; Version: 1.26.4
;; Package-Requires: ((emacs "27.1") (transient "0.3.0") (dash "2.19.1"))
;; URL: https://github.com/khoj-ai/khoj/tree/master/src/interface/emacs
@@ -127,6 +127,11 @@
(const "image")
(const "pdf")))
(defcustom khoj-default-agent "khoj"
"The default agent to chat with. See https://app.khoj.dev/agents for available options."
:group 'khoj
:type 'string)
;; --------------------------
;; Khoj Dynamic Configuration
@@ -144,6 +149,9 @@
(defconst khoj--chat-buffer-name "*🏮 Khoj Chat*"
"Name of chat buffer for Khoj.")
(defvar khoj--selected-agent khoj-default-agent
"Currently selected Khoj agent.")
(defvar khoj--content-type "org"
"The type of content to perform search on.")
@@ -656,13 +664,15 @@ Simplified fork of `org-cycle-content' from Emacs 29.1 to work with >=27.1."
;; --------------
;; Query Khoj API
;; --------------
(defun khoj--call-api (path &optional method params callback &rest cbargs)
"Sync call API at PATH with METHOD and query PARAMS as kv assoc list.
(defun khoj--call-api (path &optional method params body callback &rest cbargs)
"Sync call API at PATH with METHOD, query PARAMS and BODY as kv assoc list.
Optionally apply CALLBACK with JSON parsed response and CBARGS."
(let* ((url-request-method (or method "GET"))
(url-request-extra-headers `(("Authorization" . ,(format "Bearer %s" khoj-api-key))))
(param-string (if params (url-build-query-string params) ""))
(query-url (format "%s%s?%s&client=emacs" khoj-server-url path param-string))
(url-request-extra-headers `(("Authorization" . ,(format "Bearer %s" khoj-api-key)) ("Content-Type" . "application/json")))
(url-request-data (if body (json-encode body) nil))
(param-string (url-build-query-string (append params '((client "emacs")))))
(query-url (format "%s%s?%s" khoj-server-url path param-string))
(cbargs (if (and (listp cbargs) (listp (car cbargs))) (car cbargs) cbargs))) ; normalize cbargs to (a b) from ((a b)) if required
(with-temp-buffer
(condition-case ex
@@ -682,8 +692,8 @@ Optionally apply CALLBACK with JSON parsed response and CBARGS."
(url-request-extra-headers `(("Authorization" . ,(format "Bearer %s" khoj-api-key)) ("Content-Type" . "application/json")))
(url-request-data (if body (json-encode body) nil))
(param-string (url-build-query-string (append params '((client "emacs")))))
(cbargs (if (and (listp cbargs) (listp (car cbargs))) (car cbargs) cbargs)) ; normalize cbargs to (a b) from ((a b)) if required
(query-url (format "%s%s?%s" khoj-server-url path param-string)))
(query-url (format "%s%s?%s" khoj-server-url path param-string))
(cbargs (if (and (listp cbargs) (listp (car cbargs))) (car cbargs) cbargs))) ; normalize cbargs to (a b) from ((a b)) if required
(url-retrieve query-url
(lambda (status)
(if (plist-get status :error)
@@ -699,7 +709,7 @@ Optionally apply CALLBACK with JSON parsed response and CBARGS."
(defun khoj--get-enabled-content-types ()
"Get content types enabled for search from API."
(khoj--call-api "/api/content/types" "GET" nil `(lambda (item) (mapcar #'intern item))))
(khoj--call-api "/api/content/types" "GET" nil nil `(lambda (item) (mapcar #'intern item))))
(defun khoj--query-search-api-and-render-results (query content-type buffer-name &optional rerank is-find-similar)
"Query Khoj Search API with QUERY, CONTENT-TYPE and RERANK as query params.
@@ -913,14 +923,16 @@ Call CALLBACK func with response and CBARGS."
(let ((selected-session-id (khoj--select-conversation-session "Open")))
(khoj--load-chat-session khoj--chat-buffer-name selected-session-id)))
(defun khoj--create-chat-session ()
"Create new chat session."
(khoj--call-api "/api/chat/sessions" "POST"))
(defun khoj--create-chat-session (&optional agent)
"Create new chat session with AGENT."
(khoj--call-api "/api/chat/sessions"
"POST"
(when agent `(("agent_slug" ,agent)))))
(defun khoj--new-conversation-session ()
"Create new Khoj conversation session."
(defun khoj--new-conversation-session (&optional agent)
"Create new Khoj conversation session with AGENT."
(thread-last
(khoj--create-chat-session)
(khoj--create-chat-session agent)
(assoc 'conversation_id)
(cdr)
(khoj--chat)))
@@ -935,6 +947,15 @@ Call CALLBACK func with response and CBARGS."
(khoj--select-conversation-session "Delete")
(khoj--delete-chat-session)))
(defun khoj--get-agents ()
"Get list of available Khoj agents."
(let* ((response (khoj--call-api "/api/agents" "GET"))
(agents (mapcar (lambda (agent)
(cons (cdr (assoc 'name agent))
(cdr (assoc 'slug agent))))
response)))
agents))
(defun khoj--render-chat-message (message sender &optional receive-date)
"Render chat messages as `org-mode' list item.
MESSAGE is the text of the chat message.
@@ -1246,6 +1267,20 @@ Paragraph only starts at first text after blank line."
;; dynamically set choices to content types enabled on khoj backend
:choices (or (ignore-errors (mapcar #'symbol-name (khoj--get-enabled-content-types))) '("all" "org" "markdown" "pdf" "image")))
(transient-define-argument khoj--agent-switch ()
:class 'transient-switches
:argument-format "--agent=%s"
:argument-regexp ".+"
:init-value (lambda (obj)
(oset obj value (format "--agent=%s" khoj--selected-agent)))
:choices (or (ignore-errors (mapcar #'cdr (khoj--get-agents))) '("khoj"))
:reader (lambda (prompt initial-input history)
(let* ((agents (khoj--get-agents))
(selected (completing-read prompt agents nil t initial-input history))
(slug (cdr (assoc selected agents))))
(setq khoj--selected-agent slug)
slug)))
(transient-define-suffix khoj--search-command (&optional args)
(interactive (list (transient-args transient-current-command)))
(progn
@@ -1287,10 +1322,11 @@ Paragraph only starts at first text after blank line."
(interactive (list (transient-args transient-current-command)))
(khoj--open-conversation-session))
(transient-define-suffix khoj--new-conversation-session-command (&optional _)
(transient-define-suffix khoj--new-conversation-session-command (&optional args)
"Command to select Khoj conversation sessions to open."
(interactive (list (transient-args transient-current-command)))
(khoj--new-conversation-session))
(let ((agent-slug (transient-arg-value "--agent=" args)))
(khoj--new-conversation-session agent-slug)))
(transient-define-suffix khoj--delete-conversation-session-command (&optional _)
"Command to select Khoj conversation sessions to delete."
@@ -1298,14 +1334,15 @@ Paragraph only starts at first text after blank line."
(khoj--delete-conversation-session))
(transient-define-prefix khoj--chat-menu ()
"Open the Khoj chat menu."
["Act"
("c" "Chat" khoj--chat-command)
("o" "Open Conversation" khoj--open-conversation-session-command)
("n" "New Conversation" khoj--new-conversation-session-command)
("d" "Delete Conversation" khoj--delete-conversation-session-command)
("q" "Quit" transient-quit-one)
])
"Create the Khoj Chat Menu and Execute Commands."
[["Configure"
("a" "Select Agent" khoj--agent-switch)]]
[["Act"
("c" "Chat" khoj--chat-command)
("o" "Open Conversation" khoj--open-conversation-session-command)
("n" "New Conversation" khoj--new-conversation-session-command)
("d" "Delete Conversation" khoj--delete-conversation-session-command)
("q" "Quit" transient-quit-one)]])
(transient-define-prefix khoj--menu ()
"Create Khoj Menu to Configure and Execute Commands."

2
src/interface/obsidian/manifest.json
View File

@@ -1,7 +1,7 @@
{
"id": "khoj",
"name": "Khoj",
"version": "1.26.0",
"version": "1.26.4",
"minAppVersion": "0.15.0",
"description": "Your Second Brain",
"author": "Khoj Inc.",

2
src/interface/obsidian/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "Khoj",
"version": "1.26.0",
"version": "1.26.4",
"description": "Your Second Brain",
"author": "Debanjum Singh Solanky, Saba Imran <team@khoj.dev>",
"license": "GPL-3.0-or-later",

16
src/interface/obsidian/src/chat_view.ts
View File

@@ -484,12 +484,13 @@ export class KhojChatView extends KhojPaneView {
dt?: Date,
intentType?: string,
inferredQueries?: string[],
conversationId?: string,
) {
if (!message) return;
let chatMessageEl;
if (intentType?.includes("text-to-image")) {
let imageMarkdown = this.generateImageMarkdown(message, intentType, inferredQueries);
if (intentType?.includes("text-to-image") || intentType === "excalidraw") {
let imageMarkdown = this.generateImageMarkdown(message, intentType, inferredQueries, conversationId);
chatMessageEl = this.renderMessage(chatEl, imageMarkdown, sender, dt);
} else {
chatMessageEl = this.renderMessage(chatEl, message, sender, dt);
@@ -509,7 +510,7 @@ export class KhojChatView extends KhojPaneView {
chatMessageBodyEl.appendChild(this.createReferenceSection(references));
}
generateImageMarkdown(message: string, intentType: string, inferredQueries?: string[]) {
generateImageMarkdown(message: string, intentType: string, inferredQueries?: string[], conversationId?: string): string {
let imageMarkdown = "";
if (intentType === "text-to-image") {
imageMarkdown = `![](data:image/png;base64,${message})`;
@@ -517,6 +518,10 @@ export class KhojChatView extends KhojPaneView {
imageMarkdown = `![](${message})`;
} else if (intentType === "text-to-image-v3") {
imageMarkdown = `![](data:image/webp;base64,${message})`;
} else if (intentType === "excalidraw") {
const domain = this.setting.khojUrl.endsWith("/") ? this.setting.khojUrl : `${this.setting.khojUrl}/`;
const redirectMessage = `Hey, I'm not ready to show you diagrams yet here. But you can view it in ${domain}chat?conversationId=${conversationId}`;
imageMarkdown = redirectMessage;
}
if (inferredQueries) {
imageMarkdown += "\n\n**Inferred Query**:";
@@ -884,6 +889,7 @@ export class KhojChatView extends KhojPaneView {
new Date(chatLog.created),
chatLog.intent?.type,
chatLog.intent?.["inferred-queries"],
chatBodyEl.dataset.conversationId ?? "",
);
// push the user messages to the chat history
if(chatLog.by === "you"){
@@ -1354,6 +1360,10 @@ export class KhojChatView extends KhojPaneView {
rawResponse += `![generated_image](${imageJson.image})`;
} else if (imageJson.intentType === "text-to-image-v3") {
rawResponse = `![](data:image/webp;base64,${imageJson.image})`;
} else if (imageJson.intentType === "excalidraw") {
const domain = this.setting.khojUrl.endsWith("/") ? this.setting.khojUrl : `${this.setting.khojUrl}/`;
const redirectMessage = `Hey, I'm not ready to show you diagrams yet here. But you can view it in ${domain}`;
rawResponse += redirectMessage;
}
if (inferredQuery) {
rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`;

6
src/interface/obsidian/versions.json
View File

@@ -78,5 +78,9 @@
"1.24.0": "0.15.0",
"1.24.1": "0.15.0",
"1.25.0": "0.15.0",
"1.26.0": "0.15.0"
"1.26.0": "0.15.0",
"1.26.1": "0.15.0",
"1.26.2": "0.15.0",
"1.26.3": "0.15.0",
"1.26.4": "0.15.0"
}

2
src/interface/web/app/chat/chat.module.css
View File

@@ -79,7 +79,7 @@ div.titleBar {
div.chatBoxBody {
display: grid;
height: 100%;
width: 70%;
width: 95%;
margin: auto;
}

9
src/interface/web/app/chat/layout.tsx
View File

@@ -47,7 +47,14 @@ export default function RootLayout({
child-src 'none';
object-src 'none';"
></meta>
<body className={inter.className}>{children}</body>
<body className={inter.className}>
{children}
<script
dangerouslySetInnerHTML={{
__html: `window.EXCALIDRAW_ASSET_PATH = 'https://assets.khoj.dev/@excalidraw/excalidraw/dist/';`,
}}
/>
</body>
</html>
);
}

5
src/interface/web/app/chat/page.tsx
View File

@@ -41,6 +41,8 @@ function ChatBodyData(props: ChatBodyDataProps) {
const setQueryToProcess = props.setQueryToProcess;
const onConversationIdChange = props.onConversationIdChange;
const chatHistoryCustomClassName = props.isMobileWidth ? "w-full" : "w-4/6";
useEffect(() => {
if (images.length > 0) {
const encodedImages = images.map((image) => encodeURIComponent(image));
@@ -105,10 +107,11 @@ function ChatBodyData(props: ChatBodyDataProps) {
setAgent={setAgentMetadata}
pendingMessage={processingMessage ? message : ""}
incomingMessages={props.streamedMessages}
customClassName={chatHistoryCustomClassName}
/>
</div>
<div
className={`${styles.inputBox} p-1 md:px-2 shadow-md bg-background align-middle items-center justify-center dark:bg-neutral-700 dark:border-0 dark:shadow-sm rounded-t-2xl rounded-b-none md:rounded-xl h-fit`}
className={`${styles.inputBox} p-1 md:px-2 shadow-md bg-background align-middle items-center justify-center dark:bg-neutral-700 dark:border-0 dark:shadow-sm rounded-t-2xl rounded-b-none md:rounded-xl h-fit ${chatHistoryCustomClassName} mr-auto ml-auto`}
>
<ChatInputArea
agentColor={agentMetadata?.color}

69
src/interface/web/app/common/chatFunctions.ts
View File

@@ -5,10 +5,10 @@ export interface RawReferenceData {
onlineContext?: OnlineContext;
}
export interface ResponseWithReferences {
context?: Context[];
online?: OnlineContext;
response?: string;
export interface ResponseWithIntent {
intentType: string;
response: string;
inferredQueries?: string[];
}
interface MessageChunk {
@@ -49,10 +49,14 @@ export function convertMessageChunkToJson(chunk: string): MessageChunk {
function handleJsonResponse(chunkData: any) {
const jsonData = chunkData as any;
if (jsonData.image || jsonData.detail) {
let responseWithReference = handleImageResponse(chunkData, true);
if (responseWithReference.response) return responseWithReference.response;
let responseWithIntent = handleImageResponse(chunkData, true);
return responseWithIntent;
} else if (jsonData.response) {
return jsonData.response;
return {
response: jsonData.response,
intentType: "",
inferredQueries: [],
};
} else {
throw new Error("Invalid JSON response");
}
@@ -80,8 +84,18 @@ export function processMessageChunk(
return { context, onlineContext };
} else if (chunk.type === "message") {
const chunkData = chunk.data;
// Here, handle if the response is a JSON response with an image, but the intentType is excalidraw
if (chunkData !== null && typeof chunkData === "object") {
currentMessage.rawResponse += handleJsonResponse(chunkData);
let responseWithIntent = handleJsonResponse(chunkData);
if (responseWithIntent.intentType && responseWithIntent.intentType === "excalidraw") {
currentMessage.rawResponse = responseWithIntent.response;
} else {
currentMessage.rawResponse += responseWithIntent.response;
}
currentMessage.intentType = responseWithIntent.intentType;
currentMessage.inferredQueries = responseWithIntent.inferredQueries;
} else if (
typeof chunkData === "string" &&
chunkData.trim()?.startsWith("{") &&
@@ -89,7 +103,10 @@ export function processMessageChunk(
) {
try {
const jsonData = JSON.parse(chunkData.trim());
currentMessage.rawResponse += handleJsonResponse(jsonData);
let responseWithIntent = handleJsonResponse(jsonData);
currentMessage.rawResponse += responseWithIntent.response;
currentMessage.intentType = responseWithIntent.intentType;
currentMessage.inferredQueries = responseWithIntent.inferredQueries;
} catch (e) {
currentMessage.rawResponse += JSON.stringify(chunkData);
}
@@ -111,42 +128,26 @@ export function processMessageChunk(
return { context, onlineContext };
}
export function handleImageResponse(imageJson: any, liveStream: boolean): ResponseWithReferences {
export function handleImageResponse(imageJson: any, liveStream: boolean): ResponseWithIntent {
let rawResponse = "";
if (imageJson.image) {
const inferredQuery = imageJson.inferredQueries?.[0] ?? "generated image";
// If response has image field, response is a generated image.
if (imageJson.intentType === "text-to-image") {
rawResponse += `![generated_image](data:image/png;base64,${imageJson.image})`;
} else if (imageJson.intentType === "text-to-image2") {
rawResponse += `![generated_image](${imageJson.image})`;
} else if (imageJson.intentType === "text-to-image-v3") {
rawResponse = `![](data:image/webp;base64,${imageJson.image})`;
}
if (inferredQuery && !liveStream) {
rawResponse += `\n\n${inferredQuery}`;
}
// If response has image field, response may be a generated image
rawResponse = imageJson.image;
}
let reference: ResponseWithReferences = {};
let responseWithIntent: ResponseWithIntent = {
intentType: imageJson.intentType,
response: rawResponse,
inferredQueries: imageJson.inferredQueries,
};
if (imageJson.context && imageJson.context.length > 0) {
const rawReferenceAsJson = imageJson.context;
if (rawReferenceAsJson instanceof Array) {
reference.context = rawReferenceAsJson;
} else if (typeof rawReferenceAsJson === "object" && rawReferenceAsJson !== null) {
reference.online = rawReferenceAsJson;
}
}
if (imageJson.detail) {
// The detail field contains the improved image prompt
rawResponse += imageJson.detail;
}
reference.response = rawResponse;
return reference;
return responseWithIntent;
}
export function modifyFileFilterForConversation(

5
src/interface/web/app/common/iconUtils.tsx
View File

@@ -48,6 +48,7 @@ import {
Oven,
Gavel,
Broadcast,
KeyReturn,
} from "@phosphor-icons/react";
import { Markdown, OrgMode, Pdf, Word } from "@/app/components/logo/fileLogo";
@@ -193,6 +194,10 @@ export function getIconForSlashCommand(command: string, customClassName: string
}
if (command.includes("default")) {
return <KeyReturn className={className} />;
}
if (command.includes("diagram")) {
return <Shapes className={className} />;
}

7
src/interface/web/app/components/chatHistory/chatHistory.module.css
View File

@@ -2,12 +2,7 @@ div.chatHistory {
display: flex;
flex-direction: column;
height: 100%;
}
div.chatLayout {
height: 80vh;
overflow-y: auto;
margin: 0 auto;
margin: auto;
}
div.agentIndicator a {

35
src/interface/web/app/components/chatHistory/chatHistory.tsx
View File

@@ -37,6 +37,7 @@ interface ChatHistoryProps {
pendingMessage?: string;
publicConversationSlug?: string;
setAgent: (agent: AgentData) => void;
customClassName?: string;
}
function constructTrainOfThought(
@@ -255,7 +256,7 @@ export default function ChatHistory(props: ChatHistoryProps) {
return (
<ScrollArea className={`h-[80vh] relative`} ref={scrollAreaRef}>
<div>
<div className={styles.chatHistory}>
<div className={`${styles.chatHistory} ${props.customClassName}`}>
<div ref={sentinelRef} style={{ height: "1px" }}>
{fetchingData && (
<InlineLoading message="Loading Conversation" className="opacity-50" />
@@ -322,6 +323,12 @@ export default function ChatHistory(props: ChatHistoryProps) {
by: "khoj",
automationId: "",
rawQuery: message.rawQuery,
intent: {
type: message.intentType || "",
query: message.rawQuery,
"memory-type": "",
"inferred-queries": message.inferredQueries || [],
},
}}
customClassName="fullHistory"
borderLeftColor={`${data?.agent?.color}-500`}
@@ -365,18 +372,20 @@ export default function ChatHistory(props: ChatHistoryProps) {
</div>
)}
</div>
{!isNearBottom && (
<button
title="Scroll to bottom"
className="absolute bottom-4 right-5 bg-white dark:bg-[hsl(var(--background))] text-neutral-500 dark:text-white p-2 rounded-full shadow-xl"
onClick={() => {
scrollToBottom();
setIsNearBottom(true);
}}
>
<ArrowDown size={24} />
</button>
)}
<div className={`${props.customClassName} fixed bottom-[15%] z-10`}>
{!isNearBottom && (
<button
title="Scroll to bottom"
className="absolute bottom-0 right-0 bg-white dark:bg-[hsl(var(--background))] text-neutral-500 dark:text-white p-2 rounded-full shadow-xl"
onClick={() => {
scrollToBottom();
setIsNearBottom(true);
}}
>
<ArrowDown size={24} />
</button>
)}
</div>
</div>
</ScrollArea>
);

52
src/interface/web/app/components/chatMessage/chatMessage.tsx
View File

@@ -26,6 +26,7 @@ import {
Palette,
ClipboardText,
Check,
Shapes,
} from "@phosphor-icons/react";
import DOMPurify from "dompurify";
@@ -35,6 +36,7 @@ import { AgentData } from "@/app/agents/page";
import renderMathInElement from "katex/contrib/auto-render";
import "katex/dist/katex.min.css";
import ExcalidrawComponent from "../excalidraw/excalidraw";
const md = new markdownIt({
html: true,
@@ -127,6 +129,8 @@ export interface StreamMessage {
timestamp: string;
agent?: AgentData;
images?: string[];
intentType?: string;
inferredQueries?: string[];
}
export interface ChatHistoryData {
@@ -251,6 +255,10 @@ function chooseIconFromHeader(header: string, iconColor: string) {
return <Aperture className={`${classNames}`} />;
}
if (compareHeader.includes("diagram")) {
return <Shapes className={`${classNames}`} />;
}
if (compareHeader.includes("paint")) {
return <Palette className={`${classNames}`} />;
}
@@ -282,6 +290,7 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
const [markdownRendered, setMarkdownRendered] = useState<string>("");
const [isPlaying, setIsPlaying] = useState<boolean>(false);
const [interrupted, setInterrupted] = useState<boolean>(false);
const [excalidrawData, setExcalidrawData] = useState<string>("");
const interruptedRef = useRef<boolean>(false);
const messageRef = useRef<HTMLDivElement>(null);
@@ -320,6 +329,11 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
useEffect(() => {
let message = props.chatMessage.message;
if (props.chatMessage.intent && props.chatMessage.intent.type == "excalidraw") {
message = props.chatMessage.intent["inferred-queries"][0];
setExcalidrawData(props.chatMessage.message);
}
// Replace LaTeX delimiters with placeholders
message = message
.replace(/\\\(/g, "LEFTPAREN")
@@ -337,22 +351,27 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
message = `<div class="${styles.imagesContainer}">${imagesInMd}</div>${message}`;
}
if (props.chatMessage.intent && props.chatMessage.intent.type == "text-to-image") {
message = `![generated image](data:image/png;base64,${message})`;
} else if (props.chatMessage.intent && props.chatMessage.intent.type == "text-to-image2") {
message = `![generated image](${message})`;
} else if (
props.chatMessage.intent &&
props.chatMessage.intent.type == "text-to-image-v3"
) {
message = `![generated image](data:image/webp;base64,${message})`;
}
if (
props.chatMessage.intent &&
props.chatMessage.intent.type.includes("text-to-image") &&
props.chatMessage.intent["inferred-queries"]?.length > 0
) {
message += `\n\n${props.chatMessage.intent["inferred-queries"][0]}`;
const intentTypeHandlers = {
"text-to-image": (msg: string) => `![generated image](data:image/png;base64,${msg})`,
"text-to-image2": (msg: string) => `![generated image](${msg})`,
"text-to-image-v3": (msg: string) =>
`![generated image](data:image/webp;base64,${msg})`,
excalidraw: (msg: string) => {
return msg;
},
};
if (props.chatMessage.intent) {
const { type, "inferred-queries": inferredQueries } = props.chatMessage.intent;
console.log("intent type", type);
if (type in intentTypeHandlers) {
message = intentTypeHandlers[type as keyof typeof intentTypeHandlers](message);
}
if (type.includes("text-to-image") && inferredQueries?.length > 0) {
message += `\n\n${inferredQueries[0]}`;
}
}
setTextRendered(message);
@@ -559,6 +578,7 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
className={styles.chatMessage}
dangerouslySetInnerHTML={{ __html: markdownRendered }}
/>
{excalidrawData && <ExcalidrawComponent data={excalidrawData} />}
</div>
<div className={styles.teaserReferencesContainer}>
<TeaserReferencesSection

24
src/interface/web/app/components/excalidraw/excalidraw.tsx Normal file
View File

@@ -0,0 +1,24 @@
"use client";
import dynamic from "next/dynamic";
import { Suspense } from "react";
import Loading from "../../components/loading/loading";
// Since client components get prerenderd on server as well hence importing
// the excalidraw stuff dynamically with ssr false
const ExcalidrawWrapper = dynamic(() => import("./excalidrawWrapper").then((mod) => mod.default), {
ssr: false,
});
interface ExcalidrawComponentProps {
data: any;
}
export default function ExcalidrawComponent(props: ExcalidrawComponentProps) {
return (
<Suspense fallback={<Loading />}>
<ExcalidrawWrapper data={props.data} />
</Suspense>
);
}

149
src/interface/web/app/components/excalidraw/excalidrawWrapper.tsx Normal file
View File

@@ -0,0 +1,149 @@
"use client";
import { useState, useEffect } from "react";
import dynamic from "next/dynamic";
import { ExcalidrawProps } from "@excalidraw/excalidraw/types/types";
import { ExcalidrawElement } from "@excalidraw/excalidraw/types/element/types";
import { ExcalidrawElementSkeleton } from "@excalidraw/excalidraw/types/data/transform";
const Excalidraw = dynamic<ExcalidrawProps>(
async () => (await import("@excalidraw/excalidraw")).Excalidraw,
{
ssr: false,
},
);
import { convertToExcalidrawElements } from "@excalidraw/excalidraw";
import { Button } from "@/components/ui/button";
import { ArrowsInSimple, ArrowsOutSimple } from "@phosphor-icons/react";
interface ExcalidrawWrapperProps {
data: ExcalidrawElementSkeleton[];
}
export default function ExcalidrawWrapper(props: ExcalidrawWrapperProps) {
const [excalidrawElements, setExcalidrawElements] = useState<ExcalidrawElement[]>([]);
const [expanded, setExpanded] = useState<boolean>(false);
const isValidExcalidrawElement = (element: ExcalidrawElementSkeleton): boolean => {
return (
element.x !== undefined &&
element.y !== undefined &&
element.id !== undefined &&
element.type !== undefined
);
};
useEffect(() => {
if (expanded) {
onkeydown = (e) => {
if (e.key === "Escape") {
setExpanded(false);
// Trigger a resize event to make Excalidraw adjust its size
window.dispatchEvent(new Event("resize"));
}
};
} else {
onkeydown = null;
}
}, [expanded]);
useEffect(() => {
// Do some basic validation
const basicValidSkeletons: ExcalidrawElementSkeleton[] = [];
for (const element of props.data) {
if (isValidExcalidrawElement(element as ExcalidrawElementSkeleton)) {
basicValidSkeletons.push(element as ExcalidrawElementSkeleton);
}
}
const validSkeletons: ExcalidrawElementSkeleton[] = [];
for (const element of basicValidSkeletons) {
if (element.type === "frame") {
continue;
}
if (element.type === "arrow") {
const start = basicValidSkeletons.find((child) => child.id === element.start?.id);
const end = basicValidSkeletons.find((child) => child.id === element.end?.id);
if (start && end) {
validSkeletons.push(element);
}
} else {
validSkeletons.push(element);
}
}
for (const element of basicValidSkeletons) {
if (element.type === "frame") {
const children = element.children?.map((childId) => {
return validSkeletons.find((child) => child.id === childId);
});
// Get the valid children, filter out any undefined values
const validChildrenIds: readonly string[] = children
?.map((child) => child?.id)
.filter((id) => id !== undefined) as string[];
if (validChildrenIds === undefined || validChildrenIds.length === 0) {
continue;
}
validSkeletons.push({
...element,
children: validChildrenIds,
});
}
}
const elements = convertToExcalidrawElements(validSkeletons);
setExcalidrawElements(elements);
}, []);
return (
<div className="relative">
<div
className={`${expanded ? "fixed inset-0 bg-black bg-opacity-50 backdrop-blur-sm z-50 flex items-center justify-center" : ""}`}
>
<Button
onClick={() => {
setExpanded(!expanded);
// Trigger a resize event to make Excalidraw adjust its size
window.dispatchEvent(new Event("resize"));
}}
variant={"outline"}
className={`${expanded ? "absolute top-2 left-2 z-[60]" : ""}`}
>
{expanded ? (
<ArrowsInSimple className="h-4 w-4" />
) : (
<ArrowsOutSimple className="h-4 w-4" />
)}
</Button>
<div
className={`
${expanded ? "w-[80vw] h-[80vh]" : "w-full h-[500px]"}
bg-white overflow-hidden rounded-lg relative
`}
>
<Excalidraw
initialData={{
elements: excalidrawElements,
appState: { zenModeEnabled: true },
scrollToContent: true,
}}
// TODO - Create a common function to detect if the theme is dark?
theme={localStorage.getItem("theme") === "dark" ? "dark" : "light"}
validateEmbeddable={true}
renderTopRightUI={(isMobile, appState) => {
return <></>;
}}
/>
</div>
</div>
</div>
);
}

9
src/interface/web/app/share/chat/layout.tsx
View File

@@ -27,7 +27,14 @@ export default function RootLayout({
child-src 'none';
object-src 'none';"
></meta>
<body className={inter.className}>{children}</body>
<body className={inter.className}>
{children}
<script
dangerouslySetInnerHTML={{
__html: `window.EXCALIDRAW_ASSET_PATH = 'https://assets.khoj.dev/@excalidraw/excalidraw/dist/';`,
}}
/>
</body>
</html>
);
}

18
src/interface/web/app/share/chat/page.tsx
View File

@@ -39,6 +39,8 @@ function ChatBodyData(props: ChatBodyDataProps) {
const setQueryToProcess = props.setQueryToProcess;
const streamedMessages = props.streamedMessages;
const chatHistoryCustomClassName = props.isMobileWidth ? "w-full" : "w-4/6";
useEffect(() => {
if (images.length > 0) {
const encodedImages = images.map((image) => encodeURIComponent(image));
@@ -96,10 +98,11 @@ function ChatBodyData(props: ChatBodyDataProps) {
setTitle={props.setTitle}
pendingMessage={processingMessage ? message : ""}
incomingMessages={props.streamedMessages}
customClassName={chatHistoryCustomClassName}
/>
</div>
<div
className={`${styles.inputBox} p-1 md:px-2 shadow-md bg-background align-middle items-center justify-center dark:bg-neutral-700 dark:border-0 dark:shadow-sm rounded-t-2xl rounded-b-none md:rounded-xl`}
className={`${styles.inputBox} p-1 md:px-2 shadow-md bg-background align-middle items-center justify-center dark:bg-neutral-700 dark:border-0 dark:shadow-sm rounded-t-2xl rounded-b-none md:rounded-xl h-fit ${chatHistoryCustomClassName} mr-auto ml-auto`}
>
<ChatInputArea
isLoggedIn={props.isLoggedIn}
@@ -293,6 +296,19 @@ export default function SharedChat() {
<div className={styles.chatBox}>
<div className={styles.chatBoxBody}>
{!isMobileWidth && title && (
<div
className={`${styles.chatTitleWrapper} text-nowrap text-ellipsis overflow-hidden max-w-screen-md grid items-top font-bold mr-8 pt-6 col-auto h-fit`}
>
{title && (
<h2
className={`text-lg text-ellipsis whitespace-nowrap overflow-x-hidden`}
>
{title}
</h2>
)}
</div>
)}
<Suspense fallback={<Loading />}>
<ChatBodyData
conversationId={conversationId}

2
src/interface/web/app/share/chat/sharedChat.module.css
View File

@@ -75,7 +75,7 @@ div.titleBar {
div.chatBoxBody {
display: grid;
height: 100%;
width: 70%;
width: 95%;
margin: auto;
}

5
src/interface/web/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "khoj-ai",
"version": "1.26.0",
"version": "1.26.4",
"private": true,
"scripts": {
"dev": "next dev",
@@ -63,7 +63,8 @@
"swr": "^2.2.5",
"typescript": "^5",
"vaul": "^0.9.1",
"zod": "^3.23.8"
"zod": "^3.23.8",
"@excalidraw/excalidraw": "^0.17.6"
},
"devDependencies": {
"@types/dompurify": "^3.0.5",

5
src/interface/web/yarn.lock
View File

@@ -286,6 +286,11 @@
resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.57.1.tgz#de633db3ec2ef6a3c89e2f19038063e8a122e2c2"
integrity sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==
"@excalidraw/excalidraw@^0.17.6":
version "0.17.6"
resolved "https://registry.yarnpkg.com/@excalidraw/excalidraw/-/excalidraw-0.17.6.tgz#5fd208ce69d33ca712d1804b50d7d06d5c46ac4d"
integrity sha512-fyCl+zG/Z5yhHDh5Fq2ZGmphcrALmuOdtITm8gN4d8w4ntnaopTXcTfnAAaU3VleDC6LhTkoLOTG6P5kgREiIg==
"@floating-ui/core@^1.6.0":
version "1.6.8"
resolved "https://registry.yarnpkg.com/@floating-ui/core/-/core-1.6.8.tgz#aa43561be075815879305965020f492cdb43da12"