Merge branch 'master' of github.com:khoj-ai/khoj into features/advanced-reasoning

2026-03-03 21:29:08 +00:00 · 2024-10-23 19:15:51 -07:00
parent c6c48cfc18 5120597d4e
commit 30f9225021
85 changed files with 5132 additions and 3202 deletions
--- a/src/interface/web/app/components/agentCard/agentCard.module.css
+++ b/src/interface/web/app/components/agentCard/agentCard.module.css
@@ -0,0 +1,20 @@
+.agentPersonality p {
+    white-space: inherit;
+    overflow: hidden;
+    height: 77px;
+    line-height: 1.5;
+}
+
+div.agentPersonality {
+    text-align: left;
+    grid-column: span 3;
+    overflow: hidden;
+}
+
+button.infoButton {
+    border: none;
+    background-color: transparent !important;
+    text-align: left;
+    font-family: inherit;
+    font-size: medium;
+}
--- a/src/interface/web/app/components/agentCard/agentCard.tsx
+++ b/src/interface/web/app/components/agentCard/agentCard.tsx
--- a/src/interface/web/app/components/chatHistory/chatHistory.module.css
+++ b/src/interface/web/app/components/chatHistory/chatHistory.module.css
@@ -2,12 +2,7 @@ div.chatHistory {
    display: flex;
    flex-direction: column;
    height: 100%;
-}
-
-div.chatLayout {
-    height: 80vh;
-    overflow-y: auto;
-    margin: 0 auto;
+    margin: auto;
 }

 div.agentIndicator a {
--- a/src/interface/web/app/components/chatHistory/chatHistory.tsx
+++ b/src/interface/web/app/components/chatHistory/chatHistory.tsx
@@ -37,6 +37,7 @@ interface ChatHistoryProps {
    pendingMessage?: string;
    publicConversationSlug?: string;
    setAgent: (agent: AgentData) => void;
+    customClassName?: string;
 }

 function constructTrainOfThought(
@@ -255,7 +256,7 @@ export default function ChatHistory(props: ChatHistoryProps) {
    return (
        <ScrollArea className={`h-[80vh] relative`} ref={scrollAreaRef}>
            <div>
-                <div className={styles.chatHistory}>
+                <div className={`${styles.chatHistory} ${props.customClassName}`}>
                    <div ref={sentinelRef} style={{ height: "1px" }}>
                        {fetchingData && (
                            <InlineLoading message="Loading Conversation" className="opacity-50" />
@@ -299,7 +300,7 @@ export default function ChatHistory(props: ChatHistoryProps) {
                                            created: message.timestamp,
                                            by: "you",
                                            automationId: "",
-                                            uploadedImageData: message.uploadedImageData,
+                                            images: message.images,
                                        }}
                                        customClassName="fullHistory"
                                        borderLeftColor={`${data?.agent?.color}-500`}
@@ -324,6 +325,12 @@ export default function ChatHistory(props: ChatHistoryProps) {
                                            by: "khoj",
                                            automationId: "",
                                            rawQuery: message.rawQuery,
+                                            intent: {
+                                                type: message.intentType || "",
+                                                query: message.rawQuery,
+                                                "memory-type": "",
+                                                "inferred-queries": message.inferredQueries || [],
+                                            },
                                        }}
                                        customClassName="fullHistory"
                                        borderLeftColor={`${data?.agent?.color}-500`}
@@ -344,7 +351,6 @@ export default function ChatHistory(props: ChatHistoryProps) {
                                created: new Date().getTime().toString(),
                                by: "you",
                                automationId: "",
-                                uploadedImageData: props.pendingMessage,
                            }}
                            customClassName="fullHistory"
                            borderLeftColor={`${data?.agent?.color}-500`}
@@ -369,18 +375,20 @@ export default function ChatHistory(props: ChatHistoryProps) {
                        </div>
                    )}
                </div>
-                {!isNearBottom && (
-                    <button
-                        title="Scroll to bottom"
-                        className="absolute bottom-4 right-5 bg-white dark:bg-[hsl(var(--background))] text-neutral-500 dark:text-white p-2 rounded-full shadow-xl"
-                        onClick={() => {
-                            scrollToBottom();
-                            setIsNearBottom(true);
-                        }}
-                    >
-                        <ArrowDown size={24} />
-                    </button>
-                )}
+                <div className={`${props.customClassName} fixed bottom-[15%] z-10`}>
+                    {!isNearBottom && (
+                        <button
+                            title="Scroll to bottom"
+                            className="absolute bottom-0 right-0 bg-white dark:bg-[hsl(var(--background))] text-neutral-500 dark:text-white p-2 rounded-full shadow-xl"
+                            onClick={() => {
+                                scrollToBottom();
+                                setIsNearBottom(true);
+                            }}
+                        >
+                            <ArrowDown size={24} />
+                        </button>
+                    )}
+                </div>
            </div>
        </ScrollArea>
    );
--- a/src/interface/web/app/components/chatInputArea/chatInputArea.tsx
+++ b/src/interface/web/app/components/chatInputArea/chatInputArea.tsx
@@ -1,25 +1,9 @@
 import styles from "./chatInputArea.module.css";
-import React, { useEffect, useRef, useState } from "react";
+import React, { useEffect, useRef, useState, forwardRef } from "react";

 import DOMPurify from "dompurify";
 import "katex/dist/katex.min.css";
-import {
-    ArrowRight,
-    ArrowUp,
-    Browser,
-    ChatsTeardrop,
-    GlobeSimple,
-    Gps,
-    Image,
-    Microphone,
-    Notebook,
-    Paperclip,
-    X,
-    Question,
-    Robot,
-    Shapes,
-    Stop,
-} from "@phosphor-icons/react";
+import { ArrowUp, Microphone, Paperclip, X, Stop } from "@phosphor-icons/react";

 import {
    Command,
@@ -68,7 +52,7 @@ interface ChatInputProps {
    agentColor?: string;
 }

-export default function ChatInputArea(props: ChatInputProps) {
+export const ChatInputArea = forwardRef<HTMLTextAreaElement, ChatInputProps>((props, ref) => {
    const [message, setMessage] = useState("");
    const fileInputRef = useRef<HTMLInputElement>(null);

@@ -78,15 +62,17 @@ export default function ChatInputArea(props: ChatInputProps) {
    const [loginRedirectMessage, setLoginRedirectMessage] = useState<string | null>(null);
    const [showLoginPrompt, setShowLoginPrompt] = useState(false);

-    const [recording, setRecording] = useState(false);
    const [imageUploaded, setImageUploaded] = useState(false);
-    const [imagePath, setImagePath] = useState<string>("");
-    const [imageData, setImageData] = useState<string | null>(null);
+    const [imagePaths, setImagePaths] = useState<string[]>([]);
+    const [imageData, setImageData] = useState<string[]>([]);
+
+    const [recording, setRecording] = useState(false);
    const [mediaRecorder, setMediaRecorder] = useState<MediaRecorder | null>(null);

    const [progressValue, setProgressValue] = useState(0);
    const [isDragAndDropping, setIsDragAndDropping] = useState(false);

+    const chatInputRef = ref as React.MutableRefObject<HTMLTextAreaElement>;
    useEffect(() => {
        if (!uploading) {
            setProgressValue(0);
@@ -106,27 +92,31 @@ export default function ChatInputArea(props: ChatInputProps) {

    useEffect(() => {
        async function fetchImageData() {
-            if (imagePath) {
-                const response = await fetch(imagePath);
-                const blob = await response.blob();
-                const reader = new FileReader();
-                reader.onload = function () {
-                    const base64data = reader.result;
-                    setImageData(base64data as string);
-                };
-                reader.readAsDataURL(blob);
+            if (imagePaths.length > 0) {
+                const newImageData = await Promise.all(
+                    imagePaths.map(async (path) => {
+                        const response = await fetch(path);
+                        const blob = await response.blob();
+                        return new Promise<string>((resolve) => {
+                            const reader = new FileReader();
+                            reader.onload = () => resolve(reader.result as string);
+                            reader.readAsDataURL(blob);
+                        });
+                    }),
+                );
+                setImageData(newImageData);
            }
            setUploading(false);
        }
        setUploading(true);
        fetchImageData();
-    }, [imagePath]);
+    }, [imagePaths]);

    function onSendMessage() {
        if (imageUploaded) {
            setImageUploaded(false);
-            setImagePath("");
-            props.sendImage(imageData || "");
+            setImagePaths([]);
+            imageData.forEach((data) => props.sendImage(data));
        }
        if (!message.trim()) return;

@@ -168,22 +158,29 @@ export default function ChatInputArea(props: ChatInputProps) {

    function uploadFiles(files: FileList) {
        if (!props.isLoggedIn) {
-            setLoginRedirectMessage("Whoa! You need to login to upload files");
+            setLoginRedirectMessage("Please login to chat with your files");
            setShowLoginPrompt(true);
            return;
        }
-        // check for image file
-        const image_endings = ["jpg", "jpeg", "png"];
+        // check for image files
+        const image_endings = ["jpg", "jpeg", "png", "webp"];
+        const newImagePaths: string[] = [];
        for (let i = 0; i < files.length; i++) {
            const file = files[i];
            const file_extension = file.name.split(".").pop();
            if (image_endings.includes(file_extension || "")) {
-                setImageUploaded(true);
-                setImagePath(DOMPurify.sanitize(URL.createObjectURL(file)));
-                return;
+                newImagePaths.push(DOMPurify.sanitize(URL.createObjectURL(file)));
            }
        }

+        if (newImagePaths.length > 0) {
+            setImageUploaded(true);
+            setImagePaths((prevPaths) => [...prevPaths, ...newImagePaths]);
+            // Set focus to the input for user message after uploading files
+            chatInputRef?.current?.focus();
+            return;
+        }
+
        uploadDataForIndexing(
            files,
            setWarning,
@@ -192,6 +189,9 @@ export default function ChatInputArea(props: ChatInputProps) {
            props.setUploadedFiles,
            props.conversationId,
        );
+
+        // Set focus to the input for user message after uploading files
+        chatInputRef?.current?.focus();
    }

    // Assuming this function is added within the same context as the provided excerpt
@@ -270,9 +270,8 @@ export default function ChatInputArea(props: ChatInputProps) {
        }
    }, [recording, mediaRecorder]);

-    const chatInputRef = useRef<HTMLTextAreaElement>(null);
    useEffect(() => {
-        if (!chatInputRef.current) return;
+        if (!chatInputRef?.current) return;
        chatInputRef.current.style.height = "auto";
        chatInputRef.current.style.height =
            Math.max(chatInputRef.current.scrollHeight - 24, 64) + "px";
@@ -288,9 +287,12 @@ export default function ChatInputArea(props: ChatInputProps) {
        setIsDragAndDropping(false);
    }

-    function removeImageUpload() {
-        setImageUploaded(false);
-        setImagePath("");
+    function removeImageUpload(index: number) {
+        setImagePaths((prevPaths) => prevPaths.filter((_, i) => i !== index));
+        setImageData((prevData) => prevData.filter((_, i) => i !== index));
+        if (imagePaths.length === 1) {
+            setImageUploaded(false);
+        }
    }

    return (
@@ -407,24 +409,11 @@ export default function ChatInputArea(props: ChatInputProps) {
                </div>
            )}
            <div
-                className={`${styles.actualInputArea} items-center justify-between dark:bg-neutral-700 relative ${isDragAndDropping && "animate-pulse"}`}
+                className={`${styles.actualInputArea} justify-between dark:bg-neutral-700 relative ${isDragAndDropping && "animate-pulse"}`}
                onDragOver={handleDragOver}
                onDragLeave={handleDragLeave}
                onDrop={handleDragAndDropFiles}
            >
-                {imageUploaded && (
-                    <div className="absolute bottom-[80px] left-0 right-0 dark:bg-neutral-700 bg-white pt-5 pb-5 w-full rounded-lg border dark:border-none grid grid-cols-2">
-                        <div className="pl-4 pr-4">
-                            <img src={imagePath} alt="img" className="w-auto max-h-[100px]" />
-                        </div>
-                        <div className="pl-4 pr-4">
-                            <X
-                                className="w-6 h-6 float-right dark:hover:bg-[hsl(var(--background))] hover:bg-neutral-100 rounded-sm"
-                                onClick={removeImageUpload}
-                            />
-                        </div>
-                    </div>
-                )}
                <input
                    type="file"
                    multiple={true}
@@ -432,15 +421,37 @@ export default function ChatInputArea(props: ChatInputProps) {
                    onChange={handleFileChange}
                    style={{ display: "none" }}
                />
-                <Button
-                    variant={"ghost"}
-                    className="!bg-none p-0 m-2 h-auto text-3xl rounded-full text-gray-300 hover:text-gray-500"
-                    disabled={props.sendDisabled}
-                    onClick={handleFileButtonClick}
-                >
-                    <Paperclip className="w-8 h-8" />
-                </Button>
-                <div className="grid w-full gap-1.5 relative">
+                <div className="flex items-end pb-4">
+                    <Button
+                        variant={"ghost"}
+                        className="!bg-none p-0 m-2 h-auto text-3xl rounded-full text-gray-300 hover:text-gray-500"
+                        disabled={props.sendDisabled}
+                        onClick={handleFileButtonClick}
+                    >
+                        <Paperclip className="w-8 h-8" />
+                    </Button>
+                </div>
+                <div className="flex-grow flex flex-col w-full gap-1.5 relative pb-2">
+                    <div className="flex items-center gap-2 overflow-x-auto">
+                        {imageUploaded &&
+                            imagePaths.map((path, index) => (
+                                <div key={index} className="relative flex-shrink-0 pb-3 pt-2 group">
+                                    <img
+                                        src={path}
+                                        alt={`img-${index}`}
+                                        className="w-auto h-16 object-cover rounded-xl"
+                                    />
+                                    <Button
+                                        variant="ghost"
+                                        size="icon"
+                                        className="absolute -top-0 -right-2 h-5 w-5 rounded-full bg-neutral-200 dark:bg-neutral-600 hover:bg-neutral-300 dark:hover:bg-neutral-500 opacity-0 group-hover:opacity-100 transition-opacity"
+                                        onClick={() => removeImageUpload(index)}
+                                    >
+                                        <X className="h-3 w-3" />
+                                    </Button>
+                                </div>
+                            ))}
+                    </div>
                    <Textarea
                        ref={chatInputRef}
                        className={`border-none w-full h-16 min-h-16 max-h-[128px] md:py-4 rounded-lg resize-none dark:bg-neutral-700 ${props.isMobileWidth ? "text-md" : "text-lg"}`}
@@ -449,9 +460,9 @@ export default function ChatInputArea(props: ChatInputProps) {
                        autoFocus={true}
                        value={message}
                        onKeyDown={(e) => {
-                            if (e.key === "Enter" && !e.shiftKey) {
+                            if (e.key === "Enter" && !e.shiftKey && !props.isMobileWidth) {
                                setImageUploaded(false);
-                                setImagePath("");
+                                setImagePaths([]);
                                e.preventDefault();
                                onSendMessage();
                            }
@@ -460,58 +471,62 @@ export default function ChatInputArea(props: ChatInputProps) {
                        disabled={props.sendDisabled || recording}
                    />
                </div>
-                {recording ? (
-                    <TooltipProvider>
-                        <Tooltip>
-                            <TooltipTrigger asChild>
-                                <Button
-                                    variant="default"
-                                    className={`${!recording && "hidden"} ${props.agentColor ? convertToBGClass(props.agentColor) : "bg-orange-300 hover:bg-orange-500"} rounded-full p-1 m-2 h-auto text-3xl transition transform md:hover:-translate-y-1`}
-                                    onClick={() => {
-                                        setRecording(!recording);
-                                    }}
-                                    disabled={props.sendDisabled}
-                                >
-                                    <Stop weight="fill" className="w-6 h-6" />
-                                </Button>
-                            </TooltipTrigger>
-                            <TooltipContent>
-                                Click to stop recording and transcribe your voice.
-                            </TooltipContent>
-                        </Tooltip>
-                    </TooltipProvider>
-                ) : mediaRecorder ? (
-                    <InlineLoading />
-                ) : (
-                    <TooltipProvider>
-                        <Tooltip>
-                            <TooltipTrigger asChild>
-                                <Button
-                                    variant="default"
-                                    className={`${!message || recording || "hidden"} ${props.agentColor ? convertToBGClass(props.agentColor) : "bg-orange-300 hover:bg-orange-500"} rounded-full p-1 m-2 h-auto text-3xl transition transform md:hover:-translate-y-1`}
-                                    onClick={() => {
-                                        setMessage("Listening...");
-                                        setRecording(!recording);
-                                    }}
-                                    disabled={props.sendDisabled}
-                                >
-                                    <Microphone weight="fill" className="w-6 h-6" />
-                                </Button>
-                            </TooltipTrigger>
-                            <TooltipContent>
-                                Click to transcribe your message with voice.
-                            </TooltipContent>
-                        </Tooltip>
-                    </TooltipProvider>
-                )}
-                <Button
-                    className={`${(!message || recording) && "hidden"} ${props.agentColor ? convertToBGClass(props.agentColor) : "bg-orange-300 hover:bg-orange-500"} rounded-full p-1 m-2 h-auto text-3xl transition transform md:hover:-translate-y-1`}
-                    onClick={onSendMessage}
-                    disabled={props.sendDisabled}
-                >
-                    <ArrowUp className="w-6 h-6" weight="bold" />
-                </Button>
+                <div className="flex items-end pb-4">
+                    {recording ? (
+                        <TooltipProvider>
+                            <Tooltip>
+                                <TooltipTrigger asChild>
+                                    <Button
+                                        variant="default"
+                                        className={`${!recording && "hidden"} ${props.agentColor ? convertToBGClass(props.agentColor) : "bg-orange-300 hover:bg-orange-500"} rounded-full p-1 m-2 h-auto text-3xl transition transform md:hover:-translate-y-1`}
+                                        onClick={() => {
+                                            setRecording(!recording);
+                                        }}
+                                        disabled={props.sendDisabled}
+                                    >
+                                        <Stop weight="fill" className="w-6 h-6" />
+                                    </Button>
+                                </TooltipTrigger>
+                                <TooltipContent>
+                                    Click to stop recording and transcribe your voice.
+                                </TooltipContent>
+                            </Tooltip>
+                        </TooltipProvider>
+                    ) : mediaRecorder ? (
+                        <InlineLoading />
+                    ) : (
+                        <TooltipProvider>
+                            <Tooltip>
+                                <TooltipTrigger asChild>
+                                    <Button
+                                        variant="default"
+                                        className={`${!message || recording || "hidden"} ${props.agentColor ? convertToBGClass(props.agentColor) : "bg-orange-300 hover:bg-orange-500"} rounded-full p-1 m-2 h-auto text-3xl transition transform md:hover:-translate-y-1`}
+                                        onClick={() => {
+                                            setMessage("Listening...");
+                                            setRecording(!recording);
+                                        }}
+                                        disabled={props.sendDisabled}
+                                    >
+                                        <Microphone weight="fill" className="w-6 h-6" />
+                                    </Button>
+                                </TooltipTrigger>
+                                <TooltipContent>
+                                    Click to transcribe your message with voice.
+                                </TooltipContent>
+                            </Tooltip>
+                        </TooltipProvider>
+                    )}
+                    <Button
+                        className={`${(!message || recording) && "hidden"} ${props.agentColor ? convertToBGClass(props.agentColor) : "bg-orange-300 hover:bg-orange-500"} rounded-full p-1 m-2 h-auto text-3xl transition transform md:hover:-translate-y-1`}
+                        onClick={onSendMessage}
+                        disabled={props.sendDisabled}
+                    >
+                        <ArrowUp className="w-6 h-6" weight="bold" />
+                    </Button>
+                </div>
            </div>
        </>
    );
-}
+});
+
+ChatInputArea.displayName = "ChatInputArea";
--- a/src/interface/web/app/components/chatMessage/chatMessage.module.css
+++ b/src/interface/web/app/components/chatMessage/chatMessage.module.css
@@ -57,7 +57,26 @@ div.emptyChatMessage {
    display: none;
 }

-div.chatMessageContainer img {
+div.imagesContainer {
+    display: flex;
+    overflow-x: auto;
+    padding-bottom: 8px;
+    margin-bottom: 8px;
+}
+
+div.imageWrapper {
+    flex: 0 0 auto;
+    margin-right: 8px;
+}
+
+div.imageWrapper img {
+    width: auto;
+    height: 128px;
+    object-fit: cover;
+    border-radius: 8px;
+}
+
+div.chatMessageContainer > img {
    width: auto;
    height: auto;
    max-width: 100%;
--- a/src/interface/web/app/components/chatMessage/chatMessage.tsx
+++ b/src/interface/web/app/components/chatMessage/chatMessage.tsx
@@ -28,6 +28,7 @@ import {
    ClipboardText,
    Check,
    Code,
+    Shapes,
 } from "@phosphor-icons/react";

 import DOMPurify from "dompurify";
@@ -37,6 +38,7 @@ import { AgentData } from "@/app/agents/page";

 import renderMathInElement from "katex/contrib/auto-render";
 import "katex/dist/katex.min.css";
+import ExcalidrawComponent from "../excalidraw/excalidraw";

 const md = new markdownIt({
    html: true,
@@ -137,7 +139,7 @@ export interface SingleChatMessage {
    rawQuery?: string;
    intent?: Intent;
    agent?: AgentData;
-    uploadedImageData?: string;
+    images?: string[];
 }

 export interface StreamMessage {
@@ -150,7 +152,9 @@ export interface StreamMessage {
    rawQuery: string;
    timestamp: string;
    agent?: AgentData;
-    uploadedImageData?: string;
+    images?: string[];
+    intentType?: string;
+    inferredQueries?: string[];
 }

 export interface ChatHistoryData {
@@ -232,7 +236,6 @@ interface ChatMessageProps {
    borderLeftColor?: string;
    isLastMessage?: boolean;
    agent?: AgentData;
-    uploadedImageData?: string;
 }

 interface TrainOfThoughtProps {
@@ -276,6 +279,10 @@ function chooseIconFromHeader(header: string, iconColor: string) {
        return <Aperture className={`${classNames}`} />;
    }

+    if (compareHeader.includes("diagram")) {
+        return <Shapes className={`${classNames}`} />;
+    }
+
    if (compareHeader.includes("paint")) {
        return <Palette className={`${classNames}`} />;
    }
@@ -311,6 +318,7 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
    const [markdownRendered, setMarkdownRendered] = useState<string>("");
    const [isPlaying, setIsPlaying] = useState<boolean>(false);
    const [interrupted, setInterrupted] = useState<boolean>(false);
+    const [excalidrawData, setExcalidrawData] = useState<string>("");

    const interruptedRef = useRef<boolean>(false);
    const messageRef = useRef<HTMLDivElement>(null);
@@ -347,8 +355,14 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
    }, [messageRef.current]);

    useEffect(() => {
+        // Prepare initial message for rendering
        let message = props.chatMessage.message;

+        if (props.chatMessage.intent && props.chatMessage.intent.type == "excalidraw") {
+            message = props.chatMessage.intent["inferred-queries"][0];
+            setExcalidrawData(props.chatMessage.message);
+        }
+
        // Replace LaTeX delimiters with placeholders
        message = message
            .replace(/\\\(/g, "LEFTPAREN")
@@ -356,8 +370,50 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
            .replace(/\\\[/g, "LEFTBRACKET")
            .replace(/\\\]/g, "RIGHTBRACKET");

-        if (props.chatMessage.uploadedImageData) {
-            message = `![uploaded image](${props.chatMessage.uploadedImageData})\n\n${message}`;
+        const intentTypeHandlers = {
+            "text-to-image": (msg: string) => `![generated image](data:image/png;base64,${msg})`,
+            "text-to-image2": (msg: string) => `![generated image](${msg})`,
+            "text-to-image-v3": (msg: string) =>
+                `![generated image](data:image/webp;base64,${msg})`,
+            excalidraw: (msg: string) => msg,
+        };
+
+        // Handle intent-specific rendering
+        if (props.chatMessage.intent) {
+            const { type, "inferred-queries": inferredQueries } = props.chatMessage.intent;
+
+            console.log("intent type", type);
+            if (type in intentTypeHandlers) {
+                message = intentTypeHandlers[type as keyof typeof intentTypeHandlers](message);
+            }
+
+            if (type.includes("text-to-image") && inferredQueries?.length > 0) {
+                message += `\n\n${inferredQueries[0]}`;
+            }
+        }
+        // Handle user attached images rendering
+        let messageForClipboard = message;
+        let messageToRender = message;
+        if (props.chatMessage.images && props.chatMessage.images.length > 0) {
+            const sanitizedImages = props.chatMessage.images.map((image) => {
+                const decodedImage = image.startsWith("data%3Aimage")
+                    ? decodeURIComponent(image)
+                    : image;
+                return DOMPurify.sanitize(decodedImage);
+            });
+            const imagesInMd = sanitizedImages
+                .map((sanitizedImage, index) => {
+                    return `![uploaded image ${index + 1}](${sanitizedImage})`;
+                })
+                .join("\n");
+            const imagesInHtml = sanitizedImages
+                .map((sanitizedImage, index) => {
+                    return `<div class="${styles.imageWrapper}"><img src="${sanitizedImage}" alt="uploaded image ${index + 1}" /></div>`;
+                })
+                .join("");
+            const userImagesInHtml = `<div class="${styles.imagesContainer}">${imagesInHtml}</div>`;
+            messageForClipboard = `${imagesInMd}\n\n${messageForClipboard}`;
+            messageToRender = `${userImagesInHtml}${messageToRender}`;
        }

        if (props.chatMessage.intent && props.chatMessage.intent.type == "text-to-image") {
@@ -402,10 +458,11 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
            });
        }

-        setTextRendered(message);
+        // Set the message text
+        setTextRendered(messageForClipboard);

        // Render the markdown
-        let markdownRendered = md.render(message);
+        let markdownRendered = md.render(messageToRender);

        // Replace placeholders with LaTeX delimiters
        markdownRendered = markdownRendered
@@ -416,7 +473,7 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>

        // Sanitize and set the rendered markdown
        setMarkdownRendered(DOMPurify.sanitize(markdownRendered));
-    }, [props.chatMessage.message, props.chatMessage.intent]);
+    }, [props.chatMessage.message, props.chatMessage.images, props.chatMessage.intent]);

    useEffect(() => {
        if (copySuccess) {
@@ -607,6 +664,7 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
                    className={styles.chatMessage}
                    dangerouslySetInnerHTML={{ __html: markdownRendered }}
                />
+                {excalidrawData && <ExcalidrawComponent data={excalidrawData} />}
            </div>
            <div className={styles.teaserReferencesContainer}>
                <TeaserReferencesSection
--- a/src/interface/web/app/components/excalidraw/excalidraw.tsx
+++ b/src/interface/web/app/components/excalidraw/excalidraw.tsx
@@ -0,0 +1,24 @@
+"use client";
+
+import dynamic from "next/dynamic";
+import { Suspense } from "react";
+import Loading from "../../components/loading/loading";
+
+// Since client components get prerenderd on server as well hence importing
+// the excalidraw stuff dynamically with ssr false
+
+const ExcalidrawWrapper = dynamic(() => import("./excalidrawWrapper").then((mod) => mod.default), {
+    ssr: false,
+});
+
+interface ExcalidrawComponentProps {
+    data: any;
+}
+
+export default function ExcalidrawComponent(props: ExcalidrawComponentProps) {
+    return (
+        <Suspense fallback={<Loading />}>
+            <ExcalidrawWrapper data={props.data} />
+        </Suspense>
+    );
+}
--- a/src/interface/web/app/components/excalidraw/excalidrawWrapper.tsx
+++ b/src/interface/web/app/components/excalidraw/excalidrawWrapper.tsx
@@ -0,0 +1,149 @@
+"use client";
+
+import { useState, useEffect } from "react";
+
+import dynamic from "next/dynamic";
+
+import { ExcalidrawProps } from "@excalidraw/excalidraw/types/types";
+import { ExcalidrawElement } from "@excalidraw/excalidraw/types/element/types";
+import { ExcalidrawElementSkeleton } from "@excalidraw/excalidraw/types/data/transform";
+
+const Excalidraw = dynamic<ExcalidrawProps>(
+    async () => (await import("@excalidraw/excalidraw")).Excalidraw,
+    {
+        ssr: false,
+    },
+);
+
+import { convertToExcalidrawElements } from "@excalidraw/excalidraw";
+
+import { Button } from "@/components/ui/button";
+
+import { ArrowsInSimple, ArrowsOutSimple } from "@phosphor-icons/react";
+
+interface ExcalidrawWrapperProps {
+    data: ExcalidrawElementSkeleton[];
+}
+
+export default function ExcalidrawWrapper(props: ExcalidrawWrapperProps) {
+    const [excalidrawElements, setExcalidrawElements] = useState<ExcalidrawElement[]>([]);
+    const [expanded, setExpanded] = useState<boolean>(false);
+
+    const isValidExcalidrawElement = (element: ExcalidrawElementSkeleton): boolean => {
+        return (
+            element.x !== undefined &&
+            element.y !== undefined &&
+            element.id !== undefined &&
+            element.type !== undefined
+        );
+    };
+
+    useEffect(() => {
+        if (expanded) {
+            onkeydown = (e) => {
+                if (e.key === "Escape") {
+                    setExpanded(false);
+                    // Trigger a resize event to make Excalidraw adjust its size
+                    window.dispatchEvent(new Event("resize"));
+                }
+            };
+        } else {
+            onkeydown = null;
+        }
+    }, [expanded]);
+
+    useEffect(() => {
+        // Do some basic validation
+        const basicValidSkeletons: ExcalidrawElementSkeleton[] = [];
+
+        for (const element of props.data) {
+            if (isValidExcalidrawElement(element as ExcalidrawElementSkeleton)) {
+                basicValidSkeletons.push(element as ExcalidrawElementSkeleton);
+            }
+        }
+
+        const validSkeletons: ExcalidrawElementSkeleton[] = [];
+        for (const element of basicValidSkeletons) {
+            if (element.type === "frame") {
+                continue;
+            }
+            if (element.type === "arrow") {
+                const start = basicValidSkeletons.find((child) => child.id === element.start?.id);
+                const end = basicValidSkeletons.find((child) => child.id === element.end?.id);
+                if (start && end) {
+                    validSkeletons.push(element);
+                }
+            } else {
+                validSkeletons.push(element);
+            }
+        }
+
+        for (const element of basicValidSkeletons) {
+            if (element.type === "frame") {
+                const children = element.children?.map((childId) => {
+                    return validSkeletons.find((child) => child.id === childId);
+                });
+                // Get the valid children, filter out any undefined values
+                const validChildrenIds: readonly string[] = children
+                    ?.map((child) => child?.id)
+                    .filter((id) => id !== undefined) as string[];
+
+                if (validChildrenIds === undefined || validChildrenIds.length === 0) {
+                    continue;
+                }
+
+                validSkeletons.push({
+                    ...element,
+                    children: validChildrenIds,
+                });
+            }
+        }
+
+        const elements = convertToExcalidrawElements(validSkeletons);
+        setExcalidrawElements(elements);
+    }, []);
+
+    return (
+        <div className="relative">
+            <div
+                className={`${expanded ? "fixed inset-0 bg-black bg-opacity-50 backdrop-blur-sm z-50 flex items-center justify-center" : ""}`}
+            >
+                <Button
+                    onClick={() => {
+                        setExpanded(!expanded);
+                        // Trigger a resize event to make Excalidraw adjust its size
+                        window.dispatchEvent(new Event("resize"));
+                    }}
+                    variant={"outline"}
+                    className={`${expanded ? "absolute top-2 left-2 z-[60]" : ""}`}
+                >
+                    {expanded ? (
+                        <ArrowsInSimple className="h-4 w-4" />
+                    ) : (
+                        <ArrowsOutSimple className="h-4 w-4" />
+                    )}
+                </Button>
+                <div
+                    className={`
+                    ${expanded ? "w-[80vw] h-[80vh]" : "w-full h-[500px]"}
+                    bg-white overflow-hidden rounded-lg relative
+                `}
+                >
+                    <Excalidraw
+                        initialData={{
+                            elements: excalidrawElements,
+                            appState: { zenModeEnabled: true },
+                            scrollToContent: true,
+                        }}
+                        // TODO - Create a common function to detect if the theme is dark?
+                        theme={localStorage.getItem("theme") === "dark" ? "dark" : "light"}
+                        validateEmbeddable={true}
+                        renderTopRightUI={(isMobile, appState) => {
+                            return <></>;
+                        }}
+                    />
+                </div>
+            </div>
+        </div>
+    );
+}
--- a/src/interface/web/app/components/sidePanel/chatHistorySidePanel.tsx
+++ b/src/interface/web/app/components/sidePanel/chatHistorySidePanel.tsx
@@ -98,7 +98,11 @@ import { KhojLogoType } from "@/app/components/logo/khojLogo";
 import NavMenu from "@/app/components/navMenu/navMenu";

 // Define a fetcher function
-const fetcher = (url: string) => fetch(url).then((res) => res.json());
+const fetcher = (url: string) =>
+    fetch(url).then((res) => {
+        if (!res.ok) throw new Error(`Failed to call API at ${url} with error ${res.statusText}`);
+        return res.json();
+    });

 interface GroupedChatHistory {
    [key: string]: ChatHistory[];
@@ -181,20 +185,15 @@ function FilesMenu(props: FilesMenuProps) {
    useEffect(() => {
        if (!files) return;

-        const uniqueFiles = Array.from(new Set(files));
+        let sortedUniqueFiles = Array.from(new Set(files)).sort();

-        // First, sort lexically
-        uniqueFiles.sort();
-
-        let sortedFiles = uniqueFiles;
-
-        if (addedFiles) {
-            sortedFiles = addedFiles.concat(
-                sortedFiles.filter((filename: string) => !addedFiles.includes(filename)),
+        if (Array.isArray(addedFiles)) {
+            sortedUniqueFiles = addedFiles.concat(
+                sortedUniqueFiles.filter((filename: string) => !addedFiles.includes(filename)),
            );
        }

-        setUnfilteredFiles(sortedFiles);
+        setUnfilteredFiles(sortedUniqueFiles);
    }, [files, addedFiles]);

    useEffect(() => {
@@ -204,8 +203,10 @@ function FilesMenu(props: FilesMenuProps) {
    }, [props.uploadedFiles]);

    useEffect(() => {
-        if (selectedFiles) {
+        if (Array.isArray(selectedFiles)) {
            setAddedFiles(selectedFiles);
+        } else {
+            setAddedFiles([]);
        }
    }, [selectedFiles]);

@@ -269,7 +270,7 @@ function FilesMenu(props: FilesMenuProps) {
                            </CommandItem>
                        )}
                        {unfilteredFiles.map((filename: string) =>
-                            addedFiles && addedFiles.includes(filename) ? (
+                            Array.isArray(addedFiles) && addedFiles.includes(filename) ? (
                                <CommandItem
                                    key={filename}
                                    value={filename}