Generate dynamic diagrams (via Excalidraw) (#940)

Add support for generating dynamic diagrams in flow with Excalidraw (https://github.com/excalidraw/excalidraw). This happens in three steps:
1. Default information collection & intent determination step.
2. Improving the overall guidance of the prompt for generating a JSON, Excalidraw-compatible declaration.
3. Generation of the diagram to output to the final UI.

Add support in the web UI.

src/interface/web/app/components/chatHistory/chatHistory.module.css

@@ -2,12 +2,7 @@ div.chatHistory {
     display: flex;
     flex-direction: column;
     height: 100%;
-}
-
-div.chatLayout {
-    height: 80vh;
-    overflow-y: auto;
-    margin: 0 auto;
+    margin: auto;
 }
 
 div.agentIndicator a {

src/interface/web/app/components/chatHistory/chatHistory.tsx

@@ -37,6 +37,7 @@ interface ChatHistoryProps {
     pendingMessage?: string;
     publicConversationSlug?: string;
     setAgent: (agent: AgentData) => void;
+    customClassName?: string;
 }
 
 function constructTrainOfThought(
@@ -255,7 +256,7 @@ export default function ChatHistory(props: ChatHistoryProps) {
     return (
         <ScrollArea className={`h-[80vh] relative`} ref={scrollAreaRef}>
             <div>
-                <div className={styles.chatHistory}>
+                <div className={`${styles.chatHistory} ${props.customClassName}`}>
                     <div ref={sentinelRef} style={{ height: "1px" }}>
                         {fetchingData && (
                             <InlineLoading message="Loading Conversation" className="opacity-50" />
@@ -322,6 +323,12 @@ export default function ChatHistory(props: ChatHistoryProps) {
                                             by: "khoj",
                                             automationId: "",
                                             rawQuery: message.rawQuery,
+                                            intent: {
+                                                type: message.intentType || "",
+                                                query: message.rawQuery,
+                                                "memory-type": "",
+                                                "inferred-queries": message.inferredQueries || [],
+                                            },
                                         }}
                                         customClassName="fullHistory"
                                         borderLeftColor={`${data?.agent?.color}-500`}
@@ -366,18 +373,20 @@ export default function ChatHistory(props: ChatHistoryProps) {
                         </div>
                     )}
                 </div>
-                {!isNearBottom && (
-                    <button
-                        title="Scroll to bottom"
-                        className="absolute bottom-4 right-5 bg-white dark:bg-[hsl(var(--background))] text-neutral-500 dark:text-white p-2 rounded-full shadow-xl"
-                        onClick={() => {
-                            scrollToBottom();
-                            setIsNearBottom(true);
-                        }}
-                    >
-                        <ArrowDown size={24} />
-                    </button>
-                )}
+                <div className={`${props.customClassName} fixed bottom-[15%] z-10`}>
+                    {!isNearBottom && (
+                        <button
+                            title="Scroll to bottom"
+                            className="absolute bottom-0 right-0 bg-white dark:bg-[hsl(var(--background))] text-neutral-500 dark:text-white p-2 rounded-full shadow-xl"
+                            onClick={() => {
+                                scrollToBottom();
+                                setIsNearBottom(true);
+                            }}
+                        >
+                            <ArrowDown size={24} />
+                        </button>
+                    )}
+                </div>
             </div>
         </ScrollArea>
     );

src/interface/web/app/components/chatMessage/chatMessage.tsx

@@ -26,6 +26,7 @@ import {
     Palette,
     ClipboardText,
     Check,
+    Shapes,
 } from "@phosphor-icons/react";
 
 import DOMPurify from "dompurify";
@@ -35,6 +36,7 @@ import { AgentData } from "@/app/agents/page";
 
 import renderMathInElement from "katex/contrib/auto-render";
 import "katex/dist/katex.min.css";
+import ExcalidrawComponent from "../excalidraw/excalidraw";
 
 const md = new markdownIt({
     html: true,
@@ -127,6 +129,9 @@ export interface StreamMessage {
     timestamp: string;
     agent?: AgentData;
     uploadedImageData?: string;
+    intentType?: string;
+    inferredQueries?: string[];
+    image?: string;
 }
 
 export interface ChatHistoryData {
@@ -252,6 +257,10 @@ function chooseIconFromHeader(header: string, iconColor: string) {
         return <Aperture className={`${classNames}`} />;
     }
 
+    if (compareHeader.includes("diagram")) {
+        return <Shapes className={`${classNames}`} />;
+    }
+
     if (compareHeader.includes("paint")) {
         return <Palette className={`${classNames}`} />;
     }
@@ -283,6 +292,7 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
     const [markdownRendered, setMarkdownRendered] = useState<string>("");
     const [isPlaying, setIsPlaying] = useState<boolean>(false);
     const [interrupted, setInterrupted] = useState<boolean>(false);
+    const [excalidrawData, setExcalidrawData] = useState<string>("");
 
     const interruptedRef = useRef<boolean>(false);
     const messageRef = useRef<HTMLDivElement>(null);
@@ -321,6 +331,11 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
     useEffect(() => {
         let message = props.chatMessage.message;
 
+        if (props.chatMessage.intent && props.chatMessage.intent.type == "excalidraw") {
+            message = props.chatMessage.intent["inferred-queries"][0];
+            setExcalidrawData(props.chatMessage.message);
+        }
+
         // Replace LaTeX delimiters with placeholders
         message = message
             .replace(/\\\(/g, "LEFTPAREN")
@@ -332,22 +347,27 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
             message = `![uploaded image](${props.chatMessage.uploadedImageData})\n\n${message}`;
         }
 
-        if (props.chatMessage.intent && props.chatMessage.intent.type == "text-to-image") {
-            message = `![generated image](data:image/png;base64,${message})`;
-        } else if (props.chatMessage.intent && props.chatMessage.intent.type == "text-to-image2") {
-            message = `![generated image](${message})`;
-        } else if (
-            props.chatMessage.intent &&
-            props.chatMessage.intent.type == "text-to-image-v3"
-        ) {
-            message = `![generated image](data:image/webp;base64,${message})`;
-        }
-        if (
-            props.chatMessage.intent &&
-            props.chatMessage.intent.type.includes("text-to-image") &&
-            props.chatMessage.intent["inferred-queries"]?.length > 0
-        ) {
-            message += `\n\n${props.chatMessage.intent["inferred-queries"][0]}`;
+        const intentTypeHandlers = {
+            "text-to-image": (msg: string) => `![generated image](data:image/png;base64,${msg})`,
+            "text-to-image2": (msg: string) => `![generated image](${msg})`,
+            "text-to-image-v3": (msg: string) =>
+                `![generated image](data:image/webp;base64,${msg})`,
+            excalidraw: (msg: string) => {
+                return msg;
+            },
+        };
+
+        if (props.chatMessage.intent) {
+            const { type, "inferred-queries": inferredQueries } = props.chatMessage.intent;
+
+            console.log("intent type", type);
+            if (type in intentTypeHandlers) {
+                message = intentTypeHandlers[type as keyof typeof intentTypeHandlers](message);
+            }
+
+            if (type.includes("text-to-image") && inferredQueries?.length > 0) {
+                message += `\n\n${inferredQueries[0]}`;
+            }
         }
 
         setTextRendered(message);
@@ -554,6 +574,7 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
                     className={styles.chatMessage}
                     dangerouslySetInnerHTML={{ __html: markdownRendered }}
                 />
+                {excalidrawData && <ExcalidrawComponent data={excalidrawData} />}
             </div>
             <div className={styles.teaserReferencesContainer}>
                 <TeaserReferencesSection

src/interface/web/app/components/excalidraw/excalidraw.tsx

@@ -0,0 +1,24 @@
+"use client";
+
+import dynamic from "next/dynamic";
+import { Suspense } from "react";
+import Loading from "../../components/loading/loading";
+
+// Since client components get prerenderd on server as well hence importing
+// the excalidraw stuff dynamically with ssr false
+
+const ExcalidrawWrapper = dynamic(() => import("./excalidrawWrapper").then((mod) => mod.default), {
+    ssr: false,
+});
+
+interface ExcalidrawComponentProps {
+    data: any;
+}
+
+export default function ExcalidrawComponent(props: ExcalidrawComponentProps) {
+    return (
+        <Suspense fallback={<Loading />}>
+            <ExcalidrawWrapper data={props.data} />
+        </Suspense>
+    );
+}

src/interface/web/app/components/excalidraw/excalidrawWrapper.tsx

@@ -0,0 +1,149 @@
+"use client";
+
+import { useState, useEffect } from "react";
+
+import dynamic from "next/dynamic";
+
+import { ExcalidrawProps } from "@excalidraw/excalidraw/types/types";
+import { ExcalidrawElement } from "@excalidraw/excalidraw/types/element/types";
+import { ExcalidrawElementSkeleton } from "@excalidraw/excalidraw/types/data/transform";
+
+const Excalidraw = dynamic<ExcalidrawProps>(
+    async () => (await import("@excalidraw/excalidraw")).Excalidraw,
+    {
+        ssr: false,
+    },
+);
+
+import { convertToExcalidrawElements } from "@excalidraw/excalidraw";
+
+import { Button } from "@/components/ui/button";
+
+import { ArrowsInSimple, ArrowsOutSimple } from "@phosphor-icons/react";
+
+interface ExcalidrawWrapperProps {
+    data: ExcalidrawElementSkeleton[];
+}
+
+export default function ExcalidrawWrapper(props: ExcalidrawWrapperProps) {
+    const [excalidrawElements, setExcalidrawElements] = useState<ExcalidrawElement[]>([]);
+    const [expanded, setExpanded] = useState<boolean>(false);
+
+    const isValidExcalidrawElement = (element: ExcalidrawElementSkeleton): boolean => {
+        return (
+            element.x !== undefined &&
+            element.y !== undefined &&
+            element.id !== undefined &&
+            element.type !== undefined
+        );
+    };
+
+    useEffect(() => {
+        if (expanded) {
+            onkeydown = (e) => {
+                if (e.key === "Escape") {
+                    setExpanded(false);
+                    // Trigger a resize event to make Excalidraw adjust its size
+                    window.dispatchEvent(new Event("resize"));
+                }
+            };
+        } else {
+            onkeydown = null;
+        }
+    }, [expanded]);
+
+    useEffect(() => {
+        // Do some basic validation
+        const basicValidSkeletons: ExcalidrawElementSkeleton[] = [];
+
+        for (const element of props.data) {
+            if (isValidExcalidrawElement(element as ExcalidrawElementSkeleton)) {
+                basicValidSkeletons.push(element as ExcalidrawElementSkeleton);
+            }
+        }
+
+        const validSkeletons: ExcalidrawElementSkeleton[] = [];
+        for (const element of basicValidSkeletons) {
+            if (element.type === "frame") {
+                continue;
+            }
+            if (element.type === "arrow") {
+                const start = basicValidSkeletons.find((child) => child.id === element.start?.id);
+                const end = basicValidSkeletons.find((child) => child.id === element.end?.id);
+                if (start && end) {
+                    validSkeletons.push(element);
+                }
+            } else {
+                validSkeletons.push(element);
+            }
+        }
+
+        for (const element of basicValidSkeletons) {
+            if (element.type === "frame") {
+                const children = element.children?.map((childId) => {
+                    return validSkeletons.find((child) => child.id === childId);
+                });
+                // Get the valid children, filter out any undefined values
+                const validChildrenIds: readonly string[] = children
+                    ?.map((child) => child?.id)
+                    .filter((id) => id !== undefined) as string[];
+
+                if (validChildrenIds === undefined || validChildrenIds.length === 0) {
+                    continue;
+                }
+
+                validSkeletons.push({
+                    ...element,
+                    children: validChildrenIds,
+                });
+            }
+        }
+
+        const elements = convertToExcalidrawElements(validSkeletons);
+        setExcalidrawElements(elements);
+    }, []);
+
+    return (
+        <div className="relative">
+            <div
+                className={`${expanded ? "fixed inset-0 bg-black bg-opacity-50 backdrop-blur-sm z-50 flex items-center justify-center" : ""}`}
+            >
+                <Button
+                    onClick={() => {
+                        setExpanded(!expanded);
+                        // Trigger a resize event to make Excalidraw adjust its size
+                        window.dispatchEvent(new Event("resize"));
+                    }}
+                    variant={"outline"}
+                    className={`${expanded ? "absolute top-2 left-2 z-[60]" : ""}`}
+                >
+                    {expanded ? (
+                        <ArrowsInSimple className="h-4 w-4" />
+                    ) : (
+                        <ArrowsOutSimple className="h-4 w-4" />
+                    )}
+                </Button>
+                <div
+                    className={`
+                    ${expanded ? "w-[80vw] h-[80vh]" : "w-full h-[500px]"}
+                    bg-white overflow-hidden rounded-lg relative
+                `}
+                >
+                    <Excalidraw
+                        initialData={{
+                            elements: excalidrawElements,
+                            appState: { zenModeEnabled: true },
+                            scrollToContent: true,
+                        }}
+                        // TODO - Create a common function to detect if the theme is dark?
+                        theme={localStorage.getItem("theme") === "dark" ? "dark" : "light"}
+                        validateEmbeddable={true}
+                        renderTopRightUI={(isMobile, appState) => {
+                            return <></>;
+                        }}
+                    />
+                </div>
+            </div>
+        </div>
+    );
+}