Merge branch 'master' into improve-agent-pane-on-home-screen

This commit is contained in:
Debanjum Singh Solanky
2024-10-22 20:05:29 -07:00
51 changed files with 1472 additions and 399 deletions

View File

@@ -79,7 +79,7 @@ div.titleBar {
div.chatBoxBody {
display: grid;
height: 100%;
width: 70%;
width: 95%;
margin: auto;
}

View File

@@ -47,7 +47,14 @@ export default function RootLayout({
child-src 'none';
object-src 'none';"
></meta>
<body className={inter.className}>{children}</body>
<body className={inter.className}>
{children}
<script
dangerouslySetInnerHTML={{
__html: `window.EXCALIDRAW_ASSET_PATH = 'https://assets.khoj.dev/@excalidraw/excalidraw/dist/';`,
}}
/>
</body>
</html>
);
}

View File

@@ -27,32 +27,37 @@ interface ChatBodyDataProps {
setUploadedFiles: (files: string[]) => void;
isMobileWidth?: boolean;
isLoggedIn: boolean;
setImage64: (image64: string) => void;
setImages: (images: string[]) => void;
}
function ChatBodyData(props: ChatBodyDataProps) {
const searchParams = useSearchParams();
const conversationId = searchParams.get("conversationId");
const [message, setMessage] = useState("");
const [image, setImage] = useState<string | null>(null);
const [images, setImages] = useState<string[]>([]);
const [processingMessage, setProcessingMessage] = useState(false);
const [agentMetadata, setAgentMetadata] = useState<AgentData | null>(null);
const setQueryToProcess = props.setQueryToProcess;
const onConversationIdChange = props.onConversationIdChange;
useEffect(() => {
if (image) {
props.setImage64(encodeURIComponent(image));
}
}, [image, props.setImage64]);
const chatHistoryCustomClassName = props.isMobileWidth ? "w-full" : "w-4/6";
useEffect(() => {
const storedImage = localStorage.getItem("image");
if (storedImage) {
setImage(storedImage);
props.setImage64(encodeURIComponent(storedImage));
localStorage.removeItem("image");
if (images.length > 0) {
const encodedImages = images.map((image) => encodeURIComponent(image));
props.setImages(encodedImages);
}
}, [images, props.setImages]);
useEffect(() => {
const storedImages = localStorage.getItem("images");
if (storedImages) {
const parsedImages: string[] = JSON.parse(storedImages);
setImages(parsedImages);
const encodedImages = parsedImages.map((img: string) => encodeURIComponent(img));
props.setImages(encodedImages);
localStorage.removeItem("images");
}
const storedMessage = localStorage.getItem("message");
@@ -60,7 +65,7 @@ function ChatBodyData(props: ChatBodyDataProps) {
setProcessingMessage(true);
setQueryToProcess(storedMessage);
}
}, [setQueryToProcess]);
}, [setQueryToProcess, props.setImages]);
useEffect(() => {
if (message) {
@@ -82,6 +87,7 @@ function ChatBodyData(props: ChatBodyDataProps) {
props.streamedMessages[props.streamedMessages.length - 1].completed
) {
setProcessingMessage(false);
setImages([]); // Reset images after processing
} else {
setMessage("");
}
@@ -101,16 +107,17 @@ function ChatBodyData(props: ChatBodyDataProps) {
setAgent={setAgentMetadata}
pendingMessage={processingMessage ? message : ""}
incomingMessages={props.streamedMessages}
customClassName={chatHistoryCustomClassName}
/>
</div>
<div
className={`${styles.inputBox} p-1 md:px-2 shadow-md bg-background align-middle items-center justify-center dark:bg-neutral-700 dark:border-0 dark:shadow-sm rounded-t-2xl rounded-b-none md:rounded-xl h-fit`}
className={`${styles.inputBox} p-1 md:px-2 shadow-md bg-background align-middle items-center justify-center dark:bg-neutral-700 dark:border-0 dark:shadow-sm rounded-t-2xl rounded-b-none md:rounded-xl h-fit ${chatHistoryCustomClassName} mr-auto ml-auto`}
>
<ChatInputArea
agentColor={agentMetadata?.color}
isLoggedIn={props.isLoggedIn}
sendMessage={(message) => setMessage(message)}
sendImage={(image) => setImage(image)}
sendImage={(image) => setImages((prevImages) => [...prevImages, image])}
sendDisabled={processingMessage}
chatOptionsData={props.chatOptionsData}
conversationId={conversationId}
@@ -132,7 +139,7 @@ export default function Chat() {
const [queryToProcess, setQueryToProcess] = useState<string>("");
const [processQuerySignal, setProcessQuerySignal] = useState(false);
const [uploadedFiles, setUploadedFiles] = useState<string[]>([]);
const [image64, setImage64] = useState<string>("");
const [images, setImages] = useState<string[]>([]);
const locationData = useIPLocationData() || {
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
@@ -168,7 +175,7 @@ export default function Chat() {
completed: false,
timestamp: new Date().toISOString(),
rawQuery: queryToProcess || "",
uploadedImageData: decodeURIComponent(image64),
images: images,
};
setMessages((prevMessages) => [...prevMessages, newStreamMessage]);
setProcessQuerySignal(true);
@@ -199,7 +206,7 @@ export default function Chat() {
if (done) {
setQueryToProcess("");
setProcessQuerySignal(false);
setImage64("");
setImages([]);
break;
}
@@ -247,7 +254,7 @@ export default function Chat() {
country_code: locationData.countryCode,
timezone: locationData.timezone,
}),
...(image64 && { image: image64 }),
...(images.length > 0 && { images: images }),
};
const response = await fetch(chatAPI, {
@@ -261,7 +268,8 @@ export default function Chat() {
try {
await readChatStream(response);
} catch (err) {
console.error(err);
const apiError = await response.json();
console.error(apiError);
// Retrieve latest message being processed
const currentMessage = messages.find((message) => !message.completed);
if (!currentMessage) return;
@@ -270,7 +278,11 @@ export default function Chat() {
const errorMessage = (err as Error).message;
if (errorMessage.includes("Error in input stream"))
currentMessage.rawResponse = `Woops! The connection broke while I was writing my thoughts down. Maybe try again in a bit or dislike this message if the issue persists?`;
else
else if (response.status === 429) {
"detail" in apiError
? (currentMessage.rawResponse = `${apiError.detail}`)
: (currentMessage.rawResponse = `I'm a bit overwhelmed at the moment. Could you try again in a bit or dislike this message if the issue persists?`);
} else
currentMessage.rawResponse = `Umm, not sure what just happened. I see this error message: ${errorMessage}. Could you try again or dislike this message if the issue persists?`;
// Complete message streaming teardown properly
@@ -329,7 +341,7 @@ export default function Chat() {
setUploadedFiles={setUploadedFiles}
isMobileWidth={isMobileWidth}
onConversationIdChange={handleConversationIdChange}
setImage64={setImage64}
setImages={setImages}
/>
</Suspense>
</div>

View File

@@ -5,10 +5,10 @@ export interface RawReferenceData {
onlineContext?: OnlineContext;
}
export interface ResponseWithReferences {
context?: Context[];
online?: OnlineContext;
response?: string;
export interface ResponseWithIntent {
intentType: string;
response: string;
inferredQueries?: string[];
}
interface MessageChunk {
@@ -49,10 +49,14 @@ export function convertMessageChunkToJson(chunk: string): MessageChunk {
function handleJsonResponse(chunkData: any) {
const jsonData = chunkData as any;
if (jsonData.image || jsonData.detail) {
let responseWithReference = handleImageResponse(chunkData, true);
if (responseWithReference.response) return responseWithReference.response;
let responseWithIntent = handleImageResponse(chunkData, true);
return responseWithIntent;
} else if (jsonData.response) {
return jsonData.response;
return {
response: jsonData.response,
intentType: "",
inferredQueries: [],
};
} else {
throw new Error("Invalid JSON response");
}
@@ -80,8 +84,18 @@ export function processMessageChunk(
return { context, onlineContext };
} else if (chunk.type === "message") {
const chunkData = chunk.data;
// Here, handle if the response is a JSON response with an image, but the intentType is excalidraw
if (chunkData !== null && typeof chunkData === "object") {
currentMessage.rawResponse += handleJsonResponse(chunkData);
let responseWithIntent = handleJsonResponse(chunkData);
if (responseWithIntent.intentType && responseWithIntent.intentType === "excalidraw") {
currentMessage.rawResponse = responseWithIntent.response;
} else {
currentMessage.rawResponse += responseWithIntent.response;
}
currentMessage.intentType = responseWithIntent.intentType;
currentMessage.inferredQueries = responseWithIntent.inferredQueries;
} else if (
typeof chunkData === "string" &&
chunkData.trim()?.startsWith("{") &&
@@ -89,7 +103,10 @@ export function processMessageChunk(
) {
try {
const jsonData = JSON.parse(chunkData.trim());
currentMessage.rawResponse += handleJsonResponse(jsonData);
let responseWithIntent = handleJsonResponse(jsonData);
currentMessage.rawResponse += responseWithIntent.response;
currentMessage.intentType = responseWithIntent.intentType;
currentMessage.inferredQueries = responseWithIntent.inferredQueries;
} catch (e) {
currentMessage.rawResponse += JSON.stringify(chunkData);
}
@@ -111,42 +128,26 @@ export function processMessageChunk(
return { context, onlineContext };
}
export function handleImageResponse(imageJson: any, liveStream: boolean): ResponseWithReferences {
export function handleImageResponse(imageJson: any, liveStream: boolean): ResponseWithIntent {
let rawResponse = "";
if (imageJson.image) {
const inferredQuery = imageJson.inferredQueries?.[0] ?? "generated image";
// If response has image field, response is a generated image.
if (imageJson.intentType === "text-to-image") {
rawResponse += `![generated_image](data:image/png;base64,${imageJson.image})`;
} else if (imageJson.intentType === "text-to-image2") {
rawResponse += `![generated_image](${imageJson.image})`;
} else if (imageJson.intentType === "text-to-image-v3") {
rawResponse = `![](data:image/webp;base64,${imageJson.image})`;
}
if (inferredQuery && !liveStream) {
rawResponse += `\n\n${inferredQuery}`;
}
// If response has image field, response may be a generated image
rawResponse = imageJson.image;
}
let reference: ResponseWithReferences = {};
let responseWithIntent: ResponseWithIntent = {
intentType: imageJson.intentType,
response: rawResponse,
inferredQueries: imageJson.inferredQueries,
};
if (imageJson.context && imageJson.context.length > 0) {
const rawReferenceAsJson = imageJson.context;
if (rawReferenceAsJson instanceof Array) {
reference.context = rawReferenceAsJson;
} else if (typeof rawReferenceAsJson === "object" && rawReferenceAsJson !== null) {
reference.online = rawReferenceAsJson;
}
}
if (imageJson.detail) {
// The detail field contains the improved image prompt
rawResponse += imageJson.detail;
}
reference.response = rawResponse;
return reference;
return responseWithIntent;
}
export function modifyFileFilterForConversation(

View File

@@ -48,6 +48,7 @@ import {
Oven,
Gavel,
Broadcast,
KeyReturn,
} from "@phosphor-icons/react";
import { Markdown, OrgMode, Pdf, Word } from "@/app/components/logo/fileLogo";
@@ -193,6 +194,10 @@ export function getIconForSlashCommand(command: string, customClassName: string
}
if (command.includes("default")) {
return <KeyReturn className={className} />;
}
if (command.includes("diagram")) {
return <Shapes className={className} />;
}

View File

@@ -2,12 +2,7 @@ div.chatHistory {
display: flex;
flex-direction: column;
height: 100%;
}
div.chatLayout {
height: 80vh;
overflow-y: auto;
margin: 0 auto;
margin: auto;
}
div.agentIndicator a {

View File

@@ -37,6 +37,7 @@ interface ChatHistoryProps {
pendingMessage?: string;
publicConversationSlug?: string;
setAgent: (agent: AgentData) => void;
customClassName?: string;
}
function constructTrainOfThought(
@@ -255,7 +256,7 @@ export default function ChatHistory(props: ChatHistoryProps) {
return (
<ScrollArea className={`h-[80vh] relative`} ref={scrollAreaRef}>
<div>
<div className={styles.chatHistory}>
<div className={`${styles.chatHistory} ${props.customClassName}`}>
<div ref={sentinelRef} style={{ height: "1px" }}>
{fetchingData && (
<InlineLoading message="Loading Conversation" className="opacity-50" />
@@ -298,7 +299,7 @@ export default function ChatHistory(props: ChatHistoryProps) {
created: message.timestamp,
by: "you",
automationId: "",
uploadedImageData: message.uploadedImageData,
images: message.images,
}}
customClassName="fullHistory"
borderLeftColor={`${data?.agent?.color}-500`}
@@ -322,6 +323,12 @@ export default function ChatHistory(props: ChatHistoryProps) {
by: "khoj",
automationId: "",
rawQuery: message.rawQuery,
intent: {
type: message.intentType || "",
query: message.rawQuery,
"memory-type": "",
"inferred-queries": message.inferredQueries || [],
},
}}
customClassName="fullHistory"
borderLeftColor={`${data?.agent?.color}-500`}
@@ -341,7 +348,6 @@ export default function ChatHistory(props: ChatHistoryProps) {
created: new Date().getTime().toString(),
by: "you",
automationId: "",
uploadedImageData: props.pendingMessage,
}}
customClassName="fullHistory"
borderLeftColor={`${data?.agent?.color}-500`}
@@ -366,18 +372,20 @@ export default function ChatHistory(props: ChatHistoryProps) {
</div>
)}
</div>
{!isNearBottom && (
<button
title="Scroll to bottom"
className="absolute bottom-4 right-5 bg-white dark:bg-[hsl(var(--background))] text-neutral-500 dark:text-white p-2 rounded-full shadow-xl"
onClick={() => {
scrollToBottom();
setIsNearBottom(true);
}}
>
<ArrowDown size={24} />
</button>
)}
<div className={`${props.customClassName} fixed bottom-[15%] z-10`}>
{!isNearBottom && (
<button
title="Scroll to bottom"
className="absolute bottom-0 right-0 bg-white dark:bg-[hsl(var(--background))] text-neutral-500 dark:text-white p-2 rounded-full shadow-xl"
onClick={() => {
scrollToBottom();
setIsNearBottom(true);
}}
>
<ArrowDown size={24} />
</button>
)}
</div>
</div>
</ScrollArea>
);

View File

@@ -62,10 +62,11 @@ export const ChatInputArea = forwardRef<HTMLTextAreaElement, ChatInputProps>((pr
const [loginRedirectMessage, setLoginRedirectMessage] = useState<string | null>(null);
const [showLoginPrompt, setShowLoginPrompt] = useState(false);
const [recording, setRecording] = useState(false);
const [imageUploaded, setImageUploaded] = useState(false);
const [imagePath, setImagePath] = useState<string>("");
const [imageData, setImageData] = useState<string | null>(null);
const [imagePaths, setImagePaths] = useState<string[]>([]);
const [imageData, setImageData] = useState<string[]>([]);
const [recording, setRecording] = useState(false);
const [mediaRecorder, setMediaRecorder] = useState<MediaRecorder | null>(null);
const [progressValue, setProgressValue] = useState(0);
@@ -90,27 +91,31 @@ export const ChatInputArea = forwardRef<HTMLTextAreaElement, ChatInputProps>((pr
useEffect(() => {
async function fetchImageData() {
if (imagePath) {
const response = await fetch(imagePath);
const blob = await response.blob();
const reader = new FileReader();
reader.onload = function () {
const base64data = reader.result;
setImageData(base64data as string);
};
reader.readAsDataURL(blob);
if (imagePaths.length > 0) {
const newImageData = await Promise.all(
imagePaths.map(async (path) => {
const response = await fetch(path);
const blob = await response.blob();
return new Promise<string>((resolve) => {
const reader = new FileReader();
reader.onload = () => resolve(reader.result as string);
reader.readAsDataURL(blob);
});
}),
);
setImageData(newImageData);
}
setUploading(false);
}
setUploading(true);
fetchImageData();
}, [imagePath]);
}, [imagePaths]);
function onSendMessage() {
if (imageUploaded) {
setImageUploaded(false);
setImagePath("");
props.sendImage(imageData || "");
setImagePaths([]);
imageData.forEach((data) => props.sendImage(data));
}
if (!message.trim()) return;
@@ -156,18 +161,23 @@ export const ChatInputArea = forwardRef<HTMLTextAreaElement, ChatInputProps>((pr
setShowLoginPrompt(true);
return;
}
// check for image file
// check for image files
const image_endings = ["jpg", "jpeg", "png", "webp"];
const newImagePaths: string[] = [];
for (let i = 0; i < files.length; i++) {
const file = files[i];
const file_extension = file.name.split(".").pop();
if (image_endings.includes(file_extension || "")) {
setImageUploaded(true);
setImagePath(DOMPurify.sanitize(URL.createObjectURL(file)));
return;
newImagePaths.push(DOMPurify.sanitize(URL.createObjectURL(file)));
}
}
if (newImagePaths.length > 0) {
setImageUploaded(true);
setImagePaths((prevPaths) => [...prevPaths, ...newImagePaths]);
return;
}
uploadDataForIndexing(
files,
setWarning,
@@ -272,9 +282,12 @@ export const ChatInputArea = forwardRef<HTMLTextAreaElement, ChatInputProps>((pr
setIsDragAndDropping(false);
}
function removeImageUpload() {
setImageUploaded(false);
setImagePath("");
function removeImageUpload(index: number) {
setImagePaths((prevPaths) => prevPaths.filter((_, i) => i !== index));
setImageData((prevData) => prevData.filter((_, i) => i !== index));
if (imagePaths.length === 1) {
setImageUploaded(false);
}
}
return (
@@ -391,24 +404,11 @@ export const ChatInputArea = forwardRef<HTMLTextAreaElement, ChatInputProps>((pr
</div>
)}
<div
className={`${styles.actualInputArea} items-center justify-between dark:bg-neutral-700 relative ${isDragAndDropping && "animate-pulse"}`}
className={`${styles.actualInputArea} justify-between dark:bg-neutral-700 relative ${isDragAndDropping && "animate-pulse"}`}
onDragOver={handleDragOver}
onDragLeave={handleDragLeave}
onDrop={handleDragAndDropFiles}
>
{imageUploaded && (
<div className="absolute bottom-[80px] left-0 right-0 dark:bg-neutral-700 bg-white pt-5 pb-5 w-full rounded-lg border dark:border-none grid grid-cols-2">
<div className="pl-4 pr-4">
<img src={imagePath} alt="img" className="w-auto max-h-[100px]" />
</div>
<div className="pl-4 pr-4">
<X
className="w-6 h-6 float-right dark:hover:bg-[hsl(var(--background))] hover:bg-neutral-100 rounded-sm"
onClick={removeImageUpload}
/>
</div>
</div>
)}
<input
type="file"
multiple={true}
@@ -416,15 +416,37 @@ export const ChatInputArea = forwardRef<HTMLTextAreaElement, ChatInputProps>((pr
onChange={handleFileChange}
style={{ display: "none" }}
/>
<Button
variant={"ghost"}
className="!bg-none p-0 m-2 h-auto text-3xl rounded-full text-gray-300 hover:text-gray-500"
disabled={props.sendDisabled}
onClick={handleFileButtonClick}
>
<Paperclip className="w-8 h-8" />
</Button>
<div className="grid w-full gap-1.5 relative">
<div className="flex items-end pb-4">
<Button
variant={"ghost"}
className="!bg-none p-0 m-2 h-auto text-3xl rounded-full text-gray-300 hover:text-gray-500"
disabled={props.sendDisabled}
onClick={handleFileButtonClick}
>
<Paperclip className="w-8 h-8" />
</Button>
</div>
<div className="flex-grow flex flex-col w-full gap-1.5 relative pb-2">
<div className="flex items-center gap-2 overflow-x-auto">
{imageUploaded &&
imagePaths.map((path, index) => (
<div key={index} className="relative flex-shrink-0 pb-3 pt-2 group">
<img
src={path}
alt={`img-${index}`}
className="w-auto h-16 object-cover rounded-xl"
/>
<Button
variant="ghost"
size="icon"
className="absolute -top-0 -right-2 h-5 w-5 rounded-full bg-neutral-200 dark:bg-neutral-600 hover:bg-neutral-300 dark:hover:bg-neutral-500 opacity-0 group-hover:opacity-100 transition-opacity"
onClick={() => removeImageUpload(index)}
>
<X className="h-3 w-3" />
</Button>
</div>
))}
</div>
<Textarea
ref={chatInputRef}
className={`border-none w-full h-16 min-h-16 max-h-[128px] md:py-4 rounded-lg resize-none dark:bg-neutral-700 ${props.isMobileWidth ? "text-md" : "text-lg"}`}
@@ -435,7 +457,7 @@ export const ChatInputArea = forwardRef<HTMLTextAreaElement, ChatInputProps>((pr
onKeyDown={(e) => {
if (e.key === "Enter" && !e.shiftKey) {
setImageUploaded(false);
setImagePath("");
setImagePaths([]);
e.preventDefault();
onSendMessage();
}
@@ -444,57 +466,59 @@ export const ChatInputArea = forwardRef<HTMLTextAreaElement, ChatInputProps>((pr
disabled={props.sendDisabled || recording}
/>
</div>
{recording ? (
<TooltipProvider>
<Tooltip>
<TooltipTrigger asChild>
<Button
variant="default"
className={`${!recording && "hidden"} ${props.agentColor ? convertToBGClass(props.agentColor) : "bg-orange-300 hover:bg-orange-500"} rounded-full p-1 m-2 h-auto text-3xl transition transform md:hover:-translate-y-1`}
onClick={() => {
setRecording(!recording);
}}
disabled={props.sendDisabled}
>
<Stop weight="fill" className="w-6 h-6" />
</Button>
</TooltipTrigger>
<TooltipContent>
Click to stop recording and transcribe your voice.
</TooltipContent>
</Tooltip>
</TooltipProvider>
) : mediaRecorder ? (
<InlineLoading />
) : (
<TooltipProvider>
<Tooltip>
<TooltipTrigger asChild>
<Button
variant="default"
className={`${!message || recording || "hidden"} ${props.agentColor ? convertToBGClass(props.agentColor) : "bg-orange-300 hover:bg-orange-500"} rounded-full p-1 m-2 h-auto text-3xl transition transform md:hover:-translate-y-1`}
onClick={() => {
setMessage("Listening...");
setRecording(!recording);
}}
disabled={props.sendDisabled}
>
<Microphone weight="fill" className="w-6 h-6" />
</Button>
</TooltipTrigger>
<TooltipContent>
Click to transcribe your message with voice.
</TooltipContent>
</Tooltip>
</TooltipProvider>
)}
<Button
className={`${(!message || recording) && "hidden"} ${props.agentColor ? convertToBGClass(props.agentColor) : "bg-orange-300 hover:bg-orange-500"} rounded-full p-1 m-2 h-auto text-3xl transition transform md:hover:-translate-y-1`}
onClick={onSendMessage}
disabled={props.sendDisabled}
>
<ArrowUp className="w-6 h-6" weight="bold" />
</Button>
<div className="flex items-end pb-4">
{recording ? (
<TooltipProvider>
<Tooltip>
<TooltipTrigger asChild>
<Button
variant="default"
className={`${!recording && "hidden"} ${props.agentColor ? convertToBGClass(props.agentColor) : "bg-orange-300 hover:bg-orange-500"} rounded-full p-1 m-2 h-auto text-3xl transition transform md:hover:-translate-y-1`}
onClick={() => {
setRecording(!recording);
}}
disabled={props.sendDisabled}
>
<Stop weight="fill" className="w-6 h-6" />
</Button>
</TooltipTrigger>
<TooltipContent>
Click to stop recording and transcribe your voice.
</TooltipContent>
</Tooltip>
</TooltipProvider>
) : mediaRecorder ? (
<InlineLoading />
) : (
<TooltipProvider>
<Tooltip>
<TooltipTrigger asChild>
<Button
variant="default"
className={`${!message || recording || "hidden"} ${props.agentColor ? convertToBGClass(props.agentColor) : "bg-orange-300 hover:bg-orange-500"} rounded-full p-1 m-2 h-auto text-3xl transition transform md:hover:-translate-y-1`}
onClick={() => {
setMessage("Listening...");
setRecording(!recording);
}}
disabled={props.sendDisabled}
>
<Microphone weight="fill" className="w-6 h-6" />
</Button>
</TooltipTrigger>
<TooltipContent>
Click to transcribe your message with voice.
</TooltipContent>
</Tooltip>
</TooltipProvider>
)}
<Button
className={`${(!message || recording) && "hidden"} ${props.agentColor ? convertToBGClass(props.agentColor) : "bg-orange-300 hover:bg-orange-500"} rounded-full p-1 m-2 h-auto text-3xl transition transform md:hover:-translate-y-1`}
onClick={onSendMessage}
disabled={props.sendDisabled}
>
<ArrowUp className="w-6 h-6" weight="bold" />
</Button>
</div>
</div>
</>
);

View File

@@ -57,7 +57,26 @@ div.emptyChatMessage {
display: none;
}
div.chatMessageContainer img {
div.imagesContainer {
display: flex;
overflow-x: auto;
padding-bottom: 8px;
margin-bottom: 8px;
}
div.imageWrapper {
flex: 0 0 auto;
margin-right: 8px;
}
div.imageWrapper img {
width: auto;
height: 128px;
object-fit: cover;
border-radius: 8px;
}
div.chatMessageContainer > img {
width: auto;
height: auto;
max-width: 100%;

View File

@@ -26,6 +26,7 @@ import {
Palette,
ClipboardText,
Check,
Shapes,
} from "@phosphor-icons/react";
import DOMPurify from "dompurify";
@@ -35,6 +36,7 @@ import { AgentData } from "@/app/agents/page";
import renderMathInElement from "katex/contrib/auto-render";
import "katex/dist/katex.min.css";
import ExcalidrawComponent from "../excalidraw/excalidraw";
const md = new markdownIt({
html: true,
@@ -114,7 +116,7 @@ export interface SingleChatMessage {
rawQuery?: string;
intent?: Intent;
agent?: AgentData;
uploadedImageData?: string;
images?: string[];
}
export interface StreamMessage {
@@ -126,7 +128,9 @@ export interface StreamMessage {
rawQuery: string;
timestamp: string;
agent?: AgentData;
uploadedImageData?: string;
images?: string[];
intentType?: string;
inferredQueries?: string[];
}
export interface ChatHistoryData {
@@ -208,7 +212,6 @@ interface ChatMessageProps {
borderLeftColor?: string;
isLastMessage?: boolean;
agent?: AgentData;
uploadedImageData?: string;
}
interface TrainOfThoughtProps {
@@ -252,6 +255,10 @@ function chooseIconFromHeader(header: string, iconColor: string) {
return <Aperture className={`${classNames}`} />;
}
if (compareHeader.includes("diagram")) {
return <Shapes className={`${classNames}`} />;
}
if (compareHeader.includes("paint")) {
return <Palette className={`${classNames}`} />;
}
@@ -283,6 +290,7 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
const [markdownRendered, setMarkdownRendered] = useState<string>("");
const [isPlaying, setIsPlaying] = useState<boolean>(false);
const [interrupted, setInterrupted] = useState<boolean>(false);
const [excalidrawData, setExcalidrawData] = useState<string>("");
const interruptedRef = useRef<boolean>(false);
const messageRef = useRef<HTMLDivElement>(null);
@@ -321,6 +329,11 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
useEffect(() => {
let message = props.chatMessage.message;
if (props.chatMessage.intent && props.chatMessage.intent.type == "excalidraw") {
message = props.chatMessage.intent["inferred-queries"][0];
setExcalidrawData(props.chatMessage.message);
}
// Replace LaTeX delimiters with placeholders
message = message
.replace(/\\\(/g, "LEFTPAREN")
@@ -328,26 +341,40 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
.replace(/\\\[/g, "LEFTBRACKET")
.replace(/\\\]/g, "RIGHTBRACKET");
if (props.chatMessage.uploadedImageData) {
message = `![uploaded image](${props.chatMessage.uploadedImageData})\n\n${message}`;
if (props.chatMessage.images && props.chatMessage.images.length > 0) {
const imagesInMd = props.chatMessage.images
.map((image, index) => {
const decodedImage = image.startsWith("data%3Aimage")
? decodeURIComponent(image)
: image;
const sanitizedImage = DOMPurify.sanitize(decodedImage);
return `<div class="${styles.imageWrapper}"><img src="${sanitizedImage}" alt="uploaded image ${index + 1}" /></div>`;
})
.join("");
message = `<div class="${styles.imagesContainer}">${imagesInMd}</div>${message}`;
}
if (props.chatMessage.intent && props.chatMessage.intent.type == "text-to-image") {
message = `![generated image](data:image/png;base64,${message})`;
} else if (props.chatMessage.intent && props.chatMessage.intent.type == "text-to-image2") {
message = `![generated image](${message})`;
} else if (
props.chatMessage.intent &&
props.chatMessage.intent.type == "text-to-image-v3"
) {
message = `![generated image](data:image/webp;base64,${message})`;
}
if (
props.chatMessage.intent &&
props.chatMessage.intent.type.includes("text-to-image") &&
props.chatMessage.intent["inferred-queries"]?.length > 0
) {
message += `\n\n${props.chatMessage.intent["inferred-queries"][0]}`;
const intentTypeHandlers = {
"text-to-image": (msg: string) => `![generated image](data:image/png;base64,${msg})`,
"text-to-image2": (msg: string) => `![generated image](${msg})`,
"text-to-image-v3": (msg: string) =>
`![generated image](data:image/webp;base64,${msg})`,
excalidraw: (msg: string) => {
return msg;
},
};
if (props.chatMessage.intent) {
const { type, "inferred-queries": inferredQueries } = props.chatMessage.intent;
console.log("intent type", type);
if (type in intentTypeHandlers) {
message = intentTypeHandlers[type as keyof typeof intentTypeHandlers](message);
}
if (type.includes("text-to-image") && inferredQueries?.length > 0) {
message += `\n\n${inferredQueries[0]}`;
}
}
setTextRendered(message);
@@ -364,7 +391,7 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
// Sanitize and set the rendered markdown
setMarkdownRendered(DOMPurify.sanitize(markdownRendered));
}, [props.chatMessage.message, props.chatMessage.intent]);
}, [props.chatMessage.message, props.chatMessage.images, props.chatMessage.intent]);
useEffect(() => {
if (copySuccess) {
@@ -554,6 +581,7 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
className={styles.chatMessage}
dangerouslySetInnerHTML={{ __html: markdownRendered }}
/>
{excalidrawData && <ExcalidrawComponent data={excalidrawData} />}
</div>
<div className={styles.teaserReferencesContainer}>
<TeaserReferencesSection

View File

@@ -0,0 +1,24 @@
"use client";
import dynamic from "next/dynamic";
import { Suspense } from "react";
import Loading from "../../components/loading/loading";
// Since client components get prerenderd on server as well hence importing
// the excalidraw stuff dynamically with ssr false
const ExcalidrawWrapper = dynamic(() => import("./excalidrawWrapper").then((mod) => mod.default), {
ssr: false,
});
interface ExcalidrawComponentProps {
data: any;
}
export default function ExcalidrawComponent(props: ExcalidrawComponentProps) {
return (
<Suspense fallback={<Loading />}>
<ExcalidrawWrapper data={props.data} />
</Suspense>
);
}

View File

@@ -0,0 +1,149 @@
"use client";
import { useState, useEffect } from "react";
import dynamic from "next/dynamic";
import { ExcalidrawProps } from "@excalidraw/excalidraw/types/types";
import { ExcalidrawElement } from "@excalidraw/excalidraw/types/element/types";
import { ExcalidrawElementSkeleton } from "@excalidraw/excalidraw/types/data/transform";
const Excalidraw = dynamic<ExcalidrawProps>(
async () => (await import("@excalidraw/excalidraw")).Excalidraw,
{
ssr: false,
},
);
import { convertToExcalidrawElements } from "@excalidraw/excalidraw";
import { Button } from "@/components/ui/button";
import { ArrowsInSimple, ArrowsOutSimple } from "@phosphor-icons/react";
interface ExcalidrawWrapperProps {
data: ExcalidrawElementSkeleton[];
}
export default function ExcalidrawWrapper(props: ExcalidrawWrapperProps) {
const [excalidrawElements, setExcalidrawElements] = useState<ExcalidrawElement[]>([]);
const [expanded, setExpanded] = useState<boolean>(false);
const isValidExcalidrawElement = (element: ExcalidrawElementSkeleton): boolean => {
return (
element.x !== undefined &&
element.y !== undefined &&
element.id !== undefined &&
element.type !== undefined
);
};
useEffect(() => {
if (expanded) {
onkeydown = (e) => {
if (e.key === "Escape") {
setExpanded(false);
// Trigger a resize event to make Excalidraw adjust its size
window.dispatchEvent(new Event("resize"));
}
};
} else {
onkeydown = null;
}
}, [expanded]);
useEffect(() => {
// Do some basic validation
const basicValidSkeletons: ExcalidrawElementSkeleton[] = [];
for (const element of props.data) {
if (isValidExcalidrawElement(element as ExcalidrawElementSkeleton)) {
basicValidSkeletons.push(element as ExcalidrawElementSkeleton);
}
}
const validSkeletons: ExcalidrawElementSkeleton[] = [];
for (const element of basicValidSkeletons) {
if (element.type === "frame") {
continue;
}
if (element.type === "arrow") {
const start = basicValidSkeletons.find((child) => child.id === element.start?.id);
const end = basicValidSkeletons.find((child) => child.id === element.end?.id);
if (start && end) {
validSkeletons.push(element);
}
} else {
validSkeletons.push(element);
}
}
for (const element of basicValidSkeletons) {
if (element.type === "frame") {
const children = element.children?.map((childId) => {
return validSkeletons.find((child) => child.id === childId);
});
// Get the valid children, filter out any undefined values
const validChildrenIds: readonly string[] = children
?.map((child) => child?.id)
.filter((id) => id !== undefined) as string[];
if (validChildrenIds === undefined || validChildrenIds.length === 0) {
continue;
}
validSkeletons.push({
...element,
children: validChildrenIds,
});
}
}
const elements = convertToExcalidrawElements(validSkeletons);
setExcalidrawElements(elements);
}, []);
return (
<div className="relative">
<div
className={`${expanded ? "fixed inset-0 bg-black bg-opacity-50 backdrop-blur-sm z-50 flex items-center justify-center" : ""}`}
>
<Button
onClick={() => {
setExpanded(!expanded);
// Trigger a resize event to make Excalidraw adjust its size
window.dispatchEvent(new Event("resize"));
}}
variant={"outline"}
className={`${expanded ? "absolute top-2 left-2 z-[60]" : ""}`}
>
{expanded ? (
<ArrowsInSimple className="h-4 w-4" />
) : (
<ArrowsOutSimple className="h-4 w-4" />
)}
</Button>
<div
className={`
${expanded ? "w-[80vw] h-[80vh]" : "w-full h-[500px]"}
bg-white overflow-hidden rounded-lg relative
`}
>
<Excalidraw
initialData={{
elements: excalidrawElements,
appState: { zenModeEnabled: true },
scrollToContent: true,
}}
// TODO - Create a common function to detect if the theme is dark?
theme={localStorage.getItem("theme") === "dark" ? "dark" : "light"}
validateEmbeddable={true}
renderTopRightUI={(isMobile, appState) => {
return <></>;
}}
/>
</div>
</div>
</div>
);
}

View File

@@ -51,7 +51,7 @@ function FisherYatesShuffle(array: any[]) {
function ChatBodyData(props: ChatBodyDataProps) {
const [message, setMessage] = useState("");
const [image, setImage] = useState<string | null>(null);
const [images, setImages] = useState<string[]>([]);
const [processingMessage, setProcessingMessage] = useState(false);
const [greeting, setGreeting] = useState("");
const [shuffledOptions, setShuffledOptions] = useState<Suggestion[]>([]);
@@ -151,20 +151,21 @@ function ChatBodyData(props: ChatBodyDataProps) {
try {
const newConversationId = await createNewConversation(selectedAgent || "khoj");
onConversationIdChange?.(newConversationId);
window.location.href = `/chat?conversationId=${newConversationId}`;
localStorage.setItem("message", message);
if (image) {
localStorage.setItem("image", image);
if (images.length > 0) {
localStorage.setItem("images", JSON.stringify(images));
}
window.location.href = `/chat?conversationId=${newConversationId}`;
} catch (error) {
console.error("Error creating new conversation:", error);
setProcessingMessage(false);
}
setMessage("");
setImages([]);
}
};
processMessage();
if (message) {
if (message || images.length > 0) {
setProcessingMessage(true);
}
}, [selectedAgent, message, processingMessage, onConversationIdChange]);
@@ -290,7 +291,7 @@ function ChatBodyData(props: ChatBodyDataProps) {
</ScrollArea>
)}
</div>
<div className={`mx-auto ${props.isMobileWidth ? "w-full" : "w-fit"}`}>
<div className={`mx-auto ${props.isMobileWidth ? "w-full" : "w-fit max-w-screen-md"}`}>
{!props.isMobileWidth && (
<div
className={`w-full ${styles.inputBox} shadow-lg bg-background align-middle items-center justify-center px-3 py-1 dark:bg-neutral-700 border-stone-100 dark:border-none dark:shadow-none rounded-2xl`}
@@ -298,7 +299,7 @@ function ChatBodyData(props: ChatBodyDataProps) {
<ChatInputArea
isLoggedIn={props.isLoggedIn}
sendMessage={(message) => setMessage(message)}
sendImage={(image) => setImage(image)}
sendImage={(image) => setImages((prevImages) => [...prevImages, image])}
sendDisabled={processingMessage}
chatOptionsData={props.chatOptionsData}
conversationId={null}
@@ -379,7 +380,7 @@ function ChatBodyData(props: ChatBodyDataProps) {
<ChatInputArea
isLoggedIn={props.isLoggedIn}
sendMessage={(message) => setMessage(message)}
sendImage={(image) => setImage(image)}
sendImage={(image) => setImages((prevImages) => [...prevImages, image])}
sendDisabled={processingMessage}
chatOptionsData={props.chatOptionsData}
conversationId={null}

View File

@@ -27,7 +27,14 @@ export default function RootLayout({
child-src 'none';
object-src 'none';"
></meta>
<body className={inter.className}>{children}</body>
<body className={inter.className}>
{children}
<script
dangerouslySetInnerHTML={{
__html: `window.EXCALIDRAW_ASSET_PATH = 'https://assets.khoj.dev/@excalidraw/excalidraw/dist/';`,
}}
/>
</body>
</html>
);
}

View File

@@ -28,22 +28,42 @@ interface ChatBodyDataProps {
isLoggedIn: boolean;
conversationId?: string;
setQueryToProcess: (query: string) => void;
setImage64: (image64: string) => void;
setImages: (images: string[]) => void;
}
function ChatBodyData(props: ChatBodyDataProps) {
const [message, setMessage] = useState("");
const [image, setImage] = useState<string | null>(null);
const [images, setImages] = useState<string[]>([]);
const [processingMessage, setProcessingMessage] = useState(false);
const [agentMetadata, setAgentMetadata] = useState<AgentData | null>(null);
const setQueryToProcess = props.setQueryToProcess;
const streamedMessages = props.streamedMessages;
const chatHistoryCustomClassName = props.isMobileWidth ? "w-full" : "w-4/6";
useEffect(() => {
if (image) {
props.setImage64(encodeURIComponent(image));
if (images.length > 0) {
const encodedImages = images.map((image) => encodeURIComponent(image));
props.setImages(encodedImages);
}
}, [image, props.setImage64]);
}, [images, props.setImages]);
useEffect(() => {
const storedImages = localStorage.getItem("images");
if (storedImages) {
const parsedImages: string[] = JSON.parse(storedImages);
setImages(parsedImages);
const encodedImages = parsedImages.map((img: string) => encodeURIComponent(img));
props.setImages(encodedImages);
localStorage.removeItem("images");
}
const storedMessage = localStorage.getItem("message");
if (storedMessage) {
setProcessingMessage(true);
setQueryToProcess(storedMessage);
}
}, [setQueryToProcess, props.setImages]);
useEffect(() => {
if (message) {
@@ -78,15 +98,16 @@ function ChatBodyData(props: ChatBodyDataProps) {
setTitle={props.setTitle}
pendingMessage={processingMessage ? message : ""}
incomingMessages={props.streamedMessages}
customClassName={chatHistoryCustomClassName}
/>
</div>
<div
className={`${styles.inputBox} p-1 md:px-2 shadow-md bg-background align-middle items-center justify-center dark:bg-neutral-700 dark:border-0 dark:shadow-sm rounded-t-2xl rounded-b-none md:rounded-xl`}
className={`${styles.inputBox} p-1 md:px-2 shadow-md bg-background align-middle items-center justify-center dark:bg-neutral-700 dark:border-0 dark:shadow-sm rounded-t-2xl rounded-b-none md:rounded-xl h-fit ${chatHistoryCustomClassName} mr-auto ml-auto`}
>
<ChatInputArea
isLoggedIn={props.isLoggedIn}
sendMessage={(message) => setMessage(message)}
sendImage={(image) => setImage(image)}
sendImage={(image) => setImages((prevImages) => [...prevImages, image])}
sendDisabled={processingMessage}
chatOptionsData={props.chatOptionsData}
conversationId={props.conversationId}
@@ -109,7 +130,7 @@ export default function SharedChat() {
const [processQuerySignal, setProcessQuerySignal] = useState(false);
const [uploadedFiles, setUploadedFiles] = useState<string[]>([]);
const [paramSlug, setParamSlug] = useState<string | undefined>(undefined);
const [image64, setImage64] = useState<string>("");
const [images, setImages] = useState<string[]>([]);
const locationData = useIPLocationData() || {
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
@@ -167,7 +188,7 @@ export default function SharedChat() {
completed: false,
timestamp: new Date().toISOString(),
rawQuery: queryToProcess || "",
uploadedImageData: decodeURIComponent(image64),
images: images,
};
setMessages((prevMessages) => [...prevMessages, newStreamMessage]);
setProcessQuerySignal(true);
@@ -194,7 +215,7 @@ export default function SharedChat() {
if (done) {
setQueryToProcess("");
setProcessQuerySignal(false);
setImage64("");
setImages([]);
break;
}
@@ -236,7 +257,7 @@ export default function SharedChat() {
country_code: locationData.countryCode,
timezone: locationData.timezone,
}),
...(image64 && { image: image64 }),
...(images.length > 0 && { image: images }),
};
const response = await fetch(chatAPI, {
@@ -275,6 +296,19 @@ export default function SharedChat() {
<div className={styles.chatBox}>
<div className={styles.chatBoxBody}>
{!isMobileWidth && title && (
<div
className={`${styles.chatTitleWrapper} text-nowrap text-ellipsis overflow-hidden max-w-screen-md grid items-top font-bold mr-8 pt-6 col-auto h-fit`}
>
{title && (
<h2
className={`text-lg text-ellipsis whitespace-nowrap overflow-x-hidden`}
>
{title}
</h2>
)}
</div>
)}
<Suspense fallback={<Loading />}>
<ChatBodyData
conversationId={conversationId}
@@ -286,7 +320,7 @@ export default function SharedChat() {
setTitle={setTitle}
setUploadedFiles={setUploadedFiles}
isMobileWidth={isMobileWidth}
setImage64={setImage64}
setImages={setImages}
/>
</Suspense>
</div>

View File

@@ -75,7 +75,7 @@ div.titleBar {
div.chatBoxBody {
display: grid;
height: 100%;
width: 70%;
width: 95%;
margin: auto;
}

View File

@@ -1,6 +1,6 @@
{
"name": "khoj-ai",
"version": "1.26.0",
"version": "1.26.4",
"private": true,
"scripts": {
"dev": "next dev",
@@ -63,7 +63,8 @@
"swr": "^2.2.5",
"typescript": "^5",
"vaul": "^0.9.1",
"zod": "^3.23.8"
"zod": "^3.23.8",
"@excalidraw/excalidraw": "^0.17.6"
},
"devDependencies": {
"@types/dompurify": "^3.0.5",

View File

@@ -286,6 +286,11 @@
resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.57.1.tgz#de633db3ec2ef6a3c89e2f19038063e8a122e2c2"
integrity sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==
"@excalidraw/excalidraw@^0.17.6":
version "0.17.6"
resolved "https://registry.yarnpkg.com/@excalidraw/excalidraw/-/excalidraw-0.17.6.tgz#5fd208ce69d33ca712d1804b50d7d06d5c46ac4d"
integrity sha512-fyCl+zG/Z5yhHDh5Fq2ZGmphcrALmuOdtITm8gN4d8w4ntnaopTXcTfnAAaU3VleDC6LhTkoLOTG6P5kgREiIg==
"@floating-ui/core@^1.6.0":
version "1.6.8"
resolved "https://registry.yarnpkg.com/@floating-ui/core/-/core-1.6.8.tgz#aa43561be075815879305965020f492cdb43da12"