klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-09 05:39:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Allow sharing multiple images as part of user query from the web app

Browse Source 
Previously the web app only expected a single image to be shared by
the user as part of their query.

This change allows sharing multiple images from the web app.

Closes #921
This commit is contained in:
Debanjum Singh Solanky
2024-10-17 23:08:20 -07:00
parent e2abc1a257
commit 0d6a54c10f
6 changed files with 122 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
src/interface/web/app/chat/page.tsx
View File

@@ -17,8 +17,6 @@ import { useIPLocationData, useIsMobileWidth, welcomeConsole } from "../common/u
import ChatInputArea, { ChatOptions } from "../components/chatInputArea/chatInputArea"; import ChatInputArea, { ChatOptions } from "../components/chatInputArea/chatInputArea";
import { useAuthenticatedData } from "../common/auth"; import { useAuthenticatedData } from "../common/auth";
import { AgentData } from "../agents/page"; import { AgentData } from "../agents/page";
import { DotsThreeVertical } from "@phosphor-icons/react";
import { Button } from "@/components/ui/button";
interface ChatBodyDataProps { interface ChatBodyDataProps {
chatOptionsData: ChatOptions | null; chatOptionsData: ChatOptions | null;
@@ -29,14 +27,14 @@ interface ChatBodyDataProps {
setUploadedFiles: (files: string[]) => void; setUploadedFiles: (files: string[]) => void;
isMobileWidth?: boolean; isMobileWidth?: boolean;
isLoggedIn: boolean; isLoggedIn: boolean;
setImage64: (image64: string) => void; setImages: (images: string[]) => void;
} }
function ChatBodyData(props: ChatBodyDataProps) { function ChatBodyData(props: ChatBodyDataProps) {
const searchParams = useSearchParams(); const searchParams = useSearchParams();
const conversationId = searchParams.get("conversationId"); const conversationId = searchParams.get("conversationId");
const [message, setMessage] = useState(""); const [message, setMessage] = useState("");
const [image, setImage] = useState<string | null>(null); const [images, setImages] = useState<string[]>([]);
const [processingMessage, setProcessingMessage] = useState(false); const [processingMessage, setProcessingMessage] = useState(false);
const [agentMetadata, setAgentMetadata] = useState<AgentData | null>(null); const [agentMetadata, setAgentMetadata] = useState<AgentData | null>(null);
@@ -44,17 +42,20 @@ function ChatBodyData(props: ChatBodyDataProps) {
const onConversationIdChange = props.onConversationIdChange; const onConversationIdChange = props.onConversationIdChange;
useEffect(() => { useEffect(() => {
if (image) { if (images.length > 0) {
props.setImage64(encodeURIComponent(image)); const encodedImages = images.map((image) => encodeURIComponent(image));
props.setImages(encodedImages);
} }
}, [image, props.setImage64]); }, [images, props.setImages]);
useEffect(() => { useEffect(() => {
const storedImage = localStorage.getItem("image"); const storedImages = localStorage.getItem("images");
if (storedImage) { if (storedImages) {
setImage(storedImage); const parsedImages: string[] = JSON.parse(storedImages);
props.setImage64(encodeURIComponent(storedImage)); setImages(parsedImages);
localStorage.removeItem("image"); const encodedImages = parsedImages.map((img: string) => encodeURIComponent(img));
props.setImages(encodedImages);
localStorage.removeItem("images");
} }
const storedMessage = localStorage.getItem("message"); const storedMessage = localStorage.getItem("message");
@@ -62,7 +63,7 @@ function ChatBodyData(props: ChatBodyDataProps) {
setProcessingMessage(true); setProcessingMessage(true);
setQueryToProcess(storedMessage); setQueryToProcess(storedMessage);
} }
}, [setQueryToProcess]); }, [setQueryToProcess, props.setImages]);
useEffect(() => { useEffect(() => {
if (message) { if (message) {
@@ -112,7 +113,7 @@ function ChatBodyData(props: ChatBodyDataProps) {
agentColor={agentMetadata?.color} agentColor={agentMetadata?.color}
isLoggedIn={props.isLoggedIn} isLoggedIn={props.isLoggedIn}
sendMessage={(message) => setMessage(message)} sendMessage={(message) => setMessage(message)}
sendImage={(image) => setImage(image)} sendImage={(image) => setImages((prevImages) => [...prevImages, image])}
sendDisabled={processingMessage} sendDisabled={processingMessage}
chatOptionsData={props.chatOptionsData} chatOptionsData={props.chatOptionsData}
conversationId={conversationId} conversationId={conversationId}
@@ -134,7 +135,7 @@ export default function Chat() {
const [queryToProcess, setQueryToProcess] = useState<string>(""); const [queryToProcess, setQueryToProcess] = useState<string>("");
const [processQuerySignal, setProcessQuerySignal] = useState(false); const [processQuerySignal, setProcessQuerySignal] = useState(false);
const [uploadedFiles, setUploadedFiles] = useState<string[]>([]); const [uploadedFiles, setUploadedFiles] = useState<string[]>([]);
const [image64, setImage64] = useState<string>(""); const [images, setImages] = useState<string[]>([]);
const locationData = useIPLocationData() || { const locationData = useIPLocationData() || {
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone, timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
@@ -170,7 +171,7 @@ export default function Chat() {
completed: false, completed: false,
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
rawQuery: queryToProcess || "", rawQuery: queryToProcess || "",
uploadedImageData: decodeURIComponent(image64), images: images,
}; };
setMessages((prevMessages) => [...prevMessages, newStreamMessage]); setMessages((prevMessages) => [...prevMessages, newStreamMessage]);
setProcessQuerySignal(true); setProcessQuerySignal(true);
@@ -201,7 +202,7 @@ export default function Chat() {
if (done) { if (done) {
setQueryToProcess(""); setQueryToProcess("");
setProcessQuerySignal(false); setProcessQuerySignal(false);
setImage64(""); setImages([]);
break; break;
} }
@@ -249,7 +250,7 @@ export default function Chat() {
country_code: locationData.countryCode, country_code: locationData.countryCode,
timezone: locationData.timezone, timezone: locationData.timezone,
}), }),
...(image64 && { image: image64 }), ...(images.length > 0 && { images: images }),
}; };
const response = await fetch(chatAPI, { const response = await fetch(chatAPI, {
@@ -331,7 +332,7 @@ export default function Chat() {
setUploadedFiles={setUploadedFiles} setUploadedFiles={setUploadedFiles}
isMobileWidth={isMobileWidth} isMobileWidth={isMobileWidth}
onConversationIdChange={handleConversationIdChange} onConversationIdChange={handleConversationIdChange}
setImage64={setImage64} setImages={setImages}
/> />
</Suspense> </Suspense>
</div> </div>

3
src/interface/web/app/components/chatHistory/chatHistory.tsx
View File

@@ -298,7 +298,7 @@ export default function ChatHistory(props: ChatHistoryProps) {
created: message.timestamp, created: message.timestamp,
by: "you", by: "you",
automationId: "", automationId: "",
uploadedImageData: message.uploadedImageData, images: message.images,
}} }}
customClassName="fullHistory" customClassName="fullHistory"
borderLeftColor={`${data?.agent?.color}-500`} borderLeftColor={`${data?.agent?.color}-500`}
@@ -341,7 +341,6 @@ export default function ChatHistory(props: ChatHistoryProps) {
created: new Date().getTime().toString(), created: new Date().getTime().toString(),
by: "you", by: "you",
automationId: "", automationId: "",
uploadedImageData: props.pendingMessage,
}} }}
customClassName="fullHistory" customClassName="fullHistory"
borderLeftColor={`${data?.agent?.color}-500`} borderLeftColor={`${data?.agent?.color}-500`}

87
src/interface/web/app/components/chatInputArea/chatInputArea.tsx
View File

@@ -78,10 +78,11 @@ export default function ChatInputArea(props: ChatInputProps) {
const [loginRedirectMessage, setLoginRedirectMessage] = useState<string | null>(null); const [loginRedirectMessage, setLoginRedirectMessage] = useState<string | null>(null);
const [showLoginPrompt, setShowLoginPrompt] = useState(false); const [showLoginPrompt, setShowLoginPrompt] = useState(false);
const [recording, setRecording] = useState(false);
const [imageUploaded, setImageUploaded] = useState(false); const [imageUploaded, setImageUploaded] = useState(false);
const [imagePath, setImagePath] = useState<string>(""); const [imagePaths, setImagePaths] = useState<string[]>([]);
const [imageData, setImageData] = useState<string | null>(null); const [imageData, setImageData] = useState<string[]>([]);
const [recording, setRecording] = useState(false);
const [mediaRecorder, setMediaRecorder] = useState<MediaRecorder | null>(null); const [mediaRecorder, setMediaRecorder] = useState<MediaRecorder | null>(null);
const [progressValue, setProgressValue] = useState(0); const [progressValue, setProgressValue] = useState(0);
@@ -106,27 +107,31 @@ export default function ChatInputArea(props: ChatInputProps) {
useEffect(() => { useEffect(() => {
async function fetchImageData() { async function fetchImageData() {
if (imagePath) { if (imagePaths.length > 0) {
const response = await fetch(imagePath); const newImageData = await Promise.all(
const blob = await response.blob(); imagePaths.map(async (path) => {
const reader = new FileReader(); const response = await fetch(path);
reader.onload = function () { const blob = await response.blob();
const base64data = reader.result; return new Promise<string>((resolve) => {
setImageData(base64data as string); const reader = new FileReader();
}; reader.onload = () => resolve(reader.result as string);
reader.readAsDataURL(blob); reader.readAsDataURL(blob);
});
}),
);
setImageData(newImageData);
} }
setUploading(false); setUploading(false);
} }
setUploading(true); setUploading(true);
fetchImageData(); fetchImageData();
}, [imagePath]); }, [imagePaths]);
function onSendMessage() { function onSendMessage() {
if (imageUploaded) { if (imageUploaded) {
setImageUploaded(false); setImageUploaded(false);
setImagePath(""); setImagePaths([]);
props.sendImage(imageData || ""); imageData.forEach((data) => props.sendImage(data));
} }
if (!message.trim()) return; if (!message.trim()) return;
@@ -172,18 +177,23 @@ export default function ChatInputArea(props: ChatInputProps) {
setShowLoginPrompt(true); setShowLoginPrompt(true);
return; return;
} }
// check for image file // check for image files
const image_endings = ["jpg", "jpeg", "png", "webp"]; const image_endings = ["jpg", "jpeg", "png", "webp"];
const newImagePaths: string[] = [];
for (let i = 0; i < files.length; i++) { for (let i = 0; i < files.length; i++) {
const file = files[i]; const file = files[i];
const file_extension = file.name.split(".").pop(); const file_extension = file.name.split(".").pop();
if (image_endings.includes(file_extension || "")) { if (image_endings.includes(file_extension || "")) {
setImageUploaded(true); newImagePaths.push(DOMPurify.sanitize(URL.createObjectURL(file)));
setImagePath(DOMPurify.sanitize(URL.createObjectURL(file)));
return;
} }
} }
if (newImagePaths.length > 0) {
setImageUploaded(true);
setImagePaths((prevPaths) => [...prevPaths, ...newImagePaths]);
return;
}
uploadDataForIndexing( uploadDataForIndexing(
files, files,
setWarning, setWarning,
@@ -288,9 +298,12 @@ export default function ChatInputArea(props: ChatInputProps) {
setIsDragAndDropping(false); setIsDragAndDropping(false);
} }
function removeImageUpload() { function removeImageUpload(index: number) {
setImageUploaded(false); setImagePaths((prevPaths) => prevPaths.filter((_, i) => i !== index));
setImagePath(""); setImageData((prevData) => prevData.filter((_, i) => i !== index));
if (imagePaths.length === 1) {
setImageUploaded(false);
}
} }
return ( return (
@@ -413,16 +426,24 @@ export default function ChatInputArea(props: ChatInputProps) {
onDrop={handleDragAndDropFiles} onDrop={handleDragAndDropFiles}
> >
{imageUploaded && ( {imageUploaded && (
<div className="absolute bottom-[80px] left-0 right-0 dark:bg-neutral-700 bg-white pt-5 pb-5 w-full rounded-lg border dark:border-none grid grid-cols-2"> <div className="absolute bottom-full left-0 right-0 px-12 py-2 dark:bg-neutral-700 bg-white w-full rounded-t-lg border dark:border-none flex items-center space-x-2 overflow-x-auto">
<div className="pl-4 pr-4"> {imagePaths.map((path, index) => (
<img src={imagePath} alt="img" className="w-auto max-h-[100px]" /> <div key={index} className="relative flex-shrink-0 group">
</div> <img
<div className="pl-4 pr-4"> src={path}
<X alt={`img-${index}`}
className="w-6 h-6 float-right dark:hover:bg-[hsl(var(--background))] hover:bg-neutral-100 rounded-sm" className="w-auto h-16 object-cover rounded-xl"
onClick={removeImageUpload} />
/> <Button
</div> variant="ghost"
size="icon"
className="absolute -top-2 -right-2 h-5 w-5 rounded-full bg-neutral-200 dark:bg-neutral-600 hover:bg-neutral-300 dark:hover:bg-neutral-500 opacity-0 group-hover:opacity-100 transition-opacity"
onClick={() => removeImageUpload(index)}
>
<X className="h-3 w-3" />
</Button>
</div>
))}
</div> </div>
)} )}
<input <input
@@ -451,7 +472,7 @@ export default function ChatInputArea(props: ChatInputProps) {
onKeyDown={(e) => { onKeyDown={(e) => {
if (e.key === "Enter" && !e.shiftKey) { if (e.key === "Enter" && !e.shiftKey) {
setImageUploaded(false); setImageUploaded(false);
setImagePath(""); setImagePaths([]);
e.preventDefault(); e.preventDefault();
onSendMessage(); onSendMessage();
} }

17
src/interface/web/app/components/chatMessage/chatMessage.tsx
View File

@@ -114,7 +114,7 @@ export interface SingleChatMessage {
rawQuery?: string; rawQuery?: string;
intent?: Intent; intent?: Intent;
agent?: AgentData; agent?: AgentData;
uploadedImageData?: string; images?: string[];
} }
export interface StreamMessage { export interface StreamMessage {
@@ -126,7 +126,7 @@ export interface StreamMessage {
rawQuery: string; rawQuery: string;
timestamp: string; timestamp: string;
agent?: AgentData; agent?: AgentData;
uploadedImageData?: string; images?: string[];
} }
export interface ChatHistoryData { export interface ChatHistoryData {
@@ -208,7 +208,6 @@ interface ChatMessageProps {
borderLeftColor?: string; borderLeftColor?: string;
isLastMessage?: boolean; isLastMessage?: boolean;
agent?: AgentData; agent?: AgentData;
uploadedImageData?: string;
} }
interface TrainOfThoughtProps { interface TrainOfThoughtProps {
@@ -328,8 +327,14 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
.replace(/\\\[/g, "LEFTBRACKET") .replace(/\\\[/g, "LEFTBRACKET")
.replace(/\\\]/g, "RIGHTBRACKET"); .replace(/\\\]/g, "RIGHTBRACKET");
if (props.chatMessage.uploadedImageData) { if (props.chatMessage.images && props.chatMessage.images.length > 0) {
message = `![uploaded image](${props.chatMessage.uploadedImageData})\n\n${message}`; const imagesInMd = props.chatMessage.images
.map(
(image) =>
`![uploaded image](${image.startsWith("data%3Aimage") ? decodeURIComponent(image) : image})`,
)
.join("\n\n");
message = `${imagesInMd}\n\n${message}`;
} }
if (props.chatMessage.intent && props.chatMessage.intent.type == "text-to-image") { if (props.chatMessage.intent && props.chatMessage.intent.type == "text-to-image") {
@@ -364,7 +369,7 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
// Sanitize and set the rendered markdown // Sanitize and set the rendered markdown
setMarkdownRendered(DOMPurify.sanitize(markdownRendered)); setMarkdownRendered(DOMPurify.sanitize(markdownRendered));
}, [props.chatMessage.message, props.chatMessage.intent]); }, [props.chatMessage.message, props.chatMessage.images, props.chatMessage.intent]);
useEffect(() => { useEffect(() => {
if (copySuccess) { if (copySuccess) {

13
src/interface/web/app/page.tsx
View File

@@ -44,7 +44,7 @@ function FisherYatesShuffle(array: any[]) {
function ChatBodyData(props: ChatBodyDataProps) { function ChatBodyData(props: ChatBodyDataProps) {
const [message, setMessage] = useState(""); const [message, setMessage] = useState("");
const [image, setImage] = useState<string | null>(null); const [images, setImages] = useState<string[]>([]);
const [processingMessage, setProcessingMessage] = useState(false); const [processingMessage, setProcessingMessage] = useState(false);
const [greeting, setGreeting] = useState(""); const [greeting, setGreeting] = useState("");
const [shuffledOptions, setShuffledOptions] = useState<Suggestion[]>([]); const [shuffledOptions, setShuffledOptions] = useState<Suggestion[]>([]);
@@ -140,18 +140,19 @@ function ChatBodyData(props: ChatBodyDataProps) {
onConversationIdChange?.(newConversationId); onConversationIdChange?.(newConversationId);
window.location.href = `/chat?conversationId=${newConversationId}`; window.location.href = `/chat?conversationId=${newConversationId}`;
localStorage.setItem("message", message); localStorage.setItem("message", message);
if (image) { if (images.length > 0) {
localStorage.setItem("image", image); localStorage.setItem("images", JSON.stringify(images));
} }
} catch (error) { } catch (error) {
console.error("Error creating new conversation:", error); console.error("Error creating new conversation:", error);
setProcessingMessage(false); setProcessingMessage(false);
} }
setMessage(""); setMessage("");
setImages([]);
} }
}; };
processMessage(); processMessage();
if (message) { if (message || images.length > 0) {
setProcessingMessage(true); setProcessingMessage(true);
} }
}, [selectedAgent, message, processingMessage, onConversationIdChange]); }, [selectedAgent, message, processingMessage, onConversationIdChange]);
@@ -232,7 +233,7 @@ function ChatBodyData(props: ChatBodyDataProps) {
<ChatInputArea <ChatInputArea
isLoggedIn={props.isLoggedIn} isLoggedIn={props.isLoggedIn}
sendMessage={(message) => setMessage(message)} sendMessage={(message) => setMessage(message)}
sendImage={(image) => setImage(image)} sendImage={(image) => setImages((prevImages) => [...prevImages, image])}
sendDisabled={processingMessage} sendDisabled={processingMessage}
chatOptionsData={props.chatOptionsData} chatOptionsData={props.chatOptionsData}
conversationId={null} conversationId={null}
@@ -313,7 +314,7 @@ function ChatBodyData(props: ChatBodyDataProps) {
<ChatInputArea <ChatInputArea
isLoggedIn={props.isLoggedIn} isLoggedIn={props.isLoggedIn}
sendMessage={(message) => setMessage(message)} sendMessage={(message) => setMessage(message)}
sendImage={(image) => setImage(image)} sendImage={(image) => setImages((prevImages) => [...prevImages, image])}
sendDisabled={processingMessage} sendDisabled={processingMessage}
chatOptionsData={props.chatOptionsData} chatOptionsData={props.chatOptionsData}
conversationId={null} conversationId={null}

40
src/interface/web/app/share/chat/page.tsx
View File

@@ -28,22 +28,40 @@ interface ChatBodyDataProps {
isLoggedIn: boolean; isLoggedIn: boolean;
conversationId?: string; conversationId?: string;
setQueryToProcess: (query: string) => void; setQueryToProcess: (query: string) => void;
setImage64: (image64: string) => void; setImages: (images: string[]) => void;
} }
function ChatBodyData(props: ChatBodyDataProps) { function ChatBodyData(props: ChatBodyDataProps) {
const [message, setMessage] = useState(""); const [message, setMessage] = useState("");
const [image, setImage] = useState<string | null>(null); const [images, setImages] = useState<string[]>([]);
const [processingMessage, setProcessingMessage] = useState(false); const [processingMessage, setProcessingMessage] = useState(false);
const [agentMetadata, setAgentMetadata] = useState<AgentData | null>(null); const [agentMetadata, setAgentMetadata] = useState<AgentData | null>(null);
const setQueryToProcess = props.setQueryToProcess; const setQueryToProcess = props.setQueryToProcess;
const streamedMessages = props.streamedMessages; const streamedMessages = props.streamedMessages;
useEffect(() => { useEffect(() => {
if (image) { if (images.length > 0) {
props.setImage64(encodeURIComponent(image)); const encodedImages = images.map((image) => encodeURIComponent(image));
props.setImages(encodedImages);
} }
}, [image, props.setImage64]); }, [images, props.setImages]);
useEffect(() => {
const storedImages = localStorage.getItem("images");
if (storedImages) {
const parsedImages: string[] = JSON.parse(storedImages);
setImages(parsedImages);
const encodedImages = parsedImages.map((img: string) => encodeURIComponent(img));
props.setImages(encodedImages);
localStorage.removeItem("images");
}
const storedMessage = localStorage.getItem("message");
if (storedMessage) {
setProcessingMessage(true);
setQueryToProcess(storedMessage);
}
}, [setQueryToProcess, props.setImages]);
useEffect(() => { useEffect(() => {
if (message) { if (message) {
@@ -86,7 +104,7 @@ function ChatBodyData(props: ChatBodyDataProps) {
<ChatInputArea <ChatInputArea
isLoggedIn={props.isLoggedIn} isLoggedIn={props.isLoggedIn}
sendMessage={(message) => setMessage(message)} sendMessage={(message) => setMessage(message)}
sendImage={(image) => setImage(image)} sendImage={(image) => setImages((prevImages) => [...prevImages, image])}
sendDisabled={processingMessage} sendDisabled={processingMessage}
chatOptionsData={props.chatOptionsData} chatOptionsData={props.chatOptionsData}
conversationId={props.conversationId} conversationId={props.conversationId}
@@ -109,7 +127,7 @@ export default function SharedChat() {
const [processQuerySignal, setProcessQuerySignal] = useState(false); const [processQuerySignal, setProcessQuerySignal] = useState(false);
const [uploadedFiles, setUploadedFiles] = useState<string[]>([]); const [uploadedFiles, setUploadedFiles] = useState<string[]>([]);
const [paramSlug, setParamSlug] = useState<string | undefined>(undefined); const [paramSlug, setParamSlug] = useState<string | undefined>(undefined);
const [image64, setImage64] = useState<string>(""); const [images, setImages] = useState<string[]>([]);
const locationData = useIPLocationData() || { const locationData = useIPLocationData() || {
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone, timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
@@ -167,7 +185,7 @@ export default function SharedChat() {
completed: false, completed: false,
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
rawQuery: queryToProcess || "", rawQuery: queryToProcess || "",
uploadedImageData: decodeURIComponent(image64), images: images,
}; };
setMessages((prevMessages) => [...prevMessages, newStreamMessage]); setMessages((prevMessages) => [...prevMessages, newStreamMessage]);
setProcessQuerySignal(true); setProcessQuerySignal(true);
@@ -194,7 +212,7 @@ export default function SharedChat() {
if (done) { if (done) {
setQueryToProcess(""); setQueryToProcess("");
setProcessQuerySignal(false); setProcessQuerySignal(false);
setImage64(""); setImages([]);
break; break;
} }
@@ -236,7 +254,7 @@ export default function SharedChat() {
country_code: locationData.countryCode, country_code: locationData.countryCode,
timezone: locationData.timezone, timezone: locationData.timezone,
}), }),
...(image64 && { image: image64 }), ...(images.length > 0 && { image: images }),
}; };
const response = await fetch(chatAPI, { const response = await fetch(chatAPI, {
@@ -286,7 +304,7 @@ export default function SharedChat() {
setTitle={setTitle} setTitle={setTitle}
setUploadedFiles={setUploadedFiles} setUploadedFiles={setUploadedFiles}
isMobileWidth={isMobileWidth} isMobileWidth={isMobileWidth}
setImage64={setImage64} setImages={setImages}
/> />
</Suspense> </Suspense>
</div> </div>