Merge branch 'master' of github.com:khoj-ai/khoj into features/add-chat-controls

src/interface/obsidian/src/chat_view.ts

@@ -31,6 +31,7 @@ interface ChatMessageState {
     rawResponse: string;
     rawQuery: string;
     isVoice: boolean;
+    turnId: string;
 }
 
 interface Location {
@@ -41,6 +42,17 @@ interface Location {
     timezone: string;
 }
 
+interface RenderMessageOptions {
+    chatBodyEl: Element;
+    message: string;
+    sender: string;
+    turnId?: string;
+    dt?: Date;
+    raw?: boolean;
+    willReplace?: boolean;
+    isSystemMessage?: boolean;
+}
+
 export class KhojChatView extends KhojPaneView {
     result: string;
     setting: KhojSetting;
@@ -480,6 +492,7 @@ export class KhojChatView extends KhojPaneView {
         chatEl: Element,
         message: string,
         sender: string,
+        turnId: string,
         context?: string[],
         onlineContext?: object,
         dt?: Date,
@@ -500,9 +513,21 @@ export class KhojChatView extends KhojPaneView {
             mermaidjsDiagram ||
             excalidrawDiagram) {
             let imageMarkdown = this.generateImageMarkdown(message, intentType ?? "", inferredQueries, conversationId, images, excalidrawDiagram, mermaidjsDiagram);
-            chatMessageEl = this.renderMessage(chatEl, imageMarkdown, sender, dt);
+            chatMessageEl = this.renderMessage({
+                chatBodyEl: chatEl,
+                message: imageMarkdown,
+                sender,
+                dt,
+                turnId
+            });
         } else {
-            chatMessageEl = this.renderMessage(chatEl, message, sender, dt);
+            chatMessageEl = this.renderMessage({
+                chatBodyEl: chatEl,
+                message,
+                sender,
+                dt,
+                turnId
+            });
         }
 
         // If no document or online context is provided, skip rendering the reference section
@@ -547,7 +572,7 @@ export class KhojChatView extends KhojPaneView {
         return imageMarkdown;
     }
 
-    renderMessage(chatBodyEl: Element, message: string, sender: string, dt?: Date, raw: boolean = false, willReplace: boolean = true): Element {
+    renderMessage({ chatBodyEl, message, sender, dt, turnId, raw = false, willReplace = true, isSystemMessage = false }: RenderMessageOptions): Element {
         let message_time = this.formatDate(dt ?? new Date());
 
         // Append message to conversation history HTML element.
@@ -555,7 +580,8 @@ export class KhojChatView extends KhojPaneView {
         let chatMessageEl = chatBodyEl.createDiv({
             attr: {
                 "data-meta": message_time,
-                class: `khoj-chat-message ${sender}`
+                class: `khoj-chat-message ${sender}`,
+                ...(turnId && { "data-turnId": turnId })
             },
         })
         let chatMessageBodyEl = chatMessageEl.createDiv();
@@ -574,7 +600,7 @@ export class KhojChatView extends KhojPaneView {
 
         // Add action buttons to each chat message element
         if (willReplace === true) {
-            this.renderActionButtons(message, chatMessageBodyTextEl);
+            this.renderActionButtons(message, chatMessageBodyTextEl, isSystemMessage);
         }
 
         // Remove user-select: none property to make text selectable
@@ -618,7 +644,7 @@ export class KhojChatView extends KhojPaneView {
         this.scrollChatToBottom();
     }
 
-    renderActionButtons(message: string, chatMessageBodyTextEl: HTMLElement) {
+    renderActionButtons(message: string, chatMessageBodyTextEl: HTMLElement, isSystemMessage: boolean = false) {
         let copyButton = this.contentEl.createEl('button');
         copyButton.classList.add("chat-action-button");
         copyButton.title = "Copy Message to Clipboard";
@@ -632,6 +658,25 @@ export class KhojChatView extends KhojPaneView {
         setIcon(pasteToFile, "clipboard-paste");
         pasteToFile.addEventListener('click', (event) => { pasteTextAtCursor(createCopyParentText(message, 'clipboard-paste')(event)); });
 
+
+        // Add delete button
+        let deleteButton = null;
+        if (!isSystemMessage) {
+            deleteButton = this.contentEl.createEl('button');
+            deleteButton.classList.add("chat-action-button");
+            deleteButton.title = "Delete Message";
+            setIcon(deleteButton, "trash-2");
+            deleteButton.addEventListener('click', () => {
+                const messageEl = chatMessageBodyTextEl.closest('.khoj-chat-message');
+                if (messageEl) {
+                    // Ask for confirmation before deleting
+                    if (confirm('Are you sure you want to delete this message?')) {
+                        this.deleteMessage(messageEl as HTMLElement);
+                    }
+                }
+            });
+        }
+
         // Only enable the speech feature if the user is subscribed
         let speechButton = null;
 
@@ -646,7 +691,9 @@ export class KhojChatView extends KhojPaneView {
 
         // Append buttons to parent element
         chatMessageBodyTextEl.append(copyButton, pasteToFile);
-
+        if (deleteButton) {
+            chatMessageBodyTextEl.append(deleteButton);
+        }
         if (speechButton) {
             chatMessageBodyTextEl.append(speechButton);
         }
@@ -672,7 +719,7 @@ export class KhojChatView extends KhojPaneView {
         if (chatInput) {
             chatInput.placeholder = this.startingMessage;
         }
-        this.renderMessage(chatBodyEl, "Hey 👋🏾, what's up?", "khoj");
+        this.renderMessage({chatBodyEl, message: "Hey 👋🏾, what's up?", sender: "khoj", isSystemMessage: true});
     }
 
     async toggleChatSessions(forceShow: boolean = false): Promise<boolean> {
@@ -883,7 +930,12 @@ export class KhojChatView extends KhojPaneView {
             if (responseJson.detail) {
                 // If the server returns error details in response, render a setup hint.
                 let setupMsg = "Hi 👋🏾, to start chatting add available chat models options via [the Django Admin panel](/server/admin) on the Server";
-                this.renderMessage(chatBodyEl, setupMsg, "khoj", undefined);
+                this.renderMessage({
+                    chatBodyEl,
+                    message: setupMsg,
+                    sender: "khoj",
+                    isSystemMessage: true
+                });
 
                 return false;
             } else if (responseJson.response) {
@@ -897,6 +949,7 @@ export class KhojChatView extends KhojPaneView {
                         chatBodyEl,
                         chatLog.message,
                         chatLog.by,
+                        chatLog.turnId,
                         chatLog.context,
                         chatLog.onlineContext,
                         new Date(chatLog.created),
@@ -927,7 +980,12 @@ export class KhojChatView extends KhojPaneView {
             }
         } catch (err) {
             let errorMsg = "Unable to get response from Khoj server ❤️‍🩹. Ensure server is running or contact developers for help at [team@khoj.dev](mailto:team@khoj.dev) or in [Discord](https://discord.gg/BDgyabRM6e)";
-            this.renderMessage(chatBodyEl, errorMsg, "khoj", undefined);
+            this.renderMessage({
+                chatBodyEl,
+                message: errorMsg,
+                sender: "khoj",
+                isSystemMessage: true
+            });
             return false;
         }
         return true;
@@ -972,7 +1030,7 @@ export class KhojChatView extends KhojPaneView {
                 this.textToSpeech(this.chatMessageState.rawResponse);
 
             // Append any references after all the data has been streamed
-            this.finalizeChatBodyResponse(this.chatMessageState.references, this.chatMessageState.newResponseTextEl);
+            this.finalizeChatBodyResponse(this.chatMessageState.references, this.chatMessageState.newResponseTextEl, this.chatMessageState.turnId);
 
             const liveQuery = this.chatMessageState.rawQuery;
             // Reset variables
@@ -985,6 +1043,7 @@ export class KhojChatView extends KhojPaneView {
                 rawQuery: liveQuery,
                 isVoice: false,
                 generatedAssets: "",
+                turnId: "",
             };
         } else if (chunk.type === "references") {
             this.chatMessageState.references = { "notes": chunk.data.context, "online": chunk.data.onlineContext };
@@ -1006,6 +1065,12 @@ export class KhojChatView extends KhojPaneView {
                 this.chatMessageState.rawResponse += chunkData;
                 this.handleStreamResponse(this.chatMessageState.newResponseTextEl, this.chatMessageState.rawResponse + this.chatMessageState.generatedAssets, this.chatMessageState.loadingEllipsis);
             }
+        } else if (chunk.type === "metadata") {
+            const { turnId } = chunk.data;
+            if (turnId) {
+                // Append turnId to chatMessageState
+                this.chatMessageState.turnId = turnId;
+            }
         }
     }
 
@@ -1065,7 +1130,7 @@ export class KhojChatView extends KhojPaneView {
 
         // Render user query as chat message
         let chatBodyEl = this.contentEl.getElementsByClassName("khoj-chat-body")[0] as HTMLElement;
-        this.renderMessage(chatBodyEl, query, "you");
+        this.renderMessage({chatBodyEl, message: query, sender: "you"});
 
         let conversationId = chatBodyEl.dataset.conversationId;
         if (!conversationId) {
@@ -1111,6 +1176,7 @@ export class KhojChatView extends KhojPaneView {
             rawResponse: "",
             isVoice: isVoice,
             generatedAssets: "",
+            turnId: "",
         };
 
         let response = await fetch(chatUrl, {
@@ -1410,10 +1476,15 @@ export class KhojChatView extends KhojPaneView {
         return rawResponse;
     }
 
-    finalizeChatBodyResponse(references: object, newResponseElement: HTMLElement | null) {
+    finalizeChatBodyResponse(references: object, newResponseElement: HTMLElement | null, turnId: string) {
         if (!!newResponseElement && references != null && Object.keys(references).length > 0) {
             newResponseElement.appendChild(this.createReferenceSection(references));
         }
+        if (!!newResponseElement && turnId) {
+            // Set the turnId for the new response and the previous user message
+            newResponseElement.parentElement?.setAttribute("data-turnId", turnId);
+            newResponseElement.parentElement?.previousElementSibling?.setAttribute("data-turnId", turnId);
+        }
         this.scrollChatToBottom();
         let chatInput = this.contentEl.getElementsByClassName("khoj-chat-input")[0];
         if (chatInput) chatInput.removeAttribute("disabled");
@@ -1482,4 +1553,49 @@ export class KhojChatView extends KhojPaneView {
             }
         }
     }
+
+    // Add this new method to handle message deletion
+    async deleteMessage(messageEl: HTMLElement) {
+        const chatBodyEl = this.contentEl.getElementsByClassName("khoj-chat-body")[0] as HTMLElement;
+        const conversationId = chatBodyEl.dataset.conversationId;
+
+        // Get the turnId from the message's data-turn attribute
+        const turnId = messageEl.getAttribute("data-turnId");
+        if (!turnId || !conversationId) return;
+
+        try {
+            const response = await fetch(`${this.setting.khojUrl}/api/chat/conversation/message`, {
+                method: "DELETE",
+                headers: {
+                    "Content-Type": "application/json",
+                    "Authorization": `Bearer ${this.setting.khojApiKey}`
+                },
+                body: JSON.stringify({
+                    conversation_id: conversationId,
+                    turn_id: turnId
+                })
+            });
+
+            if (response.ok) {
+                // Remove both the user message and Khoj response (the conversation turn)
+                const isKhojMessage = messageEl.classList.contains("khoj");
+                const messages = Array.from(chatBodyEl.getElementsByClassName("khoj-chat-message"));
+                const messageIndex = messages.indexOf(messageEl);
+
+                if (isKhojMessage && messageIndex > 0) {
+                    // If it is a Khoj message, remove the previous user message too
+                    messages[messageIndex - 1].remove();
+                } else if (!isKhojMessage && messageIndex < messages.length - 1) {
+                    // If it is a user message, remove the next Khoj message too
+                    messages[messageIndex + 1].remove();
+                }
+                messageEl.remove();
+            } else {
+                this.flashStatusInChatInput("Failed to delete message");
+            }
+        } catch (error) {
+            console.error("Error deleting message:", error);
+            this.flashStatusInChatInput("Error deleting message");
+        }
+    }
 }

src/interface/web/app/common/utils.ts

@@ -114,3 +114,33 @@ export function useDebounce<T>(value: T, delay: number): T {
 
     return debouncedValue;
 }
+
+export const formatDateTime = (isoString: string): string => {
+    try {
+        const date = new Date(isoString);
+        const now = new Date();
+        const diffInMinutes = Math.floor((now.getTime() - date.getTime()) / 60000);
+
+        // Show relative time for recent dates
+        if (diffInMinutes < 1) return "just now";
+        if (diffInMinutes < 60) return `${diffInMinutes} minutes ago`;
+        if (diffInMinutes < 120) return "1 hour ago";
+        if (diffInMinutes < 1440) return `${Math.floor(diffInMinutes / 60)} hours ago`;
+
+        // For older dates, show full formatted date
+        const formatter = new Intl.DateTimeFormat("en-US", {
+            month: "long",
+            day: "numeric",
+            year: "numeric",
+            hour: "numeric",
+            minute: "2-digit",
+            hour12: true,
+            timeZoneName: "short",
+        });
+
+        return formatter.format(date);
+    } catch (error) {
+        console.error("Error formatting date:", error);
+        return isoString;
+    }
+};

src/interface/web/app/search/layout.tsx

@@ -2,6 +2,7 @@ import type { Metadata } from "next";
 
 import "../globals.css";
 import { ContentSecurityPolicy } from "../common/layoutHelper";
+import { Toaster } from "@/components/ui/toaster";
 
 export const metadata: Metadata = {
     title: "Khoj AI - Search",
@@ -35,7 +36,10 @@ export default function RootLayout({
     return (
         <html>
             <ContentSecurityPolicy />
-            <body>{children}</body>
+            <body>
+                {children}
+                <Toaster />
+            </body>
         </html>
     );
 }

src/interface/web/app/settings/page.tsx

@@ -3,7 +3,7 @@
 import styles from "./settings.module.css";
 import "intl-tel-input/styles";
 
-import { Suspense, useEffect, useRef, useState } from "react";
+import { Suspense, useEffect, useState } from "react";
 import { useToast } from "@/components/ui/use-toast";
 
 import { useUserConfig, ModelOptions, UserConfig, SubscriptionStates } from "../common/auth";
@@ -23,14 +23,6 @@ import {
     DropdownMenuTrigger,
 } from "@/components/ui/dropdown-menu";
 import { Table, TableBody, TableCell, TableRow } from "@/components/ui/table";
-import {
-    CommandInput,
-    CommandList,
-    CommandEmpty,
-    CommandGroup,
-    CommandItem,
-    CommandDialog,
-} from "@/components/ui/command";
 
 import {
     ArrowRight,
@@ -56,9 +48,10 @@ import {
     ArrowCircleUp,
     ArrowCircleDown,
     ArrowsClockwise,
-    Check,
     CaretDown,
     Waveform,
+    MagnifyingGlass,
+    Brain,
     EyeSlash,
     Eye,
 } from "@phosphor-icons/react";
@@ -66,312 +59,11 @@ import {
 import Loading from "../components/loading/loading";
 
 import IntlTelInput from "intl-tel-input/react";
-import { uploadDataForIndexing } from "../common/chatFunctions";
-import {
-    AlertDialog,
-    AlertDialogAction,
-    AlertDialogContent,
-    AlertDialogDescription,
-    AlertDialogHeader,
-    AlertDialogTitle,
-} from "@/components/ui/alert-dialog";
-import { Progress } from "@/components/ui/progress";
-import Link from "next/link";
 import { SidebarInset, SidebarProvider, SidebarTrigger } from "@/components/ui/sidebar";
 import { AppSidebar } from "../components/appSidebar/appSidebar";
 import { Separator } from "@/components/ui/separator";
 import { KhojLogoType } from "../components/logo/khojLogo";
 
-const ManageFilesModal: React.FC<{ onClose: () => void }> = ({ onClose }) => {
-    const [syncedFiles, setSyncedFiles] = useState<string[]>([]);
-    const [selectedFiles, setSelectedFiles] = useState<string[]>([]);
-    const [searchQuery, setSearchQuery] = useState("");
-    const [isDragAndDropping, setIsDragAndDropping] = useState(false);
-
-    const [warning, setWarning] = useState<string | null>(null);
-    const [error, setError] = useState<string | null>(null);
-    const [uploading, setUploading] = useState(false);
-    const [progressValue, setProgressValue] = useState(0);
-    const [uploadedFiles, setUploadedFiles] = useState<string[]>([]);
-    const fileInputRef = useRef<HTMLInputElement>(null);
-
-    useEffect(() => {
-        if (!uploading) {
-            setProgressValue(0);
-        }
-
-        if (uploading) {
-            const interval = setInterval(() => {
-                setProgressValue((prev) => {
-                    const increment = Math.floor(Math.random() * 5) + 1; // Generates a random number between 1 and 5
-                    const nextValue = prev + increment;
-                    return nextValue < 100 ? nextValue : 100; // Ensures progress does not exceed 100
-                });
-            }, 800);
-            return () => clearInterval(interval);
-        }
-    }, [uploading]);
-
-    useEffect(() => {
-        const fetchFiles = async () => {
-            try {
-                const response = await fetch("/api/content/computer");
-                if (!response.ok) throw new Error("Failed to fetch files");
-
-                // Extract resonse
-                const syncedFiles = await response.json();
-                // Validate response
-                if (Array.isArray(syncedFiles)) {
-                    // Set synced files state
-                    setSyncedFiles(syncedFiles.toSorted());
-                } else {
-                    console.error("Unexpected data format from API");
-                }
-            } catch (error) {
-                console.error("Error fetching files:", error);
-            }
-        };
-
-        fetchFiles();
-    }, [uploadedFiles]);
-
-    const filteredFiles = syncedFiles.filter((file) =>
-        file.toLowerCase().includes(searchQuery.toLowerCase()),
-    );
-
-    const deleteSelected = async () => {
-        let filesToDelete = selectedFiles.length > 0 ? selectedFiles : filteredFiles;
-
-        if (filesToDelete.length === 0) {
-            return;
-        }
-
-        try {
-            const response = await fetch("/api/content/files", {
-                method: "DELETE",
-                headers: {
-                    "Content-Type": "application/json",
-                },
-                body: JSON.stringify({ files: filesToDelete }),
-            });
-
-            if (!response.ok) throw new Error("Failed to delete files");
-
-            // Update the syncedFiles state
-            setSyncedFiles((prevFiles) =>
-                prevFiles.filter((file) => !filesToDelete.includes(file)),
-            );
-
-            // Reset selectedFiles
-            setSelectedFiles([]);
-        } catch (error) {
-            console.error("Error deleting files:", error);
-        }
-    };
-
-    const deleteFile = async (filename: string) => {
-        try {
-            const response = await fetch(
-                `/api/content/file?filename=${encodeURIComponent(filename)}`,
-                {
-                    method: "DELETE",
-                    headers: {
-                        "Content-Type": "application/json",
-                    },
-                },
-            );
-
-            if (!response.ok) throw new Error("Failed to delete file");
-
-            // Update the syncedFiles state
-            setSyncedFiles((prevFiles) => prevFiles.filter((file) => file !== filename));
-
-            // Remove the file from selectedFiles if it's there
-            setSelectedFiles((prevSelected) => prevSelected.filter((file) => file !== filename));
-        } catch (error) {
-            console.error("Error deleting file:", error);
-        }
-    };
-
-    function handleDragOver(event: React.DragEvent<HTMLDivElement>) {
-        event.preventDefault();
-        setIsDragAndDropping(true);
-    }
-
-    function handleDragLeave(event: React.DragEvent<HTMLDivElement>) {
-        event.preventDefault();
-        setIsDragAndDropping(false);
-    }
-
-    function handleDragAndDropFiles(event: React.DragEvent<HTMLDivElement>) {
-        event.preventDefault();
-        setIsDragAndDropping(false);
-
-        if (!event.dataTransfer.files) return;
-
-        uploadFiles(event.dataTransfer.files);
-    }
-
-    function openFileInput() {
-        if (fileInputRef && fileInputRef.current) {
-            fileInputRef.current.click();
-        }
-    }
-
-    function handleFileChange(event: React.ChangeEvent<HTMLInputElement>) {
-        if (!event.target.files) return;
-
-        uploadFiles(event.target.files);
-    }
-
-    function uploadFiles(files: FileList) {
-        uploadDataForIndexing(files, setWarning, setUploading, setError, setUploadedFiles);
-    }
-
-    return (
-        <CommandDialog open={true} onOpenChange={onClose}>
-            <AlertDialog open={warning !== null || error != null}>
-                <AlertDialogContent>
-                    <AlertDialogHeader>
-                        <AlertDialogTitle>Alert</AlertDialogTitle>
-                    </AlertDialogHeader>
-                    <AlertDialogDescription>{warning || error}</AlertDialogDescription>
-                    <AlertDialogAction
-                        className="bg-slate-400 hover:bg-slate-500"
-                        onClick={() => {
-                            setWarning(null);
-                            setError(null);
-                            setUploading(false);
-                        }}
-                    >
-                        Close
-                    </AlertDialogAction>
-                </AlertDialogContent>
-            </AlertDialog>
-            <div
-                className={`flex flex-col h-full`}
-                onDragOver={handleDragOver}
-                onDragLeave={handleDragLeave}
-                onDrop={handleDragAndDropFiles}
-                onClick={openFileInput}
-            >
-                <input
-                    type="file"
-                    multiple
-                    ref={fileInputRef}
-                    style={{ display: "none" }}
-                    onChange={handleFileChange}
-                />
-                <div className="flex-none p-4">
-                    Upload files
-                    {uploading && (
-                        <Progress
-                            indicatorColor="bg-slate-500"
-                            className="w-full h-2 rounded-full"
-                            value={progressValue}
-                        />
-                    )}
-                </div>
-                <div
-                    className={`flex-none p-4 bg-secondary border-b ${isDragAndDropping ? "animate-pulse" : ""} rounded-lg`}
-                >
-                    <div className="flex items-center justify-center w-full h-32 border-2 border-dashed border-gray-300 rounded-lg">
-                        {isDragAndDropping ? (
-                            <div className="flex items-center justify-center w-full h-full">
-                                <Waveform className="h-6 w-6 mr-2" />
-                                <span>Drop files to upload</span>
-                            </div>
-                        ) : (
-                            <div className="flex items-center justify-center w-full h-full">
-                                <Plus className="h-6 w-6 mr-2" />
-                                <span>Drag and drop files here</span>
-                            </div>
-                        )}
-                    </div>
-                </div>
-            </div>
-            <div className="flex flex-col h-full">
-                <div className="flex-none p-4 bg-background border-b">
-                    <CommandInput
-                        placeholder="Find synced files"
-                        value={searchQuery}
-                        onValueChange={setSearchQuery}
-                    />
-                </div>
-                <div className="flex-grow overflow-auto">
-                    <CommandList>
-                        <CommandEmpty>
-                            {syncedFiles.length === 0 ? (
-                                <div className="flex items-center justify-center">
-                                    <ExclamationMark className="h-4 w-4 mr-2" weight="bold" />
-                                    No files synced
-                                </div>
-                            ) : (
-                                <div>
-                                    Could not find a good match.
-                                    <Link href="/search" className="block">
-                                        Need advanced search? Click here.
-                                    </Link>
-                                </div>
-                            )}
-                        </CommandEmpty>
-                        <CommandGroup heading="Synced files">
-                            {filteredFiles.map((filename: string) => (
-                                <CommandItem
-                                    key={filename}
-                                    value={filename}
-                                    onSelect={(value) => {
-                                        setSelectedFiles((prev) =>
-                                            prev.includes(value)
-                                                ? prev.filter((f) => f !== value)
-                                                : [...prev, value],
-                                        );
-                                    }}
-                                >
-                                    <div className="flex items-center justify-between w-full">
-                                        <div
-                                            className={`flex items-center ${selectedFiles.includes(filename) ? "font-semibold" : ""}`}
-                                        >
-                                            {selectedFiles.includes(filename) && (
-                                                <Check className="h-4 w-4 mr-2" />
-                                            )}
-                                            <span className="break-all">{filename}</span>
-                                        </div>
-                                        <Button
-                                            variant="outline"
-                                            size="sm"
-                                            onClick={() => deleteFile(filename)}
-                                            className="ml-auto"
-                                        >
-                                            <Trash className="h-4 w-4" />
-                                        </Button>
-                                    </div>
-                                </CommandItem>
-                            ))}
-                        </CommandGroup>
-                    </CommandList>
-                </div>
-
-                <div className="flex-none p-4 bg-background border-t">
-                    <div className="flex justify-between">
-                        <Button
-                            variant="outline"
-                            size="sm"
-                            onClick={deleteSelected}
-                            className="mr-2"
-                        >
-                            <Trash className="h-4 w-4 mr-2" />
-                            {selectedFiles.length > 0
-                                ? `Delete Selected (${selectedFiles.length})`
-                                : "Delete All"}
-                        </Button>
-                    </div>
-                </div>
-            </div>
-        </CommandDialog>
-    );
-};
-
 interface DropdownComponentProps {
     items: ModelOptions[];
     selected: number;
@@ -604,7 +296,6 @@ export default function SettingsView() {
     const [numberValidationState, setNumberValidationState] = useState<PhoneNumberValidationState>(
         PhoneNumberValidationState.Verified,
     );
-    const [isManageFilesModalOpen, setIsManageFilesModalOpen] = useState(false);
     const { toast } = useToast();
     const isMobileWidth = useIsMobileWidth();
 
@@ -1162,18 +853,13 @@ export default function SettingsView() {
                                             </Card>
                                         </div>
                                     </div>
-                                    {isManageFilesModalOpen && (
-                                        <ManageFilesModal
-                                            onClose={() => setIsManageFilesModalOpen(false)}
-                                        />
-                                    )}
                                     <div className="section grid gap-8">
                                         <div className="text-2xl">Content</div>
                                         <div className="cards flex flex-wrap gap-16">
                                             <Card id="computer" className={cardClassName}>
-                                                <CardHeader className="flex flex-row text-2xl">
-                                                    <Laptop className="h-8 w-8 mr-2" />
-                                                    Files
+                                                <CardHeader className="flex flex-row text-xl">
+                                                    <Brain className="h-8 w-8 mr-2" />
+                                                    Knowledge Base
                                                     {userConfig.enabled_content_source.computer && (
                                                         <CheckCircle
                                                             className="h-6 w-6 ml-auto text-green-500"
@@ -1182,20 +868,19 @@ export default function SettingsView() {
                                                     )}
                                                 </CardHeader>
                                                 <CardContent className="overflow-hidden pb-12 text-gray-400">
-                                                    Manage your synced files
+                                                    Manage and search through your digital brain.
                                                 </CardContent>
                                                 <CardFooter className="flex flex-wrap gap-4">
                                                     <Button
                                                         variant="outline"
                                                         size="sm"
+                                                        title="Search thorugh files"
                                                         onClick={() =>
-                                                            setIsManageFilesModalOpen(true)
+                                                            (window.location.href = "/search")
                                                         }
                                                     >
-                                                        <>
-                                                            <Files className="h-5 w-5 inline mr-1" />
-                                                            Manage
-                                                        </>
+                                                        <MagnifyingGlass className="h-5 w-5 inline mr-1" />
+                                                        Search
                                                     </Button>
                                                     <Button
                                                         variant="outline"
@@ -1206,7 +891,7 @@ export default function SettingsView() {
                                                         }
                                                     >
                                                         <CloudSlash className="h-5 w-5 inline mr-1" />
-                                                        Disable
+                                                        Clear All
                                                     </Button>
                                                 </CardFooter>
                                             </Card>

src/interface/web/components/ui/pagination.tsx

@@ -0,0 +1,118 @@
+import * as React from "react"
+import { ChevronLeft, ChevronRight, MoreHorizontal } from "lucide-react"
+
+import { cn } from "@/lib/utils"
+import { ButtonProps, buttonVariants } from "@/components/ui/button"
+
+const Pagination = ({ className, ...props }: React.ComponentProps<"nav">) => (
+  <nav
+    role="navigation"
+    aria-label="pagination"
+    className={cn("mx-auto flex w-full justify-center", className)}
+    {...props}
+  />
+)
+Pagination.displayName = "Pagination"
+
+const PaginationContent = React.forwardRef<
+  HTMLUListElement,
+  React.ComponentProps<"ul">
+>(({ className, ...props }, ref) => (
+  <ul
+    ref={ref}
+    className={cn("flex flex-row items-center gap-1", className)}
+    {...props}
+  />
+))
+PaginationContent.displayName = "PaginationContent"
+
+const PaginationItem = React.forwardRef<
+  HTMLLIElement,
+  React.ComponentProps<"li">
+>(({ className, ...props }, ref) => (
+  <li ref={ref} className={cn("", className)} {...props} />
+))
+PaginationItem.displayName = "PaginationItem"
+
+type PaginationLinkProps = {
+  isActive?: boolean
+} & Pick<ButtonProps, "size"> &
+  React.ComponentProps<"a">
+
+const PaginationLink = ({
+  className,
+  isActive,
+  size = "icon",
+  ...props
+}: PaginationLinkProps) => (
+  <a
+    aria-current={isActive ? "page" : undefined}
+    className={cn(
+      buttonVariants({
+        variant: isActive ? "outline" : "ghost",
+        size,
+      }),
+      "no-underline",
+      className
+    )}
+    {...props}
+  />
+)
+PaginationLink.displayName = "PaginationLink"
+
+const PaginationPrevious = ({
+  className,
+  ...props
+}: React.ComponentProps<typeof PaginationLink>) => (
+  <PaginationLink
+    aria-label="Go to previous page"
+    size="default"
+    className={cn("gap-1 pl-2.5", className)}
+    {...props}
+  >
+    <ChevronLeft className="h-4 w-4" />
+    <span>Previous</span>
+  </PaginationLink>
+)
+PaginationPrevious.displayName = "PaginationPrevious"
+
+const PaginationNext = ({
+  className,
+  ...props
+}: React.ComponentProps<typeof PaginationLink>) => (
+  <PaginationLink
+    aria-label="Go to next page"
+    size="default"
+    className={cn("gap-1 pr-2.5", className)}
+    {...props}
+  >
+    <span>Next</span>
+    <ChevronRight className="h-4 w-4" />
+  </PaginationLink>
+)
+PaginationNext.displayName = "PaginationNext"
+
+const PaginationEllipsis = ({
+  className,
+  ...props
+}: React.ComponentProps<"span">) => (
+  <span
+    aria-hidden
+    className={cn("flex h-9 w-9 items-center justify-center", className)}
+    {...props}
+  >
+    <MoreHorizontal className="h-4 w-4" />
+    <span className="sr-only">More pages</span>
+  </span>
+)
+PaginationEllipsis.displayName = "PaginationEllipsis"
+
+export {
+  Pagination,
+  PaginationContent,
+  PaginationEllipsis,
+  PaginationItem,
+  PaginationLink,
+  PaginationNext,
+  PaginationPrevious,
+}

src/khoj/database/adapters/__init__.py

@@ -1523,9 +1523,16 @@ class FileObjectAdapters:
         return await sync_to_async(list)(FileObject.objects.filter(user=user, file_name__in=file_names))
 
     @staticmethod
-    @arequire_valid_user
-    async def aget_all_file_objects(user: KhojUser):
-        return await sync_to_async(list)(FileObject.objects.filter(user=user))
+    @require_valid_user
+    async def aget_all_file_objects(user: KhojUser, start: int = 0, limit: int = 10):
+        query = FileObject.objects.filter(user=user).order_by("-updated_at")[start : start + limit]
+        return await sync_to_async(list)(query)
+
+    @staticmethod
+    @require_valid_user
+    async def aget_number_of_pages(user: KhojUser, limit: int = 10):
+        count = await FileObject.objects.filter(user=user).acount()
+        return math.ceil(count / limit)
 
     @staticmethod
     @arequire_valid_user

src/khoj/database/management/commands/delete_orphaned_fileobjects.py

@@ -0,0 +1,56 @@
+from django.core.management.base import BaseCommand
+from django.db.models import Exists, OuterRef
+
+from khoj.database.models import Entry, FileObject
+
+
+class Command(BaseCommand):
+    help = "Deletes FileObjects that have no associated Entries"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--apply",
+            action="store_true",
+            help="Actually perform the deletion. Without this flag, only shows what would be deleted.",
+        )
+
+    def handle(self, *args, **options):
+        # Find FileObjects with no related entries using subquery
+        orphaned_files = FileObject.objects.annotate(
+            has_entries=Exists(Entry.objects.filter(file_object=OuterRef("pk")))
+        ).filter(has_entries=False)
+
+        total_orphaned = orphaned_files.count()
+        mode = "DELETE" if options["apply"] else "DRY RUN"
+        self.stdout.write(f"[{mode}] Found {total_orphaned} orphaned FileObjects")
+
+        if total_orphaned == 0:
+            self.stdout.write("No orphaned FileObjects to process")
+            return
+
+        # Process in batches of 1000
+        batch_size = 1000
+        processed = 0
+
+        while processed < total_orphaned:
+            # Get batch of IDs to process
+            batch_ids = list(orphaned_files.values_list("id", flat=True)[:batch_size])
+            if not batch_ids:
+                break
+
+            if options["apply"]:
+                # Delete by ID to avoid slice/limit issues
+                count = FileObject.objects.filter(id__in=batch_ids).delete()[0]
+                processed += count
+                self.stdout.write(f"Deleted {processed}/{total_orphaned} orphaned FileObjects")
+            else:
+                processed += len(batch_ids)
+                self.stdout.write(f"Would delete {processed}/{total_orphaned} orphaned FileObjects")
+
+            # Re-query to get fresh state
+            orphaned_files = FileObject.objects.annotate(
+                has_entries=Exists(Entry.objects.filter(file_object=OuterRef("pk")))
+            ).filter(has_entries=False)
+
+        action = "Deleted" if options["apply"] else "Would delete"
+        self.stdout.write(self.style.SUCCESS(f"{action} {processed} orphaned FileObjects"))

src/khoj/database/migrations/0079_entry_file_object.py

@@ -0,0 +1,75 @@
+# Generated by Django 5.0.10 on 2025-01-10 18:28
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+def migrate_entry_objects(apps, schema_editor):
+    Entry = apps.get_model("database", "Entry")
+    FileObject = apps.get_model("database", "FileObject")
+    db_alias = schema_editor.connection.alias
+
+    # Create lookup dictionary of all file objects
+    file_objects_map = {(fo.user_id, fo.file_name): fo for fo in FileObject.objects.using(db_alias).all()}
+
+    # Process entries in chunks of 1000
+    chunk_size = 1000
+    processed = 0
+
+    processed_entry_ids = set()
+
+    while True:
+        entries = list(
+            Entry.objects.using(db_alias)
+            .select_related("user")
+            .filter(file_object__isnull=True)
+            .exclude(id__in=processed_entry_ids)
+            .only("id", "user", "file_path")[:chunk_size]
+        )
+
+        if not entries:
+            break
+
+        processed_entry_ids.update([entry.id for entry in entries])
+
+        entries_to_update = []
+        for entry in entries:
+            try:
+                file_object = file_objects_map.get((entry.user_id, entry.file_path))
+                if file_object:
+                    entry.file_object = file_object
+                    entries_to_update.append(entry)
+            except Exception as e:
+                print(f"Error processing entry {entry.id}: {str(e)}")
+                continue
+
+        if entries_to_update:
+            Entry.objects.using(db_alias).bulk_update(entries_to_update, ["file_object"], batch_size=chunk_size)
+
+        processed += len(entries)
+        print(f"Processed {processed} entries")
+
+
+def reverse_migration(apps, schema_editor):
+    pass
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0078_khojuser_email_verification_code_expiry"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="entry",
+            name="file_object",
+            field=models.ForeignKey(
+                blank=True,
+                default=None,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                to="database.fileobject",
+            ),
+        ),
+        migrations.RunPython(migrate_entry_objects, reverse_migration),
+    ]

src/khoj/database/migrations/0081_merge_20250120_1633.py

@@ -0,0 +1,14 @@
+# Generated by Django 5.0.10 on 2025-01-20 16:33
+
+from typing import List
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0079_entry_file_object"),
+        ("database", "0080_speechtotextmodeloptions_ai_model_api"),
+    ]
+
+    operations: List[str] = []

src/khoj/database/models/__init__.py

@@ -330,6 +330,7 @@ class ProcessLock(DbBaseModel):
         INDEX_CONTENT = "index_content"
         SCHEDULED_JOB = "scheduled_job"
         SCHEDULE_LEADER = "schedule_leader"
+        APPLY_MIGRATIONS = "apply_migrations"
 
     # We need to make sure that some operations are thread-safe. To do so, add locks for potentially shared operations.
     # For example, we need to make sure that only one process is updating the embeddings at a time.
@@ -672,6 +673,14 @@ class ReflectiveQuestion(DbBaseModel):
     user = models.ForeignKey(KhojUser, on_delete=models.CASCADE, default=None, null=True, blank=True)
 
 
+class FileObject(DbBaseModel):
+    # Contains the full text of a file that has associated Entry objects
+    file_name = models.CharField(max_length=400, default=None, null=True, blank=True)
+    raw_text = models.TextField()
+    user = models.ForeignKey(KhojUser, on_delete=models.CASCADE, default=None, null=True, blank=True)
+    agent = models.ForeignKey(Agent, on_delete=models.CASCADE, default=None, null=True, blank=True)
+
+
 class Entry(DbBaseModel):
     class EntryType(models.TextChoices):
         IMAGE = "image"
@@ -703,20 +712,13 @@ class Entry(DbBaseModel):
     hashed_value = models.CharField(max_length=100)
     corpus_id = models.UUIDField(default=uuid.uuid4, editable=False)
     search_model = models.ForeignKey(SearchModelConfig, on_delete=models.SET_NULL, default=None, null=True, blank=True)
+    file_object = models.ForeignKey(FileObject, on_delete=models.CASCADE, default=None, null=True, blank=True)
 
     def save(self, *args, **kwargs):
         if self.user and self.agent:
             raise ValidationError("An Entry cannot be associated with both a user and an agent.")
 
 
-class FileObject(DbBaseModel):
-    # Same as Entry but raw will be a much larger string
-    file_name = models.CharField(max_length=400, default=None, null=True, blank=True)
-    raw_text = models.TextField()
-    user = models.ForeignKey(KhojUser, on_delete=models.CASCADE, default=None, null=True, blank=True)
-    agent = models.ForeignKey(Agent, on_delete=models.CASCADE, default=None, null=True, blank=True)
-
-
 class EntryDates(DbBaseModel):
     date = models.DateField()
     entry = models.ForeignKey(Entry, on_delete=models.CASCADE, related_name="embeddings_dates")

src/khoj/processor/content/text_to_entries.py

@@ -152,8 +152,22 @@ class TextToEntries(ABC):
         with timer("Generated embeddings for entries to add to database in", logger):
             entries_to_process = [hash_to_current_entries[hashed_val] for hashed_val in hashes_to_process]
             data_to_embed = [getattr(entry, key) for entry in entries_to_process]
+            modified_files = {entry.file for entry in entries_to_process}
             embeddings += self.embeddings_model[model.name].embed_documents(data_to_embed)
 
+        file_to_file_object_map = {}
+        if file_to_text_map and modified_files:
+            with timer("Indexed text of modified file in", logger):
+                # create or update text of each updated file indexed on DB
+                for modified_file in modified_files:
+                    raw_text = file_to_text_map[modified_file]
+                    file_object = FileObjectAdapters.get_file_object_by_name(user, modified_file)
+                    if file_object:
+                        FileObjectAdapters.update_raw_text(file_object, raw_text)
+                    else:
+                        file_object = FileObjectAdapters.create_file_object(user, modified_file, raw_text)
+                    file_to_file_object_map[modified_file] = file_object
+
         added_entries: list[DbEntry] = []
         with timer("Added entries to database in", logger):
             num_items = len(hashes_to_process)
@@ -165,6 +179,7 @@ class TextToEntries(ABC):
                 batch_embeddings_to_create: List[DbEntry] = []
                 for entry_hash, new_entry in entry_batch:
                     entry = hash_to_current_entries[entry_hash]
+                    file_object = file_to_file_object_map.get(entry.file, None)
                     batch_embeddings_to_create.append(
                         DbEntry(
                             user=user,
@@ -178,6 +193,7 @@ class TextToEntries(ABC):
                             hashed_value=entry_hash,
                             corpus_id=entry.corpus_id,
                             search_model=model,
+                            file_object=file_object,
                         )
                     )
                 try:
@@ -190,19 +206,6 @@ class TextToEntries(ABC):
                     logger.error(f"Error adding entries to database:\n{batch_indexing_error}\n---\n{e}", exc_info=True)
             logger.debug(f"Added {len(added_entries)} {file_type} entries to database")
 
-        if file_to_text_map:
-            with timer("Indexed text of modified file in", logger):
-                # get the set of modified files from added_entries
-                modified_files = {entry.file_path for entry in added_entries}
-                # create or update text of each updated file indexed on DB
-                for modified_file in modified_files:
-                    raw_text = file_to_text_map[modified_file]
-                    file_object = FileObjectAdapters.get_file_object_by_name(user, modified_file)
-                    if file_object:
-                        FileObjectAdapters.update_raw_text(file_object, raw_text)
-                    else:
-                        FileObjectAdapters.create_file_object(user, modified_file, raw_text)
-
         new_dates = []
         with timer("Indexed dates from added entries in", logger):
             for added_entry in added_entries:

src/khoj/routers/api_content.py

@@ -22,6 +22,7 @@ from starlette.authentication import requires
 from khoj.database import adapters
 from khoj.database.adapters import (
     EntryAdapters,
+    FileObjectAdapters,
     get_user_github_config,
     get_user_notion_config,
 )
@@ -270,6 +271,8 @@ async def delete_content_files(
 
     await EntryAdapters.adelete_entry_by_file(user, filename)
 
+    await FileObjectAdapters.adelete_file_object_by_name(user, filename)
+
     return {"status": "ok"}
 
 
@@ -294,6 +297,8 @@ async def delete_content_file(
     )
 
     deleted_count = await EntryAdapters.adelete_entries_by_filenames(user, files.files)
+    for file in files.files:
+        await FileObjectAdapters.adelete_file_object_by_name(user, file)
 
     return {"status": "ok", "deleted_count": deleted_count}
 
@@ -325,6 +330,77 @@ def get_content_types(request: Request, client: Optional[str] = None):
     return list(configured_content_types & all_content_types)
 
 
+@api_content.get("/files", response_model=Dict[str, str])
+@requires(["authenticated"])
+async def get_all_files(
+    request: Request, client: Optional[str] = None, truncated: Optional[bool] = True, page: int = 0
+):
+    user = request.user.object
+
+    update_telemetry_state(
+        request=request,
+        telemetry_type="api",
+        api="get_all_filenames",
+        client=client,
+    )
+
+    files_data = []
+    page_size = 10
+
+    file_objects = await FileObjectAdapters.aget_all_file_objects(user, start=page * page_size, limit=page_size)
+
+    num_pages = await FileObjectAdapters.aget_number_of_pages(user, page_size)
+
+    for file_object in file_objects:
+        files_data.append(
+            {
+                "file_name": file_object.file_name,
+                "raw_text": file_object.raw_text[:1000] if truncated else file_object.raw_text,
+                "updated_at": str(file_object.updated_at),
+            }
+        )
+
+    data_packet = {
+        "files": files_data,
+        "num_pages": num_pages,
+    }
+
+    return Response(content=json.dumps(data_packet), media_type="application/json", status_code=200)
+
+
+@api_content.get("/file", response_model=Dict[str, str])
+@requires(["authenticated"])
+async def get_file_object(
+    request: Request,
+    file_name: str,
+    client: Optional[str] = None,
+):
+    user = request.user.object
+
+    file_object = (await FileObjectAdapters.aget_file_objects_by_name(user, file_name))[0]
+    if not file_object:
+        return Response(
+            content=json.dumps({"error": "File not found"}),
+            media_type="application/json",
+            status_code=404,
+        )
+
+    update_telemetry_state(
+        request=request,
+        telemetry_type="api",
+        api="get_file",
+        client=client,
+    )
+
+    return Response(
+        content=json.dumps(
+            {"id": file_object.id, "file_name": file_object.file_name, "raw_text": file_object.raw_text}
+        ),
+        media_type="application/json",
+        status_code=200,
+    )
+
+
 @api_content.get("/{content_source}", response_model=List[str])
 @requires(["authenticated"])
 async def get_content_source(