Initial commit: add a dedicated page for managing the knowledge base

- One current issue in the Khoj application is that managing the files being referenced as the user's knowledge base is slightly opaque and difficult to access
- Add a migration for associating the fileobjects directly with the Entry objects, making it easier to get data via foreign key
- Add the new page that shows all indexed files in the search view, also allowing you to upload new docs directly from that page
- Support new APIs for getting / deleting files
This commit is contained in:
sabaimran
2025-01-10 16:24:50 -08:00
parent 65f1c27963
commit 454a752071
10 changed files with 788 additions and 56 deletions

View File

@@ -94,3 +94,33 @@ export function useDebounce<T>(value: T, delay: number): T {
return debouncedValue;
}
export const formatDateTime = (isoString: string): string => {
try {
const date = new Date(isoString);
const now = new Date();
const diffInMinutes = Math.floor((now.getTime() - date.getTime()) / 60000);
// Show relative time for recent dates
if (diffInMinutes < 1) return "just now";
if (diffInMinutes < 60) return `${diffInMinutes} minutes ago`;
if (diffInMinutes < 120) return "1 hour ago";
if (diffInMinutes < 1440) return `${Math.floor(diffInMinutes / 60)} hours ago`;
// For older dates, show full formatted date
const formatter = new Intl.DateTimeFormat("en-US", {
month: "long",
day: "numeric",
year: "numeric",
hour: "numeric",
minute: "2-digit",
hour12: true,
timeZoneName: "short",
});
return formatter.format(date);
} catch (error) {
console.error("Error formatting date:", error);
return isoString;
}
};

View File

@@ -17,7 +17,7 @@ import {
KhojSearchLogo,
} from "../logo/khojLogo";
import { Gear } from "@phosphor-icons/react/dist/ssr";
import { Plus } from "@phosphor-icons/react";
import { Book, Plus } from "@phosphor-icons/react";
import { useEffect, useState } from "react";
import AllConversations from "../allConversations/allConversations";
import FooterMenu from "../navMenu/navMenu";
@@ -26,6 +26,7 @@ import { useIsMobileWidth } from "@/app/common/utils";
import { UserPlusIcon } from "lucide-react";
import { useAuthenticatedData } from "@/app/common/auth";
import LoginPrompt from "../loginPrompt/loginPrompt";
import { url } from "inspector";
// Menu items.
const items = [
@@ -54,6 +55,11 @@ const items = [
url: "/settings",
icon: Gear,
},
{
title: "Knowledge Base",
url: "/knowledge",
icon: Book,
},
];
const SIDEBAR_KEYBOARD_SHORTCUT = "b";

View File

@@ -0,0 +1,93 @@
"use client";
import { useState, useEffect } from "react";
import { SidebarInset, SidebarProvider, SidebarTrigger } from "@/components/ui/sidebar";
import { AppSidebar } from "../components/appSidebar/appSidebar";
import { Separator } from "@/components/ui/separator";
import { KhojLogoType } from "../components/logo/khojLogo";
import { Card, CardHeader, CardTitle, CardContent } from "@/components/ui/card";
import { useIsMobileWidth } from "../common/utils";
import { InlineLoading } from "../components/loading/loading";
interface FileObject {
file_name: string;
raw_text: string;
}
export default function KnowledgeBase() {
const [files, setFiles] = useState<FileObject[]>([]);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const isMobileWidth = useIsMobileWidth();
useEffect(() => {
const fetchFiles = async () => {
try {
const response = await fetch("/api/content/all");
if (!response.ok) throw new Error("Failed to fetch files");
const filesList = await response.json();
if (Array.isArray(filesList)) {
setFiles(filesList.toSorted());
}
} catch (error) {
setError("Failed to load files");
console.error("Error fetching files:", error);
} finally {
setLoading(false);
}
};
fetchFiles();
}, []);
return (
<SidebarProvider>
<AppSidebar conversationId={""} />
<SidebarInset>
<header className="flex h-16 shrink-0 items-center gap-2 border-b px-4">
<SidebarTrigger className="-ml-1" />
<Separator orientation="vertical" className="mr-2 h-4" />
{isMobileWidth ? (
<a className="p-0 no-underline" href="/">
<KhojLogoType className="h-auto w-16" />
</a>
) : (
<h2 className="text-lg">Knowledge Base</h2>
)}
</header>
<main>
<div className="md:w-3/4 sm:w-full mx-auto pt-6 md:pt-8">
{loading && (
<div className="mt-4 flex items-center justify-center">
<InlineLoading
className="mt-4"
message={"Loading"}
iconClassName="h-5 w-5"
/>
</div>
)}
{error && <div className="text-red-500">{error}</div>}
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
{files.map((file, index) => (
<Card key={index}>
<CardHeader>
<CardTitle className="text-sm font-medium">
{file.file_name.split("/").pop()}
</CardTitle>
</CardHeader>
<CardContent>
<p className="text-sm text-muted-foreground">
{file.raw_text.slice(0, 100)}...
</p>
</CardContent>
</Card>
))}
</div>
</div>
</main>
</SidebarInset>
</SidebarProvider>
);
}

View File

@@ -2,6 +2,7 @@ import type { Metadata } from "next";
import "../globals.css";
import { ContentSecurityPolicy } from "../common/layoutHelper";
import { Toaster } from "@/components/ui/toaster";
export const metadata: Metadata = {
title: "Khoj AI - Search",
@@ -35,7 +36,10 @@ export default function RootLayout({
return (
<html>
<ContentSecurityPolicy />
<body>{children}</body>
<body>
{children}
<Toaster />
</body>
</html>
);
}

View File

@@ -24,16 +24,52 @@ import {
MagnifyingGlass,
NoteBlank,
NotionLogo,
Eye,
Trash,
ArrowsOutSimple,
DotsThreeVertical,
Waveform,
Plus,
} from "@phosphor-icons/react";
import { Button } from "@/components/ui/button";
import Link from "next/link";
import { getIconFromFilename } from "../common/iconUtils";
import { useIsMobileWidth } from "../common/utils";
import { formatDateTime, useIsMobileWidth } from "../common/utils";
import { SidebarInset, SidebarProvider, SidebarTrigger } from "@/components/ui/sidebar";
import { AppSidebar } from "../components/appSidebar/appSidebar";
import { Separator } from "@/components/ui/separator";
import { KhojLogoType } from "../components/logo/khojLogo";
import { InlineLoading } from "../components/loading/loading";
import {
AlertDialog,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle,
AlertDialogCancel,
AlertDialogAction,
AlertDialogTrigger,
} from "@/components/ui/alert-dialog";
import {
Dialog,
DialogContent,
DialogHeader,
DialogTitle,
DialogTrigger,
} from "@/components/ui/dialog";
import { useToast } from "@/components/ui/use-toast";
import { Scroll } from "lucide-react";
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuLabel,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import { uploadDataForIndexing } from "../common/chatFunctions";
import { CommandDialog } from "@/components/ui/command";
import { Progress } from "@/components/ui/progress";
interface AdditionalData {
file: string;
source: string;
@@ -49,6 +85,12 @@ interface SearchResult {
"corpus-id": string;
}
interface FileObject {
file_name: string;
raw_text: string;
updated_at: string;
}
function getNoteTypeIcon(source: string) {
if (source === "notion") {
return <NotionLogo className="text-muted-foreground" />;
@@ -92,7 +134,7 @@ function Note(props: NoteResultProps) {
const fileIcon = getIconFromFilename(fileName || ".txt", "h-4 w-4 inline mr-2");
return (
<Card className="bg-secondary h-full shadow-sm rounded-lg border border-muted mb-4">
<Card className="bg-secondary h-full shadow-sm rounded-lg border border-muted mb-4 animate-fade-in-up">
<CardHeader>
<CardTitle className="inline-flex gap-2">
{getNoteTypeIcon(note.additional.source)}
@@ -139,7 +181,7 @@ function focusNote(note: SearchResult) {
const fileIcon = getIconFromFilename(fileName || ".txt", "h-4 w-4 inline mr-2");
return (
<Card className="bg-secondary h-full shadow-sm rounded-lg bg-gradient-to-b from-background to-slate-50 dark:to-gray-950 border border-muted mb-4">
<Card className="bg-secondary h-full shadow-sm rounded-lg border border-muted mb-4">
<CardHeader>
<CardTitle>{fileName}</CardTitle>
</CardHeader>
@@ -167,27 +209,147 @@ function focusNote(note: SearchResult) {
);
}
const UploadFiles: React.FC<{
onClose: () => void;
setUploadedFiles: (files: string[]) => void;
}> = ({ onClose, setUploadedFiles }) => {
const [syncedFiles, setSyncedFiles] = useState<string[]>([]);
const [selectedFiles, setSelectedFiles] = useState<string[]>([]);
const [searchQuery, setSearchQuery] = useState("");
const [isDragAndDropping, setIsDragAndDropping] = useState(false);
const [warning, setWarning] = useState<string | null>(null);
const [error, setError] = useState<string | null>(null);
const [uploading, setUploading] = useState(false);
const [progressValue, setProgressValue] = useState(0);
const fileInputRef = useRef<HTMLInputElement>(null);
useEffect(() => {
if (!uploading) {
setProgressValue(0);
}
if (uploading) {
const interval = setInterval(() => {
setProgressValue((prev) => {
const increment = Math.floor(Math.random() * 5) + 1; // Generates a random number between 1 and 5
const nextValue = prev + increment;
return nextValue < 100 ? nextValue : 100; // Ensures progress does not exceed 100
});
}, 800);
return () => clearInterval(interval);
}
}, [uploading]);
const filteredFiles = syncedFiles.filter((file) =>
file.toLowerCase().includes(searchQuery.toLowerCase()),
);
function handleDragOver(event: React.DragEvent<HTMLDivElement>) {
event.preventDefault();
setIsDragAndDropping(true);
}
function handleDragLeave(event: React.DragEvent<HTMLDivElement>) {
event.preventDefault();
setIsDragAndDropping(false);
}
function handleDragAndDropFiles(event: React.DragEvent<HTMLDivElement>) {
event.preventDefault();
setIsDragAndDropping(false);
if (!event.dataTransfer.files) return;
uploadFiles(event.dataTransfer.files);
}
function openFileInput() {
if (fileInputRef && fileInputRef.current) {
fileInputRef.current.click();
}
}
function handleFileChange(event: React.ChangeEvent<HTMLInputElement>) {
if (!event.target.files) return;
uploadFiles(event.target.files);
}
function uploadFiles(files: FileList) {
uploadDataForIndexing(files, setWarning, setUploading, setError, setUploadedFiles);
}
return (
<div
className={`flex flex-col h-full`}
onDragOver={handleDragOver}
onDragLeave={handleDragLeave}
onDrop={handleDragAndDropFiles}
onClick={openFileInput}
>
<input
type="file"
multiple
ref={fileInputRef}
style={{ display: "none" }}
onChange={handleFileChange}
/>
<div className="flex-none p-4">
{uploading && (
<Progress
indicatorColor="bg-slate-500"
className="w-full h-2 rounded-full"
value={progressValue}
/>
)}
</div>
<div
className={`flex-none p-4 bg-secondary border-b ${isDragAndDropping ? "animate-pulse" : ""} rounded-lg`}
>
<div className="flex items-center justify-center w-full h-32 border-2 border-dashed border-gray-300 rounded-lg">
{isDragAndDropping ? (
<div className="flex items-center justify-center w-full h-full">
<Waveform className="h-6 w-6 mr-2" />
<span>Drop files to upload</span>
</div>
) : (
<div className="flex items-center justify-center w-full h-full">
<Plus className="h-6 w-6 mr-2" />
<span>Drag and drop files here</span>
</div>
)}
</div>
</div>
</div>
);
};
export default function Search() {
const [searchQuery, setSearchQuery] = useState("");
const [searchResults, setSearchResults] = useState<SearchResult[] | null>(null);
const [searchResultsLoading, setSearchResultsLoading] = useState(false);
const [focusSearchResult, setFocusSearchResult] = useState<SearchResult | null>(null);
const [exampleQuery, setExampleQuery] = useState("");
const [files, setFiles] = useState<FileObject[]>([]);
const [error, setError] = useState<string | null>(null);
const [fileObjectsLoading, setFileObjectsLoading] = useState(true);
const searchTimeoutRef = useRef<NodeJS.Timeout | null>(null);
const [selectedFile, setSelectedFile] = useState<string | null>(null);
const [selectedFileFullText, setSelectedFileFullText] = useState<string | null>(null);
const [isDeleting, setIsDeleting] = useState(false);
const [uploadedFiles, setUploadedFiles] = useState<string[]>([]);
const [selectedFiles, setSelectedFiles] = useState<string[]>([]);
const [filteredFiles, setFilteredFiles] = useState<string[]>([]);
const { toast } = useToast();
const isMobileWidth = useIsMobileWidth();
useEffect(() => {
setExampleQuery(
naturalLanguageSearchQueryExamples[
Math.floor(Math.random() * naturalLanguageSearchQueryExamples.length)
],
);
}, []);
function search() {
if (searchResultsLoading || !searchQuery.trim()) return;
setSearchResultsLoading(true);
const apiUrl = `/api/search?q=${encodeURIComponent(searchQuery)}&client=web`;
fetch(apiUrl, {
method: "GET",
@@ -205,8 +367,69 @@ export default function Search() {
});
}
const deleteSelected = async () => {
let filesToDelete = selectedFiles.length > 0 ? selectedFiles : filteredFiles;
if (filesToDelete.length === 0) {
return;
}
try {
const response = await fetch("/api/content/files", {
method: "DELETE",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({ files: filesToDelete }),
});
if (!response.ok) throw new Error("Failed to delete files");
// Update the syncedFiles state
setUploadedFiles((prevFiles) =>
prevFiles.filter((file) => !filesToDelete.includes(file)),
);
// Reset selectedFiles
setSelectedFiles([]);
} catch (error) {
console.error("Error deleting files:", error);
}
};
const fetchFiles = async () => {
try {
const response = await fetch("/api/content/all");
if (!response.ok) throw new Error("Failed to fetch files");
const filesList = await response.json();
if (Array.isArray(filesList)) {
setFiles(filesList.toSorted());
}
} catch (error) {
setError("Failed to load files");
console.error("Error fetching files:", error);
} finally {
setFileObjectsLoading(false);
}
};
const fetchSpecificFile = async (fileName: string) => {
try {
const response = await fetch(`/api/content/file?file_name=${fileName}`);
if (!response.ok) throw new Error("Failed to fetch file");
const file = await response.json();
setSelectedFileFullText(file.raw_text);
} catch (error) {
setError("Failed to load file");
console.error("Error fetching file:", error);
}
};
useEffect(() => {
if (!searchQuery.trim()) {
setSearchResults(null);
return;
}
@@ -229,6 +452,48 @@ export default function Search() {
};
}, [searchQuery]);
useEffect(() => {
if (selectedFile) {
fetchSpecificFile(selectedFile);
}
}, [selectedFile]);
useEffect(() => {
fetchFiles();
}, []);
useEffect(() => {
if (uploadedFiles.length > 0) {
fetchFiles();
}
}, [uploadedFiles]);
const handleDelete = async (fileName: string) => {
setIsDeleting(true);
try {
const response = await fetch(`/api/content/file?filename=${fileName}`, {
method: "DELETE",
});
if (!response.ok) throw new Error("Failed to delete file");
toast({
title: "File deleted",
description: `File ${fileName} has been deleted`,
variant: "default",
});
// Refresh files list
fetchFiles();
} catch (error) {
toast({
title: "Error deleting file",
description: `Failed to delete file ${fileName}`,
variant: "destructive",
});
} finally {
setIsDeleting(false);
}
};
return (
<SidebarProvider>
<AppSidebar conversationId={""} />
@@ -251,20 +516,34 @@ export default function Search() {
<div className="flex justify-between items-center border-2 border-muted p-1 gap-1 rounded-lg">
<Input
autoFocus={true}
className="border-none pl-4"
className="border-none pl-4 focus-visible:ring-transparent focus-visible:ring-offset-transparent"
onChange={(e) => setSearchQuery(e.currentTarget.value)}
onKeyDown={(e) => e.key === "Enter" && search()}
type="search"
placeholder="Search Documents"
/>
<button
className="px-2 gap-2 inline-flex items-center rounded border-l border-gray-300 hover:text-gray-500"
<Button
className="px-2 gap-2 inline-flex rounded-none items-center border-l border-gray-300 hover:text-gray-500"
variant={"ghost"}
onClick={() => search()}
>
<MagnifyingGlass className="h-4 w-4" />
<span>Find</span>
</button>
</Button>
</div>
<UploadFiles
onClose={() => {}}
setUploadedFiles={setUploadedFiles}
/>
{searchResultsLoading && (
<div className="mt-4 flex items-center justify-center">
<InlineLoading
className="mt-4"
message={"Searching"}
iconClassName="h-5 w-5"
/>
</div>
)}
{focusSearchResult && (
<div className="mt-4">
<Button
@@ -279,6 +558,7 @@ export default function Search() {
</div>
)}
{!focusSearchResult &&
!searchResultsLoading &&
searchResults &&
searchResults.length > 0 && (
<div className="mt-4 max-w-[92vw] break-all">
@@ -297,23 +577,149 @@ export default function Search() {
</ScrollArea>
</div>
)}
{searchResults == null && (
<Card className="flex flex-col items-center justify-center border-none shadow-none">
<CardHeader className="flex flex-col items-center justify-center">
<CardDescription className="border-muted-foreground border w-fit rounded-lg mb-2 text-center text-lg p-4">
<FileMagnifyingGlass
weight="fill"
className="text-muted-foreground h-10 w-10"
{searchResults === null && (
<div className="w-full mt-4">
{fileObjectsLoading && (
<div className="mt-4 flex items-center justify-center">
<InlineLoading
className="mt-4"
message={"Loading"}
iconClassName="h-5 w-5"
/>
</CardDescription>
<CardTitle className="text-center">
Search across your documents
</CardTitle>
</CardHeader>
<CardContent className="text-muted-foreground items-center justify-center text-center flex">
<Lightbulb className="inline mr-2" /> {exampleQuery}
</CardContent>
</Card>
</div>
)}
{error && <div className="text-red-500">{error}</div>}
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
{files.map((file, index) => (
<Card
key={index}
className="animate-fade-in-up bg-secondary h-52"
>
<CardHeader className="p-2">
<CardTitle
className="flex items-center gap-2"
title={file.file_name}
>
<div className="text-sm font-medium truncate hover:text-clip hover:whitespace-normal">
{file.file_name.split("/").pop()}
</div>
<DropdownMenu>
<DropdownMenuTrigger>
<Button variant={"ghost"}>
<DotsThreeVertical className="h-4 w-4" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent className="flex flex-col gap-0 w-fit">
<DropdownMenuItem className="p-0">
<AlertDialog>
<AlertDialogTrigger>
<Button
variant={
"ghost"
}
className="flex items-center gap-2 p-1 text-sm"
>
<Trash className="h-4 w-4" />
<span className="text-xs">
Delete
</span>
</Button>
</AlertDialogTrigger>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>
Delete File
</AlertDialogTitle>
</AlertDialogHeader>
<AlertDialogDescription>
Are you sure you
want to delete
this file?
</AlertDialogDescription>
<AlertDialogFooter>
<AlertDialogCancel>
Cancel
</AlertDialogCancel>
<AlertDialogAction
onClick={() =>
handleDelete(
file.file_name,
)
}
>
{isDeleting
? "Deleting..."
: "Delete"}
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
</DropdownMenuItem>
<DropdownMenuItem className="p-0">
<Dialog>
<DialogTrigger>
<Button
variant={
"ghost"
}
className="flex items-center gap-2 p-1 text-sm"
onClick={() => {
setSelectedFileFullText(
null,
);
setSelectedFile(
file.file_name,
);
}}
>
<ArrowsOutSimple className="h-4 w-4" />
<span className="text-xs">
View Full
Text
</span>
</Button>
</DialogTrigger>
<DialogContent>
<DialogHeader>
<DialogTitle>
{file.file_name
.split(
"/",
)
.pop()}
</DialogTitle>
</DialogHeader>
<ScrollArea className="h-[50vh]">
<p className="whitespace-pre-wrap break-words text-sm font-normal">
{
selectedFileFullText
}
</p>
</ScrollArea>
</DialogContent>
</Dialog>
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
</CardTitle>
</CardHeader>
<CardContent className="p-2">
<ScrollArea className="h-24">
<p className="whitespace-pre-wrap break-words text-sm font-normal text-muted-foreground p-2 rounded-lg bg-background">
{file.raw_text.slice(0, 100)}...
</p>
</ScrollArea>
</CardContent>
<CardFooter className="flex justify-end gap-2 p-2">
<div className="text-muted-foreground text-xs">
{formatDateTime(file.updated_at)}
</div>
</CardFooter>
</Card>
))}
</div>
</div>
)}
{searchResults && searchResults.length === 0 && (
<Card className="flex flex-col items-center justify-center border-none shadow-none">