klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-02 21:19:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Allow browsing and discovery of knowlege base in the search page #1073

Browse Source 
Currently, it's rather opaque and difficult to substantially browse through the uploaded knowledge base. Effectively, you can only do this through the small file modal in the settings page. 

Update to include all indexed files in the search page for viewing & deletion. Function to delete all files is still in the settings page.

Add a migration that associates file objects with `entry`s using a foreign key. Add a migration command that deletes dangling fileobjects.
This commit is contained in:
sabaimran
2025-01-20 16:14:25 -08:00
committed by GitHub
parent 4b35bee365 8ad60f53d3
commit e0dcd11f34
12 changed files with 1128 additions and 482 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
src/interface/web/app/common/utils.ts
View File

@@ -94,3 +94,33 @@ export function useDebounce<T>(value: T, delay: number): T {
return debouncedValue;
}
export const formatDateTime = (isoString: string): string => {
try {
const date = new Date(isoString);
const now = new Date();
const diffInMinutes = Math.floor((now.getTime() - date.getTime()) / 60000);
// Show relative time for recent dates
if (diffInMinutes < 1) return "just now";
if (diffInMinutes < 60) return `${diffInMinutes} minutes ago`;
if (diffInMinutes < 120) return "1 hour ago";
if (diffInMinutes < 1440) return `${Math.floor(diffInMinutes / 60)} hours ago`;
// For older dates, show full formatted date
const formatter = new Intl.DateTimeFormat("en-US", {
month: "long",
day: "numeric",
year: "numeric",
hour: "numeric",
minute: "2-digit",
hour12: true,
timeZoneName: "short",
});
return formatter.format(date);
} catch (error) {
console.error("Error formatting date:", error);
return isoString;
}
};

6
src/interface/web/app/search/layout.tsx
View File

@@ -2,6 +2,7 @@ import type { Metadata } from "next";
import "../globals.css";
import { ContentSecurityPolicy } from "../common/layoutHelper";
import { Toaster } from "@/components/ui/toaster";
export const metadata: Metadata = {
title: "Khoj AI - Search",
@@ -35,7 +36,10 @@ export default function RootLayout({
return (
<html>
<ContentSecurityPolicy />
<body>{children}</body>
<body>
{children}
<Toaster />
</body>
</html>
);
}

836
src/interface/web/app/search/page.tsx
View File

@@ -17,23 +17,72 @@ import {
ArrowLeft,
ArrowRight,
FileDashed,
FileMagnifyingGlass,
GithubLogo,
Lightbulb,
LinkSimple,
MagnifyingGlass,
NoteBlank,
NotionLogo,
Trash,
DotsThreeVertical,
Waveform,
Plus,
Download,
Brain,
Check,
BoxArrowDown,
Funnel,
} from "@phosphor-icons/react";
import { Button } from "@/components/ui/button";
import Link from "next/link";
import { getIconFromFilename } from "../common/iconUtils";
import { useIsMobileWidth } from "../common/utils";
import { formatDateTime, useIsMobileWidth } from "../common/utils";
import { SidebarInset, SidebarProvider, SidebarTrigger } from "@/components/ui/sidebar";
import { AppSidebar } from "../components/appSidebar/appSidebar";
import { Separator } from "@/components/ui/separator";
import { KhojLogoType } from "../components/logo/khojLogo";
import { InlineLoading } from "../components/loading/loading";
import {
Dialog,
DialogContent,
DialogDescription,
DialogHeader,
DialogTitle,
DialogTrigger,
} from "@/components/ui/dialog";
import { useToast } from "@/components/ui/use-toast";
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import {
Pagination,
PaginationContent,
PaginationEllipsis,
PaginationItem,
PaginationLink,
PaginationNext,
PaginationPrevious,
} from "@/components/ui/pagination";
import {
Command,
CommandEmpty,
CommandGroup,
CommandInput,
CommandItem,
CommandList,
} from "@/components/ui/command"
import {
Popover,
PopoverContent,
PopoverTrigger,
} from "@/components/ui/popover"
import { uploadDataForIndexing } from "../common/chatFunctions";
import { Progress } from "@/components/ui/progress";
import { cn } from "@/lib/utils";
interface AdditionalData {
file: string;
source: string;
@@ -49,6 +98,12 @@ interface SearchResult {
"corpus-id": string;
}
interface FileObject {
file_name: string;
raw_text: string;
updated_at: string;
}
function getNoteTypeIcon(source: string) {
if (source === "notion") {
return <NotionLogo className="text-muted-foreground" />;
@@ -59,24 +114,131 @@ function getNoteTypeIcon(source: string) {
return <NoteBlank className="text-muted-foreground" />;
}
const naturalLanguageSearchQueryExamples = [
"What does the paper say about climate change?",
"Making a cappuccino at home",
"Benefits of eating mangoes",
"How to plan a wedding on a budget",
"Appointment with Dr. Makinde on 12th August",
"Class notes lecture 3 on quantum mechanics",
"Painting concepts for acrylics",
"Abstract from the paper attention is all you need",
"Climbing Everest without oxygen",
"Solving a rubik's cube in 30 seconds",
"Facts about the planet Mars",
"How to make a website using React",
"Fish at the bottom of the ocean",
"Fish farming Kenya 2021",
"How to make a cake without an oven",
"Installing a solar panel at home",
];
interface FileCardProps {
file: FileObject;
index: number;
setSelectedFile: (file: string) => void;
setSelectedFileFullText: (file: string) => void;
handleDownload: (fileName: string, content: string) => void;
handleDelete: (fileName: string) => void;
isDeleting: boolean;
selectedFileFullText: string | null;
}
function FileCard({ file, setSelectedFile, setSelectedFileFullText, handleDownload, handleDelete, isDeleting, selectedFileFullText }: FileCardProps) {
return (
<Card
className="animate-fade-in-up bg-secondary h-52"
>
<CardHeader className="p-2">
<CardTitle
className="flex items-center gap-2 justify-between"
title={file.file_name}
>
<Dialog>
<DialogTrigger asChild>
<div
className="text-sm font-medium truncate hover:text-clip hover:whitespace-normal cursor-pointer"
onClick={() => {
setSelectedFileFullText(
'',
);
setSelectedFile(
file.file_name,
);
}}
>
{file.file_name
.split("/")
.pop()}
</div>
</DialogTrigger>
<DialogContent>
<DialogHeader>
<DialogTitle>
<div className="flex items-center gap-2">
{file.file_name
.split("/")
.pop()}
<Button
variant={
"ghost"
}
title="Download as plaintext"
onClick={() =>
handleDownload(
file.file_name,
file.raw_text,
)
}
>
<Download className="h-4 w-4" />
</Button>
</div>
</DialogTitle>
</DialogHeader>
<ScrollArea className="h-[50vh]">
<p className="whitespace-pre-wrap break-words text-sm font-normal">
{!selectedFileFullText && (
<InlineLoading
className="mt-4"
message={
"Loading"
}
iconClassName="h-5 w-5"
/>
)}
{
selectedFileFullText
}
</p>
</ScrollArea>
</DialogContent>
</Dialog>
<DropdownMenu>
<DropdownMenuTrigger>
<Button variant={"ghost"}>
<DotsThreeVertical className="h-4 w-4" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent className="flex flex-col gap-0 w-fit">
<DropdownMenuItem className="p-0">
<Button
variant={"ghost"}
className="flex items-center gap-2 p-1 text-sm"
onClick={() => {
handleDelete(
file.file_name,
);
}}
>
<Trash className="h-4 w-4" />
<span className="text-xs">
{isDeleting
? "Deleting..."
: "Delete"}
</span>
</Button>
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
</CardTitle>
</CardHeader>
<CardContent className="p-2">
<ScrollArea className="h-24 bg-background rounded-lg">
<p className="whitespace-pre-wrap break-words text-sm font-normal text-muted-foreground p-2 h-full">
{file.raw_text.slice(0, 100)}...
</p>
</ScrollArea>
</CardContent>
<CardFooter className="flex justify-end gap-2 p-2">
<div className="text-muted-foreground text-xs">
{formatDateTime(file.updated_at)}
</div>
</CardFooter>
</Card>
)
}
interface NoteResultProps {
note: SearchResult;
@@ -89,10 +251,9 @@ function Note(props: NoteResultProps) {
const fileName = isFileNameURL
? note.additional.heading
: note.additional.file.split("/").pop();
const fileIcon = getIconFromFilename(fileName || ".txt", "h-4 w-4 inline mr-2");
return (
<Card className="bg-secondary h-full shadow-sm rounded-lg border border-muted mb-4">
<Card className="bg-secondary h-full shadow-sm rounded-lg border border-muted mb-4 animate-fade-in-up">
<CardHeader>
<CardTitle className="inline-flex gap-2">
{getNoteTypeIcon(note.additional.source)}
@@ -110,8 +271,8 @@ function Note(props: NoteResultProps) {
<ArrowRight className="inline ml-2" />
</Button>
</CardContent>
<CardFooter>
{isFileNameURL ? (
{isFileNameURL && (
<CardFooter>
<a
href={note.additional.file}
target="_blank"
@@ -120,13 +281,8 @@ function Note(props: NoteResultProps) {
<LinkSimple className="inline m-2" />
{note.additional.file}
</a>
) : (
<div className="bg-muted p-2 text-sm rounded-lg text-muted-foreground">
{fileIcon}
{note.additional.file}
</div>
)}
</CardFooter>
</CardFooter>
)}
</Card>
);
}
@@ -136,15 +292,14 @@ function focusNote(note: SearchResult) {
const fileName = isFileNameURL
? note.additional.heading
: note.additional.file.split("/").pop();
const fileIcon = getIconFromFilename(fileName || ".txt", "h-4 w-4 inline mr-2");
return (
<Card className="bg-secondary h-full shadow-sm rounded-lg bg-gradient-to-b from-background to-slate-50 dark:to-gray-950 border border-muted mb-4">
<Card className="bg-secondary h-full shadow-sm rounded-lg border border-muted mb-4 animate-fade-in-up">
<CardHeader>
<CardTitle>{fileName}</CardTitle>
</CardHeader>
<CardFooter>
{isFileNameURL ? (
{isFileNameURL && (
<CardFooter>
<a
href={note.additional.file}
target="_blank"
@@ -153,13 +308,8 @@ function focusNote(note: SearchResult) {
<LinkSimple className="inline" />
{note.additional.file}
</a>
) : (
<div className="bg-muted p-3 text-sm rounded-lg text-muted-foreground flex items-center gap-2">
{fileIcon}
{note.additional.file}
</div>
)}
</CardFooter>
</CardFooter>
)}
<CardContent>
<div className="text-m">{note.entry}</div>
</CardContent>
@@ -167,25 +317,264 @@ function focusNote(note: SearchResult) {
);
}
const UploadFiles: React.FC<{
setUploadedFiles: (files: string[]) => void;
}> = ({ setUploadedFiles }) => {
const [isDragAndDropping, setIsDragAndDropping] = useState(false);
const [warning, setWarning] = useState<string | null>(null);
const [error, setError] = useState<string | null>(null);
const [uploading, setUploading] = useState(false);
const [progressValue, setProgressValue] = useState(0);
const fileInputRef = useRef<HTMLInputElement>(null);
useEffect(() => {
if (!uploading) {
setProgressValue(0);
if (!warning && !error) {
// Force close the dialog by simulating a click on the escape key
const event = new KeyboardEvent("keydown", { key: "Escape" });
document.dispatchEvent(event);
}
}
if (uploading) {
const interval = setInterval(() => {
setProgressValue((prev) => {
const increment = Math.floor(Math.random() * 5) + 1; // Generates a random number between 1 and 5
const nextValue = prev + increment;
return nextValue < 100 ? nextValue : 100; // Ensures progress does not exceed 100
});
}, 800);
return () => clearInterval(interval);
}
}, [uploading]);
function handleDragOver(event: React.DragEvent<HTMLDivElement>) {
event.preventDefault();
setIsDragAndDropping(true);
}
function handleDragLeave(event: React.DragEvent<HTMLDivElement>) {
event.preventDefault();
setIsDragAndDropping(false);
}
function handleDragAndDropFiles(event: React.DragEvent<HTMLDivElement>) {
event.preventDefault();
setIsDragAndDropping(false);
if (!event.dataTransfer.files) return;
uploadFiles(event.dataTransfer.files);
}
function openFileInput() {
if (fileInputRef && fileInputRef.current) {
fileInputRef.current.click();
}
}
function handleFileChange(event: React.ChangeEvent<HTMLInputElement>) {
if (!event.target.files) return;
uploadFiles(event.target.files);
}
function uploadFiles(files: FileList) {
uploadDataForIndexing(files, setWarning, setUploading, setError, setUploadedFiles);
}
return (
<Dialog>
<DialogTrigger asChild>
<Button variant={"secondary"} className="mt-4">
<Brain className="h-4 w-4 mr-2" />
Add Documents
</Button>
</DialogTrigger>
<DialogContent>
<DialogHeader>
<DialogTitle>Build Your Knowledge Base</DialogTitle>
<DialogDescription>
Add context for your Khoj knowledge base.
Quickly search and get personalized answers from your documents.
</DialogDescription>
</DialogHeader>
<div
className={`flex flex-col h-full`}
onDragOver={handleDragOver}
onDragLeave={handleDragLeave}
onDrop={handleDragAndDropFiles}
onClick={openFileInput}
>
<input
type="file"
multiple
ref={fileInputRef}
style={{ display: "none" }}
onChange={handleFileChange}
/>
<div className="flex-none p-4">
{uploading && (
<Progress
indicatorColor="bg-slate-500"
className="w-full h-2 rounded-full"
value={progressValue}
/>
)}
</div>
{warning && (
<div className="flex-none p-4 border-b rounded-lg">
<div className="flex items-center gap-2">
<Lightbulb className="h-6 w-6" />
<span>{warning}</span>
</div>
<Button
onClick={() => setWarning(null)}
className="mt-2"
variant={"ghost"}
>
Dismiss
</Button>
</div>
)}
{error && (
<div className="flex-none p-4 border-b rounded-lg">
<div className="flex items-center gap-2">
<Lightbulb className="h-6 w-6" />
<span>{error}</span>
</div>
<Button
onClick={() => setError(null)}
className="mt-2"
variant={"ghost"}
>
Dismiss
</Button>
</div>
)}
<div
className={`flex-none p-4 border-b ${isDragAndDropping ? "animate-pulse border-blue-500 bg-blue-500 bg-opacity-25" : "bg-secondary"} rounded-lg`}
>
<div className="flex items-center justify-center w-full h-32 border-2 border-dashed border-gray-300 rounded-lg">
{isDragAndDropping ? (
<div className="flex items-center justify-center w-full h-full">
<Waveform className="h-6 w-6 mr-2" />
<span>Drop files to upload</span>
</div>
) : (
<div className="flex items-center justify-center w-full h-full">
<Plus className="h-6 w-6 mr-2" />
<span>Drag and drop files here</span>
</div>
)}
</div>
</div>
</div>
</DialogContent>
</Dialog>
);
};
interface FileFilterComboBoxProps {
allFiles: string[];
onChooseFile: (file: string) => void;
isMobileWidth: boolean;
explicitFile?: string;
}
function FileFilterComboBox(props: FileFilterComboBoxProps) {
const [open, setOpen] = useState(false)
const [value, setValue] = useState(props.explicitFile || "")
useEffect(() => {
if (props.explicitFile) {
setValue(props.explicitFile);
}
}, [props.explicitFile]);
return (
<Popover open={open} onOpenChange={setOpen}>
<PopoverTrigger asChild>
<Button
variant="outline"
role="combobox"
aria-expanded={open}
className={`justify-between" ${props.isMobileWidth ? "w-full" : "w-[200px]"}`}
>
{value
? (
props.isMobileWidth ?
"✔️"
: "Selected"
)
: (
props.isMobileWidth ?
" "
: "Select file"
)
}
<Funnel className="opacity-50" />
</Button>
</PopoverTrigger>
<PopoverContent className="w-[200px] p-0">
<Command>
<CommandInput placeholder="Search framework..." />
<CommandList>
<CommandEmpty>No framework found.</CommandEmpty>
<CommandGroup>
{props.allFiles.map((file) => (
<CommandItem
key={file}
value={file}
onSelect={(currentValue) => {
setValue(currentValue === value ? "" : currentValue)
setOpen(false)
props.onChooseFile(currentValue)
}}
>
{file}
<Check
className={cn(
"ml-auto",
value === file ? "opacity-100" : "opacity-0"
)}
/>
</CommandItem>
))}
</CommandGroup>
</CommandList>
</Command>
</PopoverContent>
</Popover>
)
}
export default function Search() {
const [searchQuery, setSearchQuery] = useState("");
const [searchResults, setSearchResults] = useState<SearchResult[] | null>(null);
const [searchResultsLoading, setSearchResultsLoading] = useState(false);
const searchInputRef = useRef<HTMLInputElement>(null);
const [focusSearchResult, setFocusSearchResult] = useState<SearchResult | null>(null);
const [exampleQuery, setExampleQuery] = useState("");
const [fileSuggestions, setFileSuggestions] = useState<string[]>([]);
const [files, setFiles] = useState<FileObject[]>([]);
const [error, setError] = useState<string | null>(null);
const [fileObjectsLoading, setFileObjectsLoading] = useState(true);
const [allFiles, setAllFiles] = useState<string[]>([]);
const [showSuggestions, setShowSuggestions] = useState(false);
const searchTimeoutRef = useRef<NodeJS.Timeout | null>(null);
const [selectedFile, setSelectedFile] = useState<string | null>(null);
const [selectedFileFilter, setSelectedFileFilter] = useState<string | undefined>(undefined);
const [selectedFileFullText, setSelectedFileFullText] = useState<string | null>(null);
const [isDeleting, setIsDeleting] = useState(false);
const [uploadedFiles, setUploadedFiles] = useState<string[]>([]);
const [pageNumber, setPageNumber] = useState(0);
const [numPages, setNumPages] = useState(1);
const { toast } = useToast();
const isMobileWidth = useIsMobileWidth();
useEffect(() => {
setExampleQuery(
naturalLanguageSearchQueryExamples[
Math.floor(Math.random() * naturalLanguageSearchQueryExamples.length)
],
);
// Load all files once on page load
fetch('/api/content/computer', {
method: 'GET',
@@ -193,44 +582,44 @@ export default function Search() {
'Content-Type': 'application/json',
},
})
.then(response => response.json())
.then(data => {
setAllFiles(data);
})
.catch(error => {
console.error('Error loading files:', error);
});
.then(response => response.json())
.then(data => {
setAllFiles(data);
})
.catch(error => {
console.error('Error loading files:', error);
});
}, []);
function getFileSuggestions(query: string) {
const fileFilterMatch = query.match(/file:([^"\s]*|"[^"]*")?/);
if (!fileFilterMatch) {
setFileSuggestions([]);
setShowSuggestions(false);
return;
useEffect(() => {
// Replace the file: filter with the selected suggestion
const fileFilterMatch = searchQuery.match(/file:([^"\s]*|"[^"]*")?/);
if (fileFilterMatch) {
const extractedFileFilter = fileFilterMatch[1];
// Strip the double quotes
const extractedFileFilterValue = extractedFileFilter?.replace(/"/g, "");
if (extractedFileFilterValue) {
setSelectedFileFilter(extractedFileFilterValue);
}
}
}, [searchQuery]);
const filePrefix = fileFilterMatch[1]?.replace(/^"|"$/g, '').trim() || '';
const filteredSuggestions = allFiles
.filter(file => file.toLowerCase().includes(filePrefix.toLowerCase()))
.sort()
.slice(0, 10);
setFileSuggestions(filteredSuggestions);
setShowSuggestions(true);
}
function handleSearchInputChange(value: string) {
setSearchQuery(value);
if (!value.trim()) {
setSearchResults(null);
return;
}
// Clear previous search timeout
if (searchTimeoutRef.current) {
clearTimeout(searchTimeoutRef.current);
}
// Get file suggestions immediately
getFileSuggestions(value);
// Debounce search
if (value.trim()) {
searchTimeoutRef.current = setTimeout(() => {
@@ -240,16 +629,18 @@ export default function Search() {
}
function applySuggestion(suggestion: string) {
// Replace the file: filter with the selected suggestion
const newQuery = searchQuery.replace(/file:([^"\s]*|"[^"]*")?/, `file:"${suggestion}"`);
// Append the file: filter with the selected suggestion
const newQuery = `file:"${suggestion}" ${searchQuery}`;
setSearchQuery(newQuery);
setShowSuggestions(false);
searchInputRef.current?.focus();
search();
}
function search() {
if (searchResultsLoading || !searchQuery.trim()) return;
setSearchResultsLoading(true);
const apiUrl = `/api/search?q=${encodeURIComponent(searchQuery)}&client=web`;
fetch(apiUrl, {
method: "GET",
@@ -267,6 +658,102 @@ export default function Search() {
});
}
const fetchFiles = async (currentPageNumber: number) => {
try {
const url = `api/content/files?page=${currentPageNumber}`;
const response = await fetch(url);
if (!response.ok) throw new Error("Failed to fetch files");
const filesData = await response.json();
const filesList = filesData.files;
const totalPages = filesData.num_pages;
setNumPages(totalPages);
if (Array.isArray(filesList)) {
setFiles(filesList.toSorted());
}
} catch (error) {
setError("Failed to load files");
console.error("Error fetching files:", error);
} finally {
setFileObjectsLoading(false);
}
};
useEffect(() => {
fetchFiles(pageNumber);
}, [pageNumber]);
const fetchSpecificFile = async (fileName: string) => {
try {
const response = await fetch(`/api/content/file?file_name=${fileName}`);
if (!response.ok) throw new Error("Failed to fetch file");
const file = await response.json();
setSelectedFileFullText(file.raw_text);
} catch (error) {
setError("Failed to load file");
console.error("Error fetching file:", error);
}
};
const handleDownload = (fileName: string, content: string) => {
const blob = new Blob([content], { type: "text/plain" });
const url = window.URL.createObjectURL(blob);
const a = document.createElement("a");
a.href = url;
a.download = `${fileName.split("/").pop()}.txt`;
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
window.URL.revokeObjectURL(url);
};
useEffect(() => {
if (selectedFile) {
fetchSpecificFile(selectedFile);
}
}, [selectedFile]);
useEffect(() => {
fetchFiles(pageNumber);
}, []);
useEffect(() => {
if (uploadedFiles.length > 0) {
setPageNumber(0);
fetchFiles(0);
}
}, [uploadedFiles]);
const handleDelete = async (fileName: string) => {
setIsDeleting(true);
try {
const response = await fetch(`/api/content/file?filename=${fileName}`, {
method: "DELETE",
});
if (!response.ok) throw new Error("Failed to delete file");
toast({
title: "File deleted",
description: `File ${fileName} has been deleted`,
variant: "default",
});
// Refresh files list
fetchFiles(pageNumber);
} catch (error) {
toast({
title: "Error deleting file",
description: `Failed to delete file ${fileName}`,
variant: "destructive",
});
} finally {
setIsDeleting(false);
}
};
return (
<SidebarProvider>
<AppSidebar conversationId={""} />
@@ -279,54 +766,54 @@ export default function Search() {
<KhojLogoType className="h-auto w-16" />
</a>
) : (
<h2 className="text-lg">Search</h2>
<h2 className="text-lg">Search Your Knowledge Base</h2>
)}
</header>
<div>
<div className={`${styles.searchLayout}`}>
<div className="md:w-3/4 sm:w-full mx-auto pt-6 md:pt-8">
<div className="p-4 md:w-3/4 sm:w-full mx-auto">
<div className="flex justify-between items-center border-2 border-muted p-1 gap-1 rounded-lg">
<div className="relative flex-1">
<div className="w-full md:w-5/6 mx-auto pt-6 md:pt-8">
<div className="p-4 w-full mx-auto">
<div className="flex justify-between items-center border-2 border-muted p-1 gap-1 rounded-lg flex-col md:flex-row">
<div className="relative flex-1 w-full">
<Input
autoFocus={true}
className="border-none pl-4"
className="border-none pl-4 focus-visible:ring-transparent focus-visible:ring-offset-transparent"
onChange={(e) => handleSearchInputChange(e.currentTarget.value)}
onKeyDown={(e) => {
if (e.key === "Enter") {
if (showSuggestions && fileSuggestions.length > 0) {
applySuggestion(fileSuggestions[0]);
} else {
search();
}
search();
}
}}
ref={searchInputRef}
type="search"
placeholder="Search Documents (type 'file:' for file suggestions)"
placeholder="Search Documents"
value={searchQuery}
/>
{showSuggestions && fileSuggestions.length > 0 && (
<div className="absolute z-10 w-full mt-1 bg-background border rounded-md shadow-lg">
{fileSuggestions.map((suggestion, index) => (
<div
key={index}
className="px-4 py-2 hover:bg-muted cursor-pointer"
onClick={() => applySuggestion(suggestion)}
>
{suggestion}
</div>
))}
</div>
)}
</div>
<button
className="px-2 gap-2 inline-flex items-center rounded border-l border-gray-300 hover:text-gray-500"
onClick={() => search()}
>
<MagnifyingGlass className="h-4 w-4" />
<span>Find</span>
</button>
<div id="actions" className="flex gap-2">
<FileFilterComboBox allFiles={allFiles} onChooseFile={(file) => applySuggestion(file)} isMobileWidth={isMobileWidth} explicitFile={selectedFileFilter} />
<Button
className="px-2 gap-2 inline-flex rounded-none items-center border-l border-gray-300 hover:text-gray-500"
variant={"ghost"}
onClick={() => search()}
>
<MagnifyingGlass className="h-4 w-4" />
<span>Find</span>
</Button>
</div>
</div>
{searchResults === null && (
<UploadFiles setUploadedFiles={setUploadedFiles} />
)}
{searchResultsLoading && (
<div className="mt-4 flex items-center justify-center">
<InlineLoading
className="mt-4"
message={"Searching"}
iconClassName="h-5 w-5"
/>
</div>
)}
{focusSearchResult && (
<div className="mt-4">
<Button
@@ -341,9 +828,18 @@ export default function Search() {
</div>
)}
{!focusSearchResult &&
!searchResultsLoading &&
searchResults &&
searchResults.length > 0 && (
<div className="mt-4 max-w-[92vw] break-all">
<Button
onClick={() => handleSearchInputChange("")}
className="mb-4"
variant={"outline"}
>
<ArrowLeft className="inline mr-2" />
See All
</Button>
<ScrollArea className="h-[80vh]">
{searchResults.map((result, index) => {
return (
@@ -359,24 +855,104 @@ export default function Search() {
</ScrollArea>
</div>
)}
{searchResults == null && (
<Card className="flex flex-col items-center justify-center border-none shadow-none">
<CardHeader className="flex flex-col items-center justify-center">
<CardDescription className="border-muted-foreground border w-fit rounded-lg mb-2 text-center text-lg p-4">
<FileMagnifyingGlass
weight="fill"
className="text-muted-foreground h-10 w-10"
/>
</CardDescription>
<CardTitle className="text-center">
Search across your documents
</CardTitle>
</CardHeader>
<CardContent className="text-muted-foreground items-center justify-center text-center flex">
<Lightbulb className="inline mr-2" /> {exampleQuery}
</CardContent>
</Card>
)}
{!searchResultsLoading &&
searchResults === null &&
!searchQuery.trim() && (
<div className="w-full mt-4">
{fileObjectsLoading && (
<div className="mt-4 flex items-center justify-center">
<InlineLoading
className="mt-4"
message={"Loading"}
iconClassName="h-5 w-5"
/>
</div>
)}
{error && <div className="text-red-500">{error}</div>}
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
{files.map((file, index) => (
<FileCard
key={index}
file={file}
index={index}
setSelectedFile={setSelectedFile}
setSelectedFileFullText={setSelectedFileFullText}
handleDownload={handleDownload}
handleDelete={handleDelete}
isDeleting={isDeleting}
selectedFileFullText={selectedFileFullText}
/>
))}
</div>
<Pagination>
<PaginationContent>
{/* Show prev button if not on first page */}
{pageNumber > 0 && (
<PaginationItem className="list-none">
<PaginationPrevious onClick={() => setPageNumber(pageNumber - 1)} />
</PaginationItem>
)}
{/* Show first page if not on first two pages */}
{pageNumber > 1 && (
<PaginationItem className="list-none">
<PaginationLink onClick={() => setPageNumber(0)}>1</PaginationLink>
</PaginationItem>
)}
{/* Show ellipsis if there's a gap */}
{pageNumber > 2 && (
<PaginationItem className="list-none">
<PaginationEllipsis />
</PaginationItem>
)}
{/* Show previous page if not on first page */}
{pageNumber > 0 && (
<PaginationItem className="list-none">
<PaginationLink onClick={() => setPageNumber(pageNumber - 1)}>{pageNumber}</PaginationLink>
</PaginationItem>
)}
{/* Current page */}
<PaginationItem className="list-none">
<PaginationLink isActive>{pageNumber + 1}</PaginationLink>
</PaginationItem>
{/* Show next page if not on last page */}
{pageNumber < numPages - 1 && (
<PaginationItem className="list-none">
<PaginationLink onClick={() => setPageNumber(pageNumber + 1)}>{pageNumber + 2}</PaginationLink>
</PaginationItem>
)}
{/* Show ellipsis if there's a gap before last page */}
{pageNumber < numPages - 3 && (
<PaginationItem className="list-none">
<PaginationEllipsis />
</PaginationItem>
)}
{/* Show last page if not on last two pages */}
{pageNumber < numPages - 2 && (
<PaginationItem className="list-none">
<PaginationLink onClick={() => setPageNumber(numPages - 1)}>{numPages}</PaginationLink>
</PaginationItem>
)}
{/* Show next button if not on last page */}
{pageNumber < numPages - 1 && (
<PaginationItem className="list-none">
<PaginationNext onClick={() => setPageNumber(pageNumber + 1)} />
</PaginationItem>
)}
</PaginationContent>
</Pagination>
</div>
)}
{searchResults && searchResults.length === 0 && (
<Card className="flex flex-col items-center justify-center border-none shadow-none">
<CardHeader className="flex flex-col items-center justify-center">

339
src/interface/web/app/settings/page.tsx
View File

@@ -3,7 +3,7 @@
import styles from "./settings.module.css";
import "intl-tel-input/styles";
import { Suspense, useEffect, useRef, useState } from "react";
import { Suspense, useEffect, useState } from "react";
import { useToast } from "@/components/ui/use-toast";
import { useUserConfig, ModelOptions, UserConfig, SubscriptionStates } from "../common/auth";
@@ -23,14 +23,6 @@ import {
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import { Table, TableBody, TableCell, TableRow } from "@/components/ui/table";
import {
CommandInput,
CommandList,
CommandEmpty,
CommandGroup,
CommandItem,
CommandDialog,
} from "@/components/ui/command";
import {
ArrowRight,
@@ -56,9 +48,10 @@ import {
ArrowCircleUp,
ArrowCircleDown,
ArrowsClockwise,
Check,
CaretDown,
Waveform,
MagnifyingGlass,
Brain,
EyeSlash,
Eye,
} from "@phosphor-icons/react";
@@ -66,312 +59,11 @@ import {
import Loading from "../components/loading/loading";
import IntlTelInput from "intl-tel-input/react";
import { uploadDataForIndexing } from "../common/chatFunctions";
import {
AlertDialog,
AlertDialogAction,
AlertDialogContent,
AlertDialogDescription,
AlertDialogHeader,
AlertDialogTitle,
} from "@/components/ui/alert-dialog";
import { Progress } from "@/components/ui/progress";
import Link from "next/link";
import { SidebarInset, SidebarProvider, SidebarTrigger } from "@/components/ui/sidebar";
import { AppSidebar } from "../components/appSidebar/appSidebar";
import { Separator } from "@/components/ui/separator";
import { KhojLogoType } from "../components/logo/khojLogo";
const ManageFilesModal: React.FC<{ onClose: () => void }> = ({ onClose }) => {
const [syncedFiles, setSyncedFiles] = useState<string[]>([]);
const [selectedFiles, setSelectedFiles] = useState<string[]>([]);
const [searchQuery, setSearchQuery] = useState("");
const [isDragAndDropping, setIsDragAndDropping] = useState(false);
const [warning, setWarning] = useState<string | null>(null);
const [error, setError] = useState<string | null>(null);
const [uploading, setUploading] = useState(false);
const [progressValue, setProgressValue] = useState(0);
const [uploadedFiles, setUploadedFiles] = useState<string[]>([]);
const fileInputRef = useRef<HTMLInputElement>(null);
useEffect(() => {
if (!uploading) {
setProgressValue(0);
}
if (uploading) {
const interval = setInterval(() => {
setProgressValue((prev) => {
const increment = Math.floor(Math.random() * 5) + 1; // Generates a random number between 1 and 5
const nextValue = prev + increment;
return nextValue < 100 ? nextValue : 100; // Ensures progress does not exceed 100
});
}, 800);
return () => clearInterval(interval);
}
}, [uploading]);
useEffect(() => {
const fetchFiles = async () => {
try {
const response = await fetch("/api/content/computer");
if (!response.ok) throw new Error("Failed to fetch files");
// Extract resonse
const syncedFiles = await response.json();
// Validate response
if (Array.isArray(syncedFiles)) {
// Set synced files state
setSyncedFiles(syncedFiles.toSorted());
} else {
console.error("Unexpected data format from API");
}
} catch (error) {
console.error("Error fetching files:", error);
}
};
fetchFiles();
}, [uploadedFiles]);
const filteredFiles = syncedFiles.filter((file) =>
file.toLowerCase().includes(searchQuery.toLowerCase()),
);
const deleteSelected = async () => {
let filesToDelete = selectedFiles.length > 0 ? selectedFiles : filteredFiles;
if (filesToDelete.length === 0) {
return;
}
try {
const response = await fetch("/api/content/files", {
method: "DELETE",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({ files: filesToDelete }),
});
if (!response.ok) throw new Error("Failed to delete files");
// Update the syncedFiles state
setSyncedFiles((prevFiles) =>
prevFiles.filter((file) => !filesToDelete.includes(file)),
);
// Reset selectedFiles
setSelectedFiles([]);
} catch (error) {
console.error("Error deleting files:", error);
}
};
const deleteFile = async (filename: string) => {
try {
const response = await fetch(
`/api/content/file?filename=${encodeURIComponent(filename)}`,
{
method: "DELETE",
headers: {
"Content-Type": "application/json",
},
},
);
if (!response.ok) throw new Error("Failed to delete file");
// Update the syncedFiles state
setSyncedFiles((prevFiles) => prevFiles.filter((file) => file !== filename));
// Remove the file from selectedFiles if it's there
setSelectedFiles((prevSelected) => prevSelected.filter((file) => file !== filename));
} catch (error) {
console.error("Error deleting file:", error);
}
};
function handleDragOver(event: React.DragEvent<HTMLDivElement>) {
event.preventDefault();
setIsDragAndDropping(true);
}
function handleDragLeave(event: React.DragEvent<HTMLDivElement>) {
event.preventDefault();
setIsDragAndDropping(false);
}
function handleDragAndDropFiles(event: React.DragEvent<HTMLDivElement>) {
event.preventDefault();
setIsDragAndDropping(false);
if (!event.dataTransfer.files) return;
uploadFiles(event.dataTransfer.files);
}
function openFileInput() {
if (fileInputRef && fileInputRef.current) {
fileInputRef.current.click();
}
}
function handleFileChange(event: React.ChangeEvent<HTMLInputElement>) {
if (!event.target.files) return;
uploadFiles(event.target.files);
}
function uploadFiles(files: FileList) {
uploadDataForIndexing(files, setWarning, setUploading, setError, setUploadedFiles);
}
return (
<CommandDialog open={true} onOpenChange={onClose}>
<AlertDialog open={warning !== null || error != null}>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>Alert</AlertDialogTitle>
</AlertDialogHeader>
<AlertDialogDescription>{warning || error}</AlertDialogDescription>
<AlertDialogAction
className="bg-slate-400 hover:bg-slate-500"
onClick={() => {
setWarning(null);
setError(null);
setUploading(false);
}}
>
Close
</AlertDialogAction>
</AlertDialogContent>
</AlertDialog>
<div
className={`flex flex-col h-full`}
onDragOver={handleDragOver}
onDragLeave={handleDragLeave}
onDrop={handleDragAndDropFiles}
onClick={openFileInput}
>
<input
type="file"
multiple
ref={fileInputRef}
style={{ display: "none" }}
onChange={handleFileChange}
/>
<div className="flex-none p-4">
Upload files
{uploading && (
<Progress
indicatorColor="bg-slate-500"
className="w-full h-2 rounded-full"
value={progressValue}
/>
)}
</div>
<div
className={`flex-none p-4 bg-secondary border-b ${isDragAndDropping ? "animate-pulse" : ""} rounded-lg`}
>
<div className="flex items-center justify-center w-full h-32 border-2 border-dashed border-gray-300 rounded-lg">
{isDragAndDropping ? (
<div className="flex items-center justify-center w-full h-full">
<Waveform className="h-6 w-6 mr-2" />
<span>Drop files to upload</span>
</div>
) : (
<div className="flex items-center justify-center w-full h-full">
<Plus className="h-6 w-6 mr-2" />
<span>Drag and drop files here</span>
</div>
)}
</div>
</div>
</div>
<div className="flex flex-col h-full">
<div className="flex-none p-4 bg-background border-b">
<CommandInput
placeholder="Find synced files"
value={searchQuery}
onValueChange={setSearchQuery}
/>
</div>
<div className="flex-grow overflow-auto">
<CommandList>
<CommandEmpty>
{syncedFiles.length === 0 ? (
<div className="flex items-center justify-center">
<ExclamationMark className="h-4 w-4 mr-2" weight="bold" />
No files synced
</div>
) : (
<div>
Could not find a good match.
<Link href="/search" className="block">
Need advanced search? Click here.
</Link>
</div>
)}
</CommandEmpty>
<CommandGroup heading="Synced files">
{filteredFiles.map((filename: string) => (
<CommandItem
key={filename}
value={filename}
onSelect={(value) => {
setSelectedFiles((prev) =>
prev.includes(value)
? prev.filter((f) => f !== value)
: [...prev, value],
);
}}
>
<div className="flex items-center justify-between w-full">
<div
className={`flex items-center ${selectedFiles.includes(filename) ? "font-semibold" : ""}`}
>
{selectedFiles.includes(filename) && (
<Check className="h-4 w-4 mr-2" />
)}
<span className="break-all">{filename}</span>
</div>
<Button
variant="outline"
size="sm"
onClick={() => deleteFile(filename)}
className="ml-auto"
>
<Trash className="h-4 w-4" />
</Button>
</div>
</CommandItem>
))}
</CommandGroup>
</CommandList>
</div>
<div className="flex-none p-4 bg-background border-t">
<div className="flex justify-between">
<Button
variant="outline"
size="sm"
onClick={deleteSelected}
className="mr-2"
>
<Trash className="h-4 w-4 mr-2" />
{selectedFiles.length > 0
? `Delete Selected (${selectedFiles.length})`
: "Delete All"}
</Button>
</div>
</div>
</div>
</CommandDialog>
);
};
interface DropdownComponentProps {
items: ModelOptions[];
selected: number;
@@ -604,7 +296,6 @@ export default function SettingsView() {
const [numberValidationState, setNumberValidationState] = useState<PhoneNumberValidationState>(
PhoneNumberValidationState.Verified,
);
const [isManageFilesModalOpen, setIsManageFilesModalOpen] = useState(false);
const { toast } = useToast();
const isMobileWidth = useIsMobileWidth();
@@ -1162,18 +853,13 @@ export default function SettingsView() {
</Card>
</div>
</div>
{isManageFilesModalOpen && (
<ManageFilesModal
onClose={() => setIsManageFilesModalOpen(false)}
/>
)}
<div className="section grid gap-8">
<div className="text-2xl">Content</div>
<div className="cards flex flex-wrap gap-16">
<Card id="computer" className={cardClassName}>
<CardHeader className="flex flex-row text-2xl">
<Laptop className="h-8 w-8 mr-2" />
Files
<CardHeader className="flex flex-row text-xl">
<Brain className="h-8 w-8 mr-2" />
Knowledge Base
{userConfig.enabled_content_source.computer && (
<CheckCircle
className="h-6 w-6 ml-auto text-green-500"
@@ -1182,20 +868,19 @@ export default function SettingsView() {
)}
</CardHeader>
<CardContent className="overflow-hidden pb-12 text-gray-400">
Manage your synced files
Manage and search through your digital brain.
</CardContent>
<CardFooter className="flex flex-wrap gap-4">
<Button
variant="outline"
size="sm"
title="Search thorugh files"
onClick={() =>
setIsManageFilesModalOpen(true)
(window.location.href = "/search")
}
>
<>
<Files className="h-5 w-5 inline mr-1" />
Manage
</>
<MagnifyingGlass className="h-5 w-5 inline mr-1" />
Search
</Button>
<Button
variant="outline"
@@ -1206,7 +891,7 @@ export default function SettingsView() {
}
>
<CloudSlash className="h-5 w-5 inline mr-1" />
Disable
Clear All
</Button>
</CardFooter>
</Card>

118
src/interface/web/components/ui/pagination.tsx Normal file
View File

@@ -0,0 +1,118 @@
import * as React from "react"
import { ChevronLeft, ChevronRight, MoreHorizontal } from "lucide-react"
import { cn } from "@/lib/utils"
import { ButtonProps, buttonVariants } from "@/components/ui/button"
const Pagination = ({ className, ...props }: React.ComponentProps<"nav">) => (
<nav
role="navigation"
aria-label="pagination"
className={cn("mx-auto flex w-full justify-center", className)}
{...props}
/>
)
Pagination.displayName = "Pagination"
const PaginationContent = React.forwardRef<
HTMLUListElement,
React.ComponentProps<"ul">
>(({ className, ...props }, ref) => (
<ul
ref={ref}
className={cn("flex flex-row items-center gap-1", className)}
{...props}
/>
))
PaginationContent.displayName = "PaginationContent"
const PaginationItem = React.forwardRef<
HTMLLIElement,
React.ComponentProps<"li">
>(({ className, ...props }, ref) => (
<li ref={ref} className={cn("", className)} {...props} />
))
PaginationItem.displayName = "PaginationItem"
type PaginationLinkProps = {
isActive?: boolean
} & Pick<ButtonProps, "size"> &
React.ComponentProps<"a">
const PaginationLink = ({
className,
isActive,
size = "icon",
...props
}: PaginationLinkProps) => (
<a
aria-current={isActive ? "page" : undefined}
className={cn(
buttonVariants({
variant: isActive ? "outline" : "ghost",
size,
}),
"no-underline",
className
)}
{...props}
/>
)
PaginationLink.displayName = "PaginationLink"
const PaginationPrevious = ({
className,
...props
}: React.ComponentProps<typeof PaginationLink>) => (
<PaginationLink
aria-label="Go to previous page"
size="default"
className={cn("gap-1 pl-2.5", className)}
{...props}
>
<ChevronLeft className="h-4 w-4" />
<span>Previous</span>
</PaginationLink>
)
PaginationPrevious.displayName = "PaginationPrevious"
const PaginationNext = ({
className,
...props
}: React.ComponentProps<typeof PaginationLink>) => (
<PaginationLink
aria-label="Go to next page"
size="default"
className={cn("gap-1 pr-2.5", className)}
{...props}
>
<span>Next</span>
<ChevronRight className="h-4 w-4" />
</PaginationLink>
)
PaginationNext.displayName = "PaginationNext"
const PaginationEllipsis = ({
className,
...props
}: React.ComponentProps<"span">) => (
<span
aria-hidden
className={cn("flex h-9 w-9 items-center justify-center", className)}
{...props}
>
<MoreHorizontal className="h-4 w-4" />
<span className="sr-only">More pages</span>
</span>
)
PaginationEllipsis.displayName = "PaginationEllipsis"
export {
Pagination,
PaginationContent,
PaginationEllipsis,
PaginationItem,
PaginationLink,
PaginationNext,
PaginationPrevious,
}

13
src/khoj/database/adapters/__init__.py
View File

@@ -1485,9 +1485,16 @@ class FileObjectAdapters:
return await sync_to_async(list)(FileObject.objects.filter(user=user, file_name__in=file_names))
@staticmethod
@arequire_valid_user
async def aget_all_file_objects(user: KhojUser):
return await sync_to_async(list)(FileObject.objects.filter(user=user))
@require_valid_user
async def aget_all_file_objects(user: KhojUser, start: int = 0, limit: int = 10):
query = FileObject.objects.filter(user=user).order_by("-updated_at")[start : start + limit]
return await sync_to_async(list)(query)
@staticmethod
@require_valid_user
async def aget_number_of_pages(user: KhojUser, limit: int = 10):
count = await FileObject.objects.filter(user=user).acount()
return math.ceil(count / limit)
@staticmethod
@arequire_valid_user

56
src/khoj/database/management/commands/delete_orphaned_fileobjects.py Normal file
View File

@@ -0,0 +1,56 @@
from django.core.management.base import BaseCommand
from django.db.models import Exists, OuterRef
from khoj.database.models import Entry, FileObject
class Command(BaseCommand):
help = "Deletes FileObjects that have no associated Entries"
def add_arguments(self, parser):
parser.add_argument(
"--apply",
action="store_true",
help="Actually perform the deletion. Without this flag, only shows what would be deleted.",
)
def handle(self, *args, **options):
# Find FileObjects with no related entries using subquery
orphaned_files = FileObject.objects.annotate(
has_entries=Exists(Entry.objects.filter(file_object=OuterRef("pk")))
).filter(has_entries=False)
total_orphaned = orphaned_files.count()
mode = "DELETE" if options["apply"] else "DRY RUN"
self.stdout.write(f"[{mode}] Found {total_orphaned} orphaned FileObjects")
if total_orphaned == 0:
self.stdout.write("No orphaned FileObjects to process")
return
# Process in batches of 1000
batch_size = 1000
processed = 0
while processed < total_orphaned:
# Get batch of IDs to process
batch_ids = list(orphaned_files.values_list("id", flat=True)[:batch_size])
if not batch_ids:
break
if options["apply"]:
# Delete by ID to avoid slice/limit issues
count = FileObject.objects.filter(id__in=batch_ids).delete()[0]
processed += count
self.stdout.write(f"Deleted {processed}/{total_orphaned} orphaned FileObjects")
else:
processed += len(batch_ids)
self.stdout.write(f"Would delete {processed}/{total_orphaned} orphaned FileObjects")
# Re-query to get fresh state
orphaned_files = FileObject.objects.annotate(
has_entries=Exists(Entry.objects.filter(file_object=OuterRef("pk")))
).filter(has_entries=False)
action = "Deleted" if options["apply"] else "Would delete"
self.stdout.write(self.style.SUCCESS(f"{action} {processed} orphaned FileObjects"))

75
src/khoj/database/migrations/0079_entry_file_object.py Normal file
View File

@@ -0,0 +1,75 @@
# Generated by Django 5.0.10 on 2025-01-10 18:28
import django.db.models.deletion
from django.db import migrations, models
def migrate_entry_objects(apps, schema_editor):
Entry = apps.get_model("database", "Entry")
FileObject = apps.get_model("database", "FileObject")
db_alias = schema_editor.connection.alias
# Create lookup dictionary of all file objects
file_objects_map = {(fo.user_id, fo.file_name): fo for fo in FileObject.objects.using(db_alias).all()}
# Process entries in chunks of 1000
chunk_size = 1000
processed = 0
processed_entry_ids = set()
while True:
entries = list(
Entry.objects.using(db_alias)
.select_related("user")
.filter(file_object__isnull=True)
.exclude(id__in=processed_entry_ids)
.only("id", "user", "file_path")[:chunk_size]
)
if not entries:
break
processed_entry_ids.update([entry.id for entry in entries])
entries_to_update = []
for entry in entries:
try:
file_object = file_objects_map.get((entry.user_id, entry.file_path))
if file_object:
entry.file_object = file_object
entries_to_update.append(entry)
except Exception as e:
print(f"Error processing entry {entry.id}: {str(e)}")
continue
if entries_to_update:
Entry.objects.using(db_alias).bulk_update(entries_to_update, ["file_object"], batch_size=chunk_size)
processed += len(entries)
print(f"Processed {processed} entries")
def reverse_migration(apps, schema_editor):
pass
class Migration(migrations.Migration):
dependencies = [
("database", "0078_khojuser_email_verification_code_expiry"),
]
operations = [
migrations.AddField(
model_name="entry",
name="file_object",
field=models.ForeignKey(
blank=True,
default=None,
null=True,
on_delete=django.db.models.deletion.CASCADE,
to="database.fileobject",
),
),
migrations.RunPython(migrate_entry_objects, reverse_migration),
]

14
src/khoj/database/migrations/0081_merge_20250120_1633.py Normal file
View File

@@ -0,0 +1,14 @@
# Generated by Django 5.0.10 on 2025-01-20 16:33
from typing import List
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("database", "0079_entry_file_object"),
("database", "0080_speechtotextmodeloptions_ai_model_api"),
]
operations: List[str] = []

18
src/khoj/database/models/__init__.py
View File

@@ -327,6 +327,7 @@ class ProcessLock(DbBaseModel):
INDEX_CONTENT = "index_content"
SCHEDULED_JOB = "scheduled_job"
SCHEDULE_LEADER = "schedule_leader"
APPLY_MIGRATIONS = "apply_migrations"
# We need to make sure that some operations are thread-safe. To do so, add locks for potentially shared operations.
# For example, we need to make sure that only one process is updating the embeddings at a time.
@@ -669,6 +670,14 @@ class ReflectiveQuestion(DbBaseModel):
user = models.ForeignKey(KhojUser, on_delete=models.CASCADE, default=None, null=True, blank=True)
class FileObject(DbBaseModel):
# Contains the full text of a file that has associated Entry objects
file_name = models.CharField(max_length=400, default=None, null=True, blank=True)
raw_text = models.TextField()
user = models.ForeignKey(KhojUser, on_delete=models.CASCADE, default=None, null=True, blank=True)
agent = models.ForeignKey(Agent, on_delete=models.CASCADE, default=None, null=True, blank=True)
class Entry(DbBaseModel):
class EntryType(models.TextChoices):
IMAGE = "image"
@@ -700,20 +709,13 @@ class Entry(DbBaseModel):
hashed_value = models.CharField(max_length=100)
corpus_id = models.UUIDField(default=uuid.uuid4, editable=False)
search_model = models.ForeignKey(SearchModelConfig, on_delete=models.SET_NULL, default=None, null=True, blank=True)
file_object = models.ForeignKey(FileObject, on_delete=models.CASCADE, default=None, null=True, blank=True)
def save(self, *args, **kwargs):
if self.user and self.agent:
raise ValidationError("An Entry cannot be associated with both a user and an agent.")
class FileObject(DbBaseModel):
# Same as Entry but raw will be a much larger string
file_name = models.CharField(max_length=400, default=None, null=True, blank=True)
raw_text = models.TextField()
user = models.ForeignKey(KhojUser, on_delete=models.CASCADE, default=None, null=True, blank=True)
agent = models.ForeignKey(Agent, on_delete=models.CASCADE, default=None, null=True, blank=True)
class EntryDates(DbBaseModel):
date = models.DateField()
entry = models.ForeignKey(Entry, on_delete=models.CASCADE, related_name="embeddings_dates")

29
src/khoj/processor/content/text_to_entries.py
View File

@@ -152,8 +152,22 @@ class TextToEntries(ABC):
with timer("Generated embeddings for entries to add to database in", logger):
entries_to_process = [hash_to_current_entries[hashed_val] for hashed_val in hashes_to_process]
data_to_embed = [getattr(entry, key) for entry in entries_to_process]
modified_files = {entry.file for entry in entries_to_process}
embeddings += self.embeddings_model[model.name].embed_documents(data_to_embed)
file_to_file_object_map = {}
if file_to_text_map and modified_files:
with timer("Indexed text of modified file in", logger):
# create or update text of each updated file indexed on DB
for modified_file in modified_files:
raw_text = file_to_text_map[modified_file]
file_object = FileObjectAdapters.get_file_object_by_name(user, modified_file)
if file_object:
FileObjectAdapters.update_raw_text(file_object, raw_text)
else:
file_object = FileObjectAdapters.create_file_object(user, modified_file, raw_text)
file_to_file_object_map[modified_file] = file_object
added_entries: list[DbEntry] = []
with timer("Added entries to database in", logger):
num_items = len(hashes_to_process)
@@ -165,6 +179,7 @@ class TextToEntries(ABC):
batch_embeddings_to_create: List[DbEntry] = []
for entry_hash, new_entry in entry_batch:
entry = hash_to_current_entries[entry_hash]
file_object = file_to_file_object_map.get(entry.file, None)
batch_embeddings_to_create.append(
DbEntry(
user=user,
@@ -178,6 +193,7 @@ class TextToEntries(ABC):
hashed_value=entry_hash,
corpus_id=entry.corpus_id,
search_model=model,
file_object=file_object,
)
)
try:
@@ -190,19 +206,6 @@ class TextToEntries(ABC):
logger.error(f"Error adding entries to database:\n{batch_indexing_error}\n---\n{e}", exc_info=True)
logger.debug(f"Added {len(added_entries)} {file_type} entries to database")
if file_to_text_map:
with timer("Indexed text of modified file in", logger):
# get the set of modified files from added_entries
modified_files = {entry.file_path for entry in added_entries}
# create or update text of each updated file indexed on DB
for modified_file in modified_files:
raw_text = file_to_text_map[modified_file]
file_object = FileObjectAdapters.get_file_object_by_name(user, modified_file)
if file_object:
FileObjectAdapters.update_raw_text(file_object, raw_text)
else:
FileObjectAdapters.create_file_object(user, modified_file, raw_text)
new_dates = []
with timer("Indexed dates from added entries in", logger):
for added_entry in added_entries:

76
src/khoj/routers/api_content.py
View File

@@ -22,6 +22,7 @@ from starlette.authentication import requires
from khoj.database import adapters
from khoj.database.adapters import (
EntryAdapters,
FileObjectAdapters,
get_user_github_config,
get_user_notion_config,
)
@@ -270,6 +271,8 @@ async def delete_content_files(
await EntryAdapters.adelete_entry_by_file(user, filename)
await FileObjectAdapters.adelete_file_object_by_name(user, filename)
return {"status": "ok"}
@@ -294,6 +297,8 @@ async def delete_content_file(
)
deleted_count = await EntryAdapters.adelete_entries_by_filenames(user, files.files)
for file in files.files:
await FileObjectAdapters.adelete_file_object_by_name(user, file)
return {"status": "ok", "deleted_count": deleted_count}
@@ -325,6 +330,77 @@ def get_content_types(request: Request, client: Optional[str] = None):
return list(configured_content_types & all_content_types)
@api_content.get("/files", response_model=Dict[str, str])
@requires(["authenticated"])
async def get_all_files(
request: Request, client: Optional[str] = None, truncated: Optional[bool] = True, page: int = 0
):
user = request.user.object
update_telemetry_state(
request=request,
telemetry_type="api",
api="get_all_filenames",
client=client,
)
files_data = []
page_size = 10
file_objects = await FileObjectAdapters.aget_all_file_objects(user, start=page * page_size, limit=page_size)
num_pages = await FileObjectAdapters.aget_number_of_pages(user, page_size)
for file_object in file_objects:
files_data.append(
{
"file_name": file_object.file_name,
"raw_text": file_object.raw_text[:1000] if truncated else file_object.raw_text,
"updated_at": str(file_object.updated_at),
}
)
data_packet = {
"files": files_data,
"num_pages": num_pages,
}
return Response(content=json.dumps(data_packet), media_type="application/json", status_code=200)
@api_content.get("/file", response_model=Dict[str, str])
@requires(["authenticated"])
async def get_file_object(
request: Request,
file_name: str,
client: Optional[str] = None,
):
user = request.user.object
file_object = (await FileObjectAdapters.aget_file_objects_by_name(user, file_name))[0]
if not file_object:
return Response(
content=json.dumps({"error": "File not found"}),
media_type="application/json",
status_code=404,
)
update_telemetry_state(
request=request,
telemetry_type="api",
api="get_file",
client=client,
)
return Response(
content=json.dumps(
{"id": file_object.id, "file_name": file_object.file_name, "raw_text": file_object.raw_text}
),
media_type="application/json",
status_code=200,
)
@api_content.get("/{content_source}", response_model=List[str])
@requires(["authenticated"])
async def get_content_source(