From 57545c1485846aa4c5153c65278051625ee46946 Mon Sep 17 00:00:00 2001 From: sabaimran Date: Fri, 10 Jan 2025 21:06:48 -0800 Subject: [PATCH] Fix the migration script to delete orphaned fileobjects - Remove knowledge page from the sidebar - Improve speed and rendering of the documents in the search page --- .../app/components/appSidebar/appSidebar.tsx | 1 - src/interface/web/app/search/page.tsx | 404 ++++++++---------- src/interface/web/app/settings/page.tsx | 337 +-------------- .../commands/delete_orphaned_fileobjects.py | 17 +- 4 files changed, 195 insertions(+), 564 deletions(-) diff --git a/src/interface/web/app/components/appSidebar/appSidebar.tsx b/src/interface/web/app/components/appSidebar/appSidebar.tsx index b660ee52..d80314b9 100644 --- a/src/interface/web/app/components/appSidebar/appSidebar.tsx +++ b/src/interface/web/app/components/appSidebar/appSidebar.tsx @@ -26,7 +26,6 @@ import { useIsMobileWidth } from "@/app/common/utils"; import { UserPlusIcon } from "lucide-react"; import { useAuthenticatedData } from "@/app/common/auth"; import LoginPrompt from "../loginPrompt/loginPrompt"; -import { url } from "inspector"; // Menu items. const items = [ diff --git a/src/interface/web/app/search/page.tsx b/src/interface/web/app/search/page.tsx index 2d1dd6c5..4eb94179 100644 --- a/src/interface/web/app/search/page.tsx +++ b/src/interface/web/app/search/page.tsx @@ -17,7 +17,6 @@ import { ArrowLeft, ArrowRight, FileDashed, - FileMagnifyingGlass, GithubLogo, Lightbulb, LinkSimple, @@ -26,31 +25,21 @@ import { NotionLogo, Eye, Trash, - ArrowsOutSimple, DotsThreeVertical, Waveform, Plus, + Download, + Brain, + Check, } from "@phosphor-icons/react"; import { Button } from "@/components/ui/button"; import Link from "next/link"; -import { getIconFromFilename } from "../common/iconUtils"; import { formatDateTime, useIsMobileWidth } from "../common/utils"; import { SidebarInset, SidebarProvider, SidebarTrigger } from "@/components/ui/sidebar"; import { AppSidebar } from "../components/appSidebar/appSidebar"; import { Separator } from "@/components/ui/separator"; import { KhojLogoType } from "../components/logo/khojLogo"; import { InlineLoading } from "../components/loading/loading"; -import { - AlertDialog, - AlertDialogContent, - AlertDialogDescription, - AlertDialogFooter, - AlertDialogHeader, - AlertDialogTitle, - AlertDialogCancel, - AlertDialogAction, - AlertDialogTrigger, -} from "@/components/ui/alert-dialog"; import { Dialog, DialogContent, @@ -60,16 +49,13 @@ import { DialogTrigger, } from "@/components/ui/dialog"; import { useToast } from "@/components/ui/use-toast"; -import { Scroll } from "lucide-react"; import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, - DropdownMenuLabel, DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { uploadDataForIndexing } from "../common/chatFunctions"; -import { CommandDialog } from "@/components/ui/command"; import { Progress } from "@/components/ui/progress"; interface AdditionalData { file: string; @@ -102,25 +88,6 @@ function getNoteTypeIcon(source: string) { return ; } -const naturalLanguageSearchQueryExamples = [ - "What does the paper say about climate change?", - "Making a cappuccino at home", - "Benefits of eating mangoes", - "How to plan a wedding on a budget", - "Appointment with Dr. Makinde on 12th August", - "Class notes lecture 3 on quantum mechanics", - "Painting concepts for acrylics", - "Abstract from the paper attention is all you need", - "Climbing Everest without oxygen", - "Solving a rubik's cube in 30 seconds", - "Facts about the planet Mars", - "How to make a website using React", - "Fish at the bottom of the ocean", - "Fish farming Kenya 2021", - "How to make a cake without an oven", - "Installing a solar panel at home", -]; - interface NoteResultProps { note: SearchResult; setFocusSearchResult: (note: SearchResult) => void; @@ -132,7 +99,6 @@ function Note(props: NoteResultProps) { const fileName = isFileNameURL ? note.additional.heading : note.additional.file.split("/").pop(); - const fileIcon = getIconFromFilename(fileName || ".txt", "h-4 w-4 inline mr-2"); return ( @@ -153,8 +119,8 @@ function Note(props: NoteResultProps) { - - {isFileNameURL ? ( + {isFileNameURL && ( + {note.additional.file} - ) : ( -
- {fileIcon} - {note.additional.file} -
- )} - + + )} ); } @@ -179,15 +140,14 @@ function focusNote(note: SearchResult) { const fileName = isFileNameURL ? note.additional.heading : note.additional.file.split("/").pop(); - const fileIcon = getIconFromFilename(fileName || ".txt", "h-4 w-4 inline mr-2"); return ( - + {fileName} - - {isFileNameURL ? ( + {isFileNameURL && ( + {note.additional.file} - ) : ( -
- {fileIcon} - {note.additional.file} -
- )} - + + )}
{note.entry}
 @@ -211,9 +166,8 @@ function focusNote(note: SearchResult) { } const UploadFiles: React.FC<{ - onClose: () => void; setUploadedFiles: (files: string[]) => void; -}> = ({ onClose, setUploadedFiles }) => { +}> = ({ setUploadedFiles }) => { const [isDragAndDropping, setIsDragAndDropping] = useState(false); const [warning, setWarning] = useState(null); @@ -225,6 +179,11 @@ const UploadFiles: React.FC<{ useEffect(() => { if (!uploading) { setProgressValue(0); + if (!warning && !error) { + // Force close the dialog by simulating a click on the escape key + const event = new KeyboardEvent("keydown", { key: "Escape" }); + document.dispatchEvent(event); + } } if (uploading) { @@ -278,16 +237,16 @@ const UploadFiles: React.FC<{ - Build Knowledge Base + Build Your Knowledge Base - Adding files to your Khoj knowledge base allows your AI to search through - your own documents. This helps you get personalized answers, grounded in - your own data. + Add your files to supercharge Khoj's AI with your knowledge. Get instant, + personalized answers powered by your own documents and data.
)}
{isDragAndDropping ? ( @@ -379,8 +338,6 @@ export default function Search() { const [selectedFileFullText, setSelectedFileFullText] = useState(null); const [isDeleting, setIsDeleting] = useState(false); const [uploadedFiles, setUploadedFiles] = useState([]); - const [selectedFiles, setSelectedFiles] = useState([]); - const [filteredFiles, setFilteredFiles] = useState([]); const { toast } = useToast(); @@ -408,36 +365,6 @@ export default function Search() { }); } - const deleteSelected = async () => { - let filesToDelete = selectedFiles.length > 0 ? selectedFiles : filteredFiles; - - if (filesToDelete.length === 0) { - return; - } - - try { - const response = await fetch("/api/content/files", { - method: "DELETE", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ files: filesToDelete }), - }); - - if (!response.ok) throw new Error("Failed to delete files"); - - // Update the syncedFiles state - setUploadedFiles((prevFiles) => - prevFiles.filter((file) => !filesToDelete.includes(file)), - ); - - // Reset selectedFiles - setSelectedFiles([]); - } catch (error) { - console.error("Error deleting files:", error); - } - }; - const fetchFiles = async () => { try { const response = await fetch("/api/content/all"); @@ -468,6 +395,18 @@ export default function Search() { } }; + const handleDownload = (fileName: string, content: string) => { + const blob = new Blob([content], { type: "text/plain" }); + const url = window.URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = `${fileName.split("/").pop()}.txt`; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + window.URL.revokeObjectURL(url); + }; + useEffect(() => { if (!searchQuery.trim()) { setSearchResults(null); @@ -547,19 +486,20 @@ export default function Search() { ) : ( -

Search

+

Search Your Knowledge Base

)}
-
-
+
+
setSearchQuery(e.currentTarget.value)} onKeyDown={(e) => e.key === "Enter" && search()} + value={searchQuery} type="search" placeholder="Search Documents" /> @@ -572,10 +512,9 @@ export default function Search() { Find
- {}} - setUploadedFiles={setUploadedFiles} - /> + {searchResults === null && ( + + )} {searchResultsLoading && (
0 && (
+ {searchResults.map((result, index) => { return ( @@ -618,145 +565,138 @@ export default function Search() {
)} - {searchResults === null && ( -
- {fileObjectsLoading && ( -
- -
- )} - {error &&
{error}
} + {!searchResultsLoading && + searchResults === null && + !searchQuery.trim() && ( +
+ {fileObjectsLoading && ( +
+ +
+ )} + {error &&
{error}
} -
- {files.map((file, index) => ( - - - - - -
{ - setSelectedFileFullText( - null, - ); - setSelectedFile( - file.file_name, - ); - }} - > - {file.file_name - .split("/") - .pop()} -
- - - - +
+ {files.map((file, index) => ( + + + + + +
{ + setSelectedFileFullText( + null, + ); + setSelectedFile( + file.file_name, + ); + }} + > {file.file_name .split("/") .pop()} - - - -

- {!selectedFileFullText && ( - - )} - {selectedFileFullText} -

- - -
- - - - - - - - - - - - - - Delete File - - - - Are you sure you - want to delete - this file? - - - - Cancel - - + + + + +
+ {file.file_name + .split("/") + .pop()} + +
+ + + +

+ {!selectedFileFullText && ( + + )} + { + selectedFileFullText + } +

+ + +
+ + + + + + + + + + + + + + +

+ {file.raw_text.slice(0, 100)}... +

+ + + +
+ {formatDateTime(file.updated_at)} +
+ + + ))} +
-
- )} + )} {searchResults && searchResults.length === 0 && ( diff --git a/src/interface/web/app/settings/page.tsx b/src/interface/web/app/settings/page.tsx index cd419e40..7452e4f9 100644 --- a/src/interface/web/app/settings/page.tsx +++ b/src/interface/web/app/settings/page.tsx @@ -3,7 +3,7 @@ import styles from "./settings.module.css"; import "intl-tel-input/styles"; -import { Suspense, useEffect, useRef, useState } from "react"; +import { Suspense, useEffect, useState } from "react"; import { useToast } from "@/components/ui/use-toast"; import { useUserConfig, ModelOptions, UserConfig, SubscriptionStates } from "../common/auth"; @@ -23,14 +23,6 @@ import { DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { Table, TableBody, TableCell, TableRow } from "@/components/ui/table"; -import { - CommandInput, - CommandList, - CommandEmpty, - CommandGroup, - CommandItem, - CommandDialog, -} from "@/components/ui/command"; import { ArrowRight, @@ -56,320 +48,20 @@ import { ArrowCircleUp, ArrowCircleDown, ArrowsClockwise, - Check, CaretDown, Waveform, + MagnifyingGlass, + Brain, } from "@phosphor-icons/react"; import Loading from "../components/loading/loading"; import IntlTelInput from "intl-tel-input/react"; -import { uploadDataForIndexing } from "../common/chatFunctions"; -import { - AlertDialog, - AlertDialogAction, - AlertDialogContent, - AlertDialogDescription, - AlertDialogHeader, - AlertDialogTitle, -} from "@/components/ui/alert-dialog"; -import { Progress } from "@/components/ui/progress"; -import Link from "next/link"; import { SidebarInset, SidebarProvider, SidebarTrigger } from "@/components/ui/sidebar"; import { AppSidebar } from "../components/appSidebar/appSidebar"; import { Separator } from "@/components/ui/separator"; import { KhojLogoType } from "../components/logo/khojLogo"; -const ManageFilesModal: React.FC<{ onClose: () => void }> = ({ onClose }) => { - const [syncedFiles, setSyncedFiles] = useState([]); - const [selectedFiles, setSelectedFiles] = useState([]); - const [searchQuery, setSearchQuery] = useState(""); - const [isDragAndDropping, setIsDragAndDropping] = useState(false); - - const [warning, setWarning] = useState(null); - const [error, setError] = useState(null); - const [uploading, setUploading] = useState(false); - const [progressValue, setProgressValue] = useState(0); - const [uploadedFiles, setUploadedFiles] = useState([]); - const fileInputRef = useRef(null); - - useEffect(() => { - if (!uploading) { - setProgressValue(0); - } - - if (uploading) { - const interval = setInterval(() => { - setProgressValue((prev) => { - const increment = Math.floor(Math.random() * 5) + 1; // Generates a random number between 1 and 5 - const nextValue = prev + increment; - return nextValue < 100 ? nextValue : 100; // Ensures progress does not exceed 100 - }); - }, 800); - return () => clearInterval(interval); - } - }, [uploading]); - - useEffect(() => { - const fetchFiles = async () => { - try { - const response = await fetch("/api/content/computer"); - if (!response.ok) throw new Error("Failed to fetch files"); - - // Extract resonse - const syncedFiles = await response.json(); - // Validate response - if (Array.isArray(syncedFiles)) { - // Set synced files state - setSyncedFiles(syncedFiles.toSorted()); - } else { - console.error("Unexpected data format from API"); - } - } catch (error) { - console.error("Error fetching files:", error); - } - }; - - fetchFiles(); - }, [uploadedFiles]); - - const filteredFiles = syncedFiles.filter((file) => - file.toLowerCase().includes(searchQuery.toLowerCase()), - ); - - const deleteSelected = async () => { - let filesToDelete = selectedFiles.length > 0 ? selectedFiles : filteredFiles; - - if (filesToDelete.length === 0) { - return; - } - - try { - const response = await fetch("/api/content/files", { - method: "DELETE", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ files: filesToDelete }), - }); - - if (!response.ok) throw new Error("Failed to delete files"); - - // Update the syncedFiles state - setSyncedFiles((prevFiles) => - prevFiles.filter((file) => !filesToDelete.includes(file)), - ); - - // Reset selectedFiles - setSelectedFiles([]); - } catch (error) { - console.error("Error deleting files:", error); - } - }; - - const deleteFile = async (filename: string) => { - try { - const response = await fetch( - `/api/content/file?filename=${encodeURIComponent(filename)}`, - { - method: "DELETE", - headers: { - "Content-Type": "application/json", - }, - }, - ); - - if (!response.ok) throw new Error("Failed to delete file"); - - // Update the syncedFiles state - setSyncedFiles((prevFiles) => prevFiles.filter((file) => file !== filename)); - - // Remove the file from selectedFiles if it's there - setSelectedFiles((prevSelected) => prevSelected.filter((file) => file !== filename)); - } catch (error) { - console.error("Error deleting file:", error); - } - }; - - function handleDragOver(event: React.DragEvent) { - event.preventDefault(); - setIsDragAndDropping(true); - } - - function handleDragLeave(event: React.DragEvent) { - event.preventDefault(); - setIsDragAndDropping(false); - } - - function handleDragAndDropFiles(event: React.DragEvent) { - event.preventDefault(); - setIsDragAndDropping(false); - - if (!event.dataTransfer.files) return; - - uploadFiles(event.dataTransfer.files); - } - - function openFileInput() { - if (fileInputRef && fileInputRef.current) { - fileInputRef.current.click(); - } - } - - function handleFileChange(event: React.ChangeEvent) { - if (!event.target.files) return; - - uploadFiles(event.target.files); - } - - function uploadFiles(files: FileList) { - uploadDataForIndexing(files, setWarning, setUploading, setError, setUploadedFiles); - } - - return ( - - - - - Alert - - {warning || error} - { - setWarning(null); - setError(null); - setUploading(false); - }} - > - Close - - - -
- -
- Upload files - {uploading && ( - - )} -
-
-
- {isDragAndDropping ? ( -
- - Drop files to upload -
- ) : ( -
- - Drag and drop files here -
- )} -
-
-
-
-
- -
-
- - - {syncedFiles.length === 0 ? ( -
- - No files synced -
- ) : ( -
- Could not find a good match. - - Need advanced search? Click here. - -
- )} - - - {filteredFiles.map((filename: string) => ( - { - setSelectedFiles((prev) => - prev.includes(value) - ? prev.filter((f) => f !== value) - : [...prev, value], - ); - }} - > -
-
- {selectedFiles.includes(filename) && ( - - )} - {filename} -
- -
- - ))} - - -
- -
-
- -
-
-
- - ); -}; - interface DropdownComponentProps { items: ModelOptions[]; selected: number; @@ -508,7 +200,6 @@ export default function SettingsView() { const [numberValidationState, setNumberValidationState] = useState( PhoneNumberValidationState.Verified, ); - const [isManageFilesModalOpen, setIsManageFilesModalOpen] = useState(false); const { toast } = useToast(); const isMobileWidth = useIsMobileWidth(); @@ -1066,18 +757,13 @@ export default function SettingsView() {
- {isManageFilesModalOpen && ( - setIsManageFilesModalOpen(false)} - /> - )}
Content
- - Files + + Knowledge Base {userConfig.enabled_content_source.computer && ( - Manage your synced files + Manage and search through your digital brain. diff --git a/src/khoj/database/management/commands/delete_orphaned_fileobjects.py b/src/khoj/database/management/commands/delete_orphaned_fileobjects.py index 95cec137..99d45c6f 100644 --- a/src/khoj/database/management/commands/delete_orphaned_fileobjects.py +++ b/src/khoj/database/management/commands/delete_orphaned_fileobjects.py @@ -32,18 +32,25 @@ class Command(BaseCommand): batch_size = 1000 processed = 0 - while True: - batch = orphaned_files[:batch_size] - if not batch: + while processed < total_orphaned: + # Get batch of IDs to process + batch_ids = list(orphaned_files.values_list("id", flat=True)[:batch_size]) + if not batch_ids: break if options["apply"]: - count = batch.delete()[0] + # Delete by ID to avoid slice/limit issues + count = FileObject.objects.filter(id__in=batch_ids).delete()[0] processed += count self.stdout.write(f"Deleted {processed}/{total_orphaned} orphaned FileObjects") else: - processed += len(batch) + processed += len(batch_ids) self.stdout.write(f"Would delete {processed}/{total_orphaned} orphaned FileObjects") + # Re-query to get fresh state + orphaned_files = FileObject.objects.annotate( + has_entries=Exists(Entry.objects.filter(file_object=OuterRef("pk"))) + ).filter(has_entries=False) + action = "Deleted" if options["apply"] else "Would delete" self.stdout.write(self.style.SUCCESS(f"{action} {processed} orphaned FileObjects"))