mirror of
https://github.com/khoaliber/khoj.git
synced 2026-03-02 21:19:12 +00:00
Add some basic pagination logic to the knowledge page to prevent overloading the Api or the client
This commit is contained in:
sabaimran
@@ -55,6 +55,16 @@ import {
|
||||
DropdownMenuItem,
|
||||
DropdownMenuTrigger,
|
||||
} from "@/components/ui/dropdown-menu";
|
||||
import {
|
||||
Pagination,
|
||||
PaginationContent,
|
||||
PaginationEllipsis,
|
||||
PaginationItem,
|
||||
PaginationLink,
|
||||
PaginationNext,
|
||||
PaginationPrevious,
|
||||
} from "@/components/ui/pagination";
|
||||
|
||||
import { uploadDataForIndexing } from "../common/chatFunctions";
|
||||
import { Progress } from "@/components/ui/progress";
|
||||
interface AdditionalData {
|
||||
@@ -88,6 +98,132 @@ function getNoteTypeIcon(source: string) {
|
||||
return <NoteBlank className="text-muted-foreground" />;
|
||||
}
|
||||
|
||||
interface FileCardProps {
|
||||
file: FileObject;
|
||||
index: number;
|
||||
setSelectedFile: (file: string) => void;
|
||||
setSelectedFileFullText: (file: string) => void;
|
||||
handleDownload: (fileName: string, content: string) => void;
|
||||
handleDelete: (fileName: string) => void;
|
||||
isDeleting: boolean;
|
||||
selectedFileFullText: string | null;
|
||||
}
|
||||
|
||||
function FileCard({ file, setSelectedFile, setSelectedFileFullText, handleDownload, handleDelete, isDeleting, selectedFileFullText }: FileCardProps) {
|
||||
return (
|
||||
<Card
|
||||
className="animate-fade-in-up bg-secondary h-52"
|
||||
>
|
||||
<CardHeader className="p-2">
|
||||
<CardTitle
|
||||
className="flex items-center gap-2 justify-between"
|
||||
title={file.file_name}
|
||||
>
|
||||
<Dialog>
|
||||
<DialogTrigger asChild>
|
||||
<div
|
||||
className="text-sm font-medium truncate hover:text-clip hover:whitespace-normal cursor-pointer"
|
||||
onClick={() => {
|
||||
setSelectedFileFullText(
|
||||
'',
|
||||
);
|
||||
setSelectedFile(
|
||||
file.file_name,
|
||||
);
|
||||
}}
|
||||
>
|
||||
{file.file_name
|
||||
.split("/")
|
||||
.pop()}
|
||||
</div>
|
||||
</DialogTrigger>
|
||||
<DialogContent>
|
||||
<DialogHeader>
|
||||
<DialogTitle>
|
||||
<div className="flex items-center gap-2">
|
||||
{file.file_name
|
||||
.split("/")
|
||||
.pop()}
|
||||
<Button
|
||||
variant={
|
||||
"ghost"
|
||||
}
|
||||
title="Download as plaintext"
|
||||
onClick={() =>
|
||||
handleDownload(
|
||||
file.file_name,
|
||||
file.raw_text,
|
||||
)
|
||||
}
|
||||
>
|
||||
<Download className="h-4 w-4" />
|
||||
</Button>
|
||||
</div>
|
||||
</DialogTitle>
|
||||
</DialogHeader>
|
||||
<ScrollArea className="h-[50vh]">
|
||||
<p className="whitespace-pre-wrap break-words text-sm font-normal">
|
||||
{!selectedFileFullText && (
|
||||
<InlineLoading
|
||||
className="mt-4"
|
||||
message={
|
||||
"Loading"
|
||||
}
|
||||
iconClassName="h-5 w-5"
|
||||
/>
|
||||
)}
|
||||
{
|
||||
selectedFileFullText
|
||||
}
|
||||
</p>
|
||||
</ScrollArea>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger>
|
||||
<Button variant={"ghost"}>
|
||||
<DotsThreeVertical className="h-4 w-4" />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent className="flex flex-col gap-0 w-fit">
|
||||
<DropdownMenuItem className="p-0">
|
||||
<Button
|
||||
variant={"ghost"}
|
||||
className="flex items-center gap-2 p-1 text-sm"
|
||||
onClick={() => {
|
||||
handleDelete(
|
||||
file.file_name,
|
||||
);
|
||||
}}
|
||||
>
|
||||
<Trash className="h-4 w-4" />
|
||||
<span className="text-xs">
|
||||
{isDeleting
|
||||
? "Deleting..."
|
||||
: "Delete"}
|
||||
</span>
|
||||
</Button>
|
||||
</DropdownMenuItem>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="p-2">
|
||||
<ScrollArea className="h-24 bg-background rounded-lg">
|
||||
<p className="whitespace-pre-wrap break-words text-sm font-normal text-muted-foreground p-2 h-full">
|
||||
{file.raw_text.slice(0, 100)}...
|
||||
</p>
|
||||
</ScrollArea>
|
||||
</CardContent>
|
||||
<CardFooter className="flex justify-end gap-2 p-2">
|
||||
<div className="text-muted-foreground text-xs">
|
||||
{formatDateTime(file.updated_at)}
|
||||
</div>
|
||||
</CardFooter>
|
||||
</Card>
|
||||
)
|
||||
}
|
||||
|
||||
interface NoteResultProps {
|
||||
note: SearchResult;
|
||||
setFocusSearchResult: (note: SearchResult) => void;
|
||||
@@ -338,6 +474,8 @@ export default function Search() {
|
||||
const [selectedFileFullText, setSelectedFileFullText] = useState<string | null>(null);
|
||||
const [isDeleting, setIsDeleting] = useState(false);
|
||||
const [uploadedFiles, setUploadedFiles] = useState<string[]>([]);
|
||||
const [pageNumber, setPageNumber] = useState(0);
|
||||
const [numPages, setNumPages] = useState(1);
|
||||
|
||||
const { toast } = useToast();
|
||||
|
||||
@@ -365,15 +503,22 @@ export default function Search() {
|
||||
});
|
||||
}
|
||||
|
||||
const fetchFiles = async () => {
|
||||
const fetchFiles = async (currentPageNumber: number) => {
|
||||
try {
|
||||
const response = await fetch("/api/content/all");
|
||||
const url = `api/content/all?page=${currentPageNumber}`;
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) throw new Error("Failed to fetch files");
|
||||
|
||||
const filesList = await response.json();
|
||||
const filesData = await response.json();
|
||||
const filesList = filesData.files;
|
||||
const totalPages = filesData.num_pages;
|
||||
|
||||
setNumPages(totalPages);
|
||||
|
||||
if (Array.isArray(filesList)) {
|
||||
setFiles(filesList.toSorted());
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
setError("Failed to load files");
|
||||
console.error("Error fetching files:", error);
|
||||
@@ -382,6 +527,10 @@ export default function Search() {
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
fetchFiles(pageNumber);
|
||||
}, [pageNumber]);
|
||||
|
||||
const fetchSpecificFile = async (fileName: string) => {
|
||||
try {
|
||||
const response = await fetch(`/api/content/file?file_name=${fileName}`);
|
||||
@@ -439,12 +588,13 @@ export default function Search() {
|
||||
}, [selectedFile]);
|
||||
|
||||
useEffect(() => {
|
||||
fetchFiles();
|
||||
fetchFiles(pageNumber);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (uploadedFiles.length > 0) {
|
||||
fetchFiles();
|
||||
setPageNumber(0);
|
||||
fetchFiles(0);
|
||||
}
|
||||
}, [uploadedFiles]);
|
||||
|
||||
@@ -462,7 +612,8 @@ export default function Search() {
|
||||
});
|
||||
|
||||
// Refresh files list
|
||||
fetchFiles();
|
||||
setPageNumber(0);
|
||||
fetchFiles(0);
|
||||
} catch (error) {
|
||||
toast({
|
||||
title: "Error deleting file",
|
||||
@@ -582,119 +733,85 @@ export default function Search() {
|
||||
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
|
||||
{files.map((file, index) => (
|
||||
<Card
|
||||
<FileCard
|
||||
key={index}
|
||||
className="animate-fade-in-up bg-secondary h-52"
|
||||
>
|
||||
<CardHeader className="p-2">
|
||||
<CardTitle
|
||||
className="flex items-center gap-2 justify-between"
|
||||
title={file.file_name}
|
||||
>
|
||||
<Dialog>
|
||||
<DialogTrigger asChild>
|
||||
<div
|
||||
className="text-sm font-medium truncate hover:text-clip hover:whitespace-normal cursor-pointer"
|
||||
onClick={() => {
|
||||
setSelectedFileFullText(
|
||||
null,
|
||||
);
|
||||
setSelectedFile(
|
||||
file.file_name,
|
||||
);
|
||||
}}
|
||||
>
|
||||
{file.file_name
|
||||
.split("/")
|
||||
.pop()}
|
||||
</div>
|
||||
</DialogTrigger>
|
||||
<DialogContent>
|
||||
<DialogHeader>
|
||||
<DialogTitle>
|
||||
<div className="flex items-center gap-2">
|
||||
{file.file_name
|
||||
.split("/")
|
||||
.pop()}
|
||||
<Button
|
||||
variant={
|
||||
"ghost"
|
||||
}
|
||||
title="Download as plaintext"
|
||||
onClick={() =>
|
||||
handleDownload(
|
||||
file.file_name,
|
||||
file.raw_text,
|
||||
)
|
||||
}
|
||||
>
|
||||
<Download className="h-4 w-4" />
|
||||
</Button>
|
||||
</div>
|
||||
</DialogTitle>
|
||||
</DialogHeader>
|
||||
<ScrollArea className="h-[50vh]">
|
||||
<p className="whitespace-pre-wrap break-words text-sm font-normal">
|
||||
{!selectedFileFullText && (
|
||||
<InlineLoading
|
||||
className="mt-4"
|
||||
message={
|
||||
"Loading"
|
||||
}
|
||||
iconClassName="h-5 w-5"
|
||||
/>
|
||||
)}
|
||||
{
|
||||
selectedFileFullText
|
||||
}
|
||||
</p>
|
||||
</ScrollArea>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger>
|
||||
<Button variant={"ghost"}>
|
||||
<DotsThreeVertical className="h-4 w-4" />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent className="flex flex-col gap-0 w-fit">
|
||||
<DropdownMenuItem className="p-0">
|
||||
<Button
|
||||
variant={"ghost"}
|
||||
className="flex items-center gap-2 p-1 text-sm"
|
||||
onClick={() => {
|
||||
handleDelete(
|
||||
file.file_name,
|
||||
);
|
||||
}}
|
||||
>
|
||||
<Trash className="h-4 w-4" />
|
||||
<span className="text-xs">
|
||||
{isDeleting
|
||||
? "Deleting..."
|
||||
: "Delete"}
|
||||
</span>
|
||||
</Button>
|
||||
</DropdownMenuItem>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="p-2">
|
||||
<ScrollArea className="h-24 bg-background rounded-lg">
|
||||
<p className="whitespace-pre-wrap break-words text-sm font-normal text-muted-foreground p-2 h-full">
|
||||
{file.raw_text.slice(0, 100)}...
|
||||
</p>
|
||||
</ScrollArea>
|
||||
</CardContent>
|
||||
<CardFooter className="flex justify-end gap-2 p-2">
|
||||
<div className="text-muted-foreground text-xs">
|
||||
{formatDateTime(file.updated_at)}
|
||||
</div>
|
||||
</CardFooter>
|
||||
</Card>
|
||||
file={file}
|
||||
index={index}
|
||||
setSelectedFile={setSelectedFile}
|
||||
setSelectedFileFullText={setSelectedFileFullText}
|
||||
handleDownload={handleDownload}
|
||||
handleDelete={handleDelete}
|
||||
isDeleting={isDeleting}
|
||||
selectedFileFullText={selectedFileFullText}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<Pagination>
|
||||
<PaginationContent>
|
||||
{/* Show prev button if not on first page */}
|
||||
{pageNumber > 0 && (
|
||||
<PaginationItem className="list-none">
|
||||
<PaginationPrevious onClick={() => setPageNumber(pageNumber - 1)} />
|
||||
</PaginationItem>
|
||||
)}
|
||||
|
||||
{/* Show first page if not on first two pages */}
|
||||
{pageNumber > 1 && (
|
||||
<PaginationItem className="list-none">
|
||||
<PaginationLink onClick={() => setPageNumber(0)}>1</PaginationLink>
|
||||
</PaginationItem>
|
||||
)}
|
||||
|
||||
{/* Show ellipsis if there's a gap */}
|
||||
{pageNumber > 2 && (
|
||||
<PaginationItem className="list-none">
|
||||
<PaginationEllipsis />
|
||||
</PaginationItem>
|
||||
)}
|
||||
|
||||
{/* Show previous page if not on first page */}
|
||||
{pageNumber > 0 && (
|
||||
<PaginationItem className="list-none">
|
||||
<PaginationLink onClick={() => setPageNumber(pageNumber - 1)}>{pageNumber}</PaginationLink>
|
||||
</PaginationItem>
|
||||
)}
|
||||
|
||||
{/* Current page */}
|
||||
<PaginationItem className="list-none">
|
||||
<PaginationLink isActive>{pageNumber + 1}</PaginationLink>
|
||||
</PaginationItem>
|
||||
|
||||
{/* Show next page if not on last page */}
|
||||
{pageNumber < numPages - 1 && (
|
||||
<PaginationItem className="list-none">
|
||||
<PaginationLink onClick={() => setPageNumber(pageNumber + 1)}>{pageNumber + 2}</PaginationLink>
|
||||
</PaginationItem>
|
||||
)}
|
||||
|
||||
{/* Show ellipsis if there's a gap before last page */}
|
||||
{pageNumber < numPages - 3 && (
|
||||
<PaginationItem className="list-none">
|
||||
<PaginationEllipsis />
|
||||
</PaginationItem>
|
||||
)}
|
||||
|
||||
{/* Show last page if not on last two pages */}
|
||||
{pageNumber < numPages - 2 && (
|
||||
<PaginationItem className="list-none">
|
||||
<PaginationLink onClick={() => setPageNumber(numPages - 1)}>{numPages}</PaginationLink>
|
||||
</PaginationItem>
|
||||
)}
|
||||
|
||||
{/* Show next button if not on last page */}
|
||||
{pageNumber < numPages - 1 && (
|
||||
<PaginationItem className="list-none">
|
||||
<PaginationNext onClick={() => setPageNumber(pageNumber + 1)} />
|
||||
</PaginationItem>
|
||||
)}
|
||||
</PaginationContent>
|
||||
</Pagination>
|
||||
|
||||
</div>
|
||||
)}
|
||||
{searchResults && searchResults.length === 0 && (
|
||||
|
||||
118
src/interface/web/components/ui/pagination.tsx
Normal file
118
src/interface/web/components/ui/pagination.tsx
Normal file
@@ -0,0 +1,118 @@
|
||||
import * as React from "react"
|
||||
import { ChevronLeft, ChevronRight, MoreHorizontal } from "lucide-react"
|
||||
|
||||
import { cn } from "@/lib/utils"
|
||||
import { ButtonProps, buttonVariants } from "@/components/ui/button"
|
||||
|
||||
const Pagination = ({ className, ...props }: React.ComponentProps<"nav">) => (
|
||||
<nav
|
||||
role="navigation"
|
||||
aria-label="pagination"
|
||||
className={cn("mx-auto flex w-full justify-center", className)}
|
||||
{...props}
|
||||
/>
|
||||
)
|
||||
Pagination.displayName = "Pagination"
|
||||
|
||||
const PaginationContent = React.forwardRef<
|
||||
HTMLUListElement,
|
||||
React.ComponentProps<"ul">
|
||||
>(({ className, ...props }, ref) => (
|
||||
<ul
|
||||
ref={ref}
|
||||
className={cn("flex flex-row items-center gap-1", className)}
|
||||
{...props}
|
||||
/>
|
||||
))
|
||||
PaginationContent.displayName = "PaginationContent"
|
||||
|
||||
const PaginationItem = React.forwardRef<
|
||||
HTMLLIElement,
|
||||
React.ComponentProps<"li">
|
||||
>(({ className, ...props }, ref) => (
|
||||
<li ref={ref} className={cn("", className)} {...props} />
|
||||
))
|
||||
PaginationItem.displayName = "PaginationItem"
|
||||
|
||||
type PaginationLinkProps = {
|
||||
isActive?: boolean
|
||||
} & Pick<ButtonProps, "size"> &
|
||||
React.ComponentProps<"a">
|
||||
|
||||
const PaginationLink = ({
|
||||
className,
|
||||
isActive,
|
||||
size = "icon",
|
||||
...props
|
||||
}: PaginationLinkProps) => (
|
||||
<a
|
||||
aria-current={isActive ? "page" : undefined}
|
||||
className={cn(
|
||||
buttonVariants({
|
||||
variant: isActive ? "outline" : "ghost",
|
||||
size,
|
||||
}),
|
||||
"no-underline",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
/>
|
||||
)
|
||||
PaginationLink.displayName = "PaginationLink"
|
||||
|
||||
const PaginationPrevious = ({
|
||||
className,
|
||||
...props
|
||||
}: React.ComponentProps<typeof PaginationLink>) => (
|
||||
<PaginationLink
|
||||
aria-label="Go to previous page"
|
||||
size="default"
|
||||
className={cn("gap-1 pl-2.5", className)}
|
||||
{...props}
|
||||
>
|
||||
<ChevronLeft className="h-4 w-4" />
|
||||
<span>Previous</span>
|
||||
</PaginationLink>
|
||||
)
|
||||
PaginationPrevious.displayName = "PaginationPrevious"
|
||||
|
||||
const PaginationNext = ({
|
||||
className,
|
||||
...props
|
||||
}: React.ComponentProps<typeof PaginationLink>) => (
|
||||
<PaginationLink
|
||||
aria-label="Go to next page"
|
||||
size="default"
|
||||
className={cn("gap-1 pr-2.5", className)}
|
||||
{...props}
|
||||
>
|
||||
<span>Next</span>
|
||||
<ChevronRight className="h-4 w-4" />
|
||||
</PaginationLink>
|
||||
)
|
||||
PaginationNext.displayName = "PaginationNext"
|
||||
|
||||
const PaginationEllipsis = ({
|
||||
className,
|
||||
...props
|
||||
}: React.ComponentProps<"span">) => (
|
||||
<span
|
||||
aria-hidden
|
||||
className={cn("flex h-9 w-9 items-center justify-center", className)}
|
||||
{...props}
|
||||
>
|
||||
<MoreHorizontal className="h-4 w-4" />
|
||||
<span className="sr-only">More pages</span>
|
||||
</span>
|
||||
)
|
||||
PaginationEllipsis.displayName = "PaginationEllipsis"
|
||||
|
||||
export {
|
||||
Pagination,
|
||||
PaginationContent,
|
||||
PaginationEllipsis,
|
||||
PaginationItem,
|
||||
PaginationLink,
|
||||
PaginationNext,
|
||||
PaginationPrevious,
|
||||
}
|
||||
@@ -1485,9 +1485,16 @@ class FileObjectAdapters:
|
||||
return await sync_to_async(list)(FileObject.objects.filter(user=user, file_name__in=file_names))
|
||||
|
||||
@staticmethod
|
||||
@arequire_valid_user
|
||||
async def aget_all_file_objects(user: KhojUser):
|
||||
return await sync_to_async(list)(FileObject.objects.filter(user=user).order_by("-updated_at"))
|
||||
@require_valid_user
|
||||
async def aget_all_file_objects(user: KhojUser, start: int = 0, limit: int = 10):
|
||||
query = FileObject.objects.filter(user=user).order_by("-updated_at")[start : start + limit]
|
||||
return await sync_to_async(list)(query)
|
||||
|
||||
@staticmethod
|
||||
@require_valid_user
|
||||
async def aget_number_of_pages(user: KhojUser, limit: int = 10):
|
||||
count = await FileObject.objects.filter(user=user).acount()
|
||||
return math.ceil(count / limit)
|
||||
|
||||
@staticmethod
|
||||
@arequire_valid_user
|
||||
|
||||
@@ -332,7 +332,9 @@ def get_content_types(request: Request, client: Optional[str] = None):
|
||||
|
||||
@api_content.get("/all", response_model=Dict[str, str])
|
||||
@requires(["authenticated"])
|
||||
async def get_all_content(request: Request, client: Optional[str] = None, truncated: Optional[bool] = True):
|
||||
async def get_all_content(
|
||||
request: Request, client: Optional[str] = None, truncated: Optional[bool] = True, page: int = 0
|
||||
):
|
||||
user = request.user.object
|
||||
|
||||
update_telemetry_state(
|
||||
@@ -343,7 +345,12 @@ async def get_all_content(request: Request, client: Optional[str] = None, trunca
|
||||
)
|
||||
|
||||
files_data = []
|
||||
file_objects = await FileObjectAdapters.aget_all_file_objects(user)
|
||||
page_size = 1
|
||||
|
||||
file_objects = await FileObjectAdapters.aget_all_file_objects(user, start=page * page_size, limit=page_size)
|
||||
|
||||
num_pages = await FileObjectAdapters.aget_number_of_pages(user, page_size)
|
||||
|
||||
for file_object in file_objects:
|
||||
files_data.append(
|
||||
{
|
||||
@@ -353,7 +360,12 @@ async def get_all_content(request: Request, client: Optional[str] = None, trunca
|
||||
}
|
||||
)
|
||||
|
||||
return Response(content=json.dumps(files_data), media_type="application/json", status_code=200)
|
||||
data_packet = {
|
||||
"files": files_data,
|
||||
"num_pages": num_pages,
|
||||
}
|
||||
|
||||
return Response(content=json.dumps(data_packet), media_type="application/json", status_code=200)
|
||||
|
||||
|
||||
@api_content.get("/file", response_model=Dict[str, str])
|
||||
|
||||
Reference in New Issue
Block a user