klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-09 21:29:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Skip indexing a PDF that has an indexing error (#274)

Browse Source
This commit is contained in:
sabaimran
2023-07-03 15:55:11 -07:00
committed by GitHub
parent a6f313589e
commit 017e8c1aef
1 changed files with 8 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/khoj/processor/pdf/pdf_to_jsonl.py
View File

@@ -98,10 +98,14 @@ class PdfToJsonl(TextToJsonl):
entries = [] entries = []
entry_to_location_map = [] entry_to_location_map = []
for pdf_file in pdf_files: for pdf_file in pdf_files:
loader = PyPDFLoader(pdf_file) try:
pdf_entries_per_file = [page.page_content for page in loader.load()] loader = PyPDFLoader(pdf_file)
entry_to_location_map += zip(pdf_entries_per_file, [pdf_file] * len(pdf_entries_per_file)) pdf_entries_per_file = [page.page_content for page in loader.load()]
entries.extend(pdf_entries_per_file) entry_to_location_map += zip(pdf_entries_per_file, [pdf_file] * len(pdf_entries_per_file))
entries.extend(pdf_entries_per_file)
except Exception as e:
logger.error(f"Error processing file: {pdf_file}. This file will not be indexed.")
logger.error(e)
return entries, dict(entry_to_location_map) return entries, dict(entry_to_location_map)