klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-08 05:39:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Skip plaintext file indexing if there's a parsing issue and log the file

Browse Source
This commit is contained in:
sabaimran
2023-08-29 14:34:08 -07:00
parent 74409c2c64
commit 92cbfef7ab
1 changed files with 7 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/khoj/processor/plaintext/plaintext_to_jsonl.py
View File

@@ -91,10 +91,13 @@ class PlaintextToJsonl(TextToJsonl):
for plaintext_file in plaintext_files: for plaintext_file in plaintext_files:
with open(plaintext_file, "r") as f: with open(plaintext_file, "r") as f:
try:
plaintext_content = f.read() plaintext_content = f.read()
if plaintext_file.endswith(("html", "htm", "xml")): if plaintext_file.endswith(("html", "htm", "xml")):
plaintext_content = PlaintextToJsonl.extract_html_content(plaintext_content) plaintext_content = PlaintextToJsonl.extract_html_content(plaintext_content)
entry_to_file_map.append((plaintext_content, plaintext_file)) entry_to_file_map.append((plaintext_content, plaintext_file))
except Exception as e:
logger.error(f"Error processing file: {plaintext_file} - {e}", exc_info=True)
return dict(entry_to_file_map) return dict(entry_to_file_map)