klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-07 21:29:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Skip indexing single bad markdown, plaintext file (#460)

Browse Source
This commit is contained in:
sabaimran
2023-08-23 15:34:56 -07:00
committed by GitHub
parent 84bd579077
commit e64357698d
3 changed files with 16 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/khoj/processor/markdown/markdown_to_jsonl.py
View File

@@ -105,10 +105,14 @@ class MarkdownToJsonl(TextToJsonl):
entry_to_file_map = [] entry_to_file_map = []
for markdown_file in markdown_files: for markdown_file in markdown_files:
with open(markdown_file, "r", encoding="utf8") as f: with open(markdown_file, "r", encoding="utf8") as f:
try:
markdown_content = f.read() markdown_content = f.read()
entries, entry_to_file_map = MarkdownToJsonl.process_single_markdown_file( entries, entry_to_file_map = MarkdownToJsonl.process_single_markdown_file(
markdown_content, markdown_file, entries, entry_to_file_map markdown_content, markdown_file, entries, entry_to_file_map
) )
except Exception as e:
logger.warning(f"Unable to process file: {markdown_file}. This file will not be indexed.")
logger.warning(e, exc_info=True)
return entries, dict(entry_to_file_map) return entries, dict(entry_to_file_map)

3
src/khoj/processor/org_mode/org_to_jsonl.py
View File

@@ -100,7 +100,8 @@ class OrgToJsonl(TextToJsonl):
entry_to_file_map += zip(org_file_entries, [org_file] * len(org_file_entries)) entry_to_file_map += zip(org_file_entries, [org_file] * len(org_file_entries))
entries.extend(org_file_entries) entries.extend(org_file_entries)
except Exception as e: except Exception as e:
logger.error(f"Error processing file: {org_file} with error: {e}", exc_info=True) logger.warning(f"Unable to process file: {org_file}. This file will not be indexed.")
logger.warning(e, exc_info=True)
return entries, dict(entry_to_file_map) return entries, dict(entry_to_file_map)

4
src/khoj/processor/plaintext/plaintext_to_jsonl.py
View File

@@ -91,8 +91,12 @@ class PlaintextToJsonl(TextToJsonl):
for plaintext_file in plaintext_files: for plaintext_file in plaintext_files:
with open(plaintext_file, "r") as f: with open(plaintext_file, "r") as f:
try:
plaintext_content = f.read() plaintext_content = f.read()
entry_to_file_map.append((plaintext_content, plaintext_file)) entry_to_file_map.append((plaintext_content, plaintext_file))
except Exception as e:
logger.warning(f"Unable to process file: {plaintext_file}. This file will not be indexed.")
logger.warning(e, exc_info=True)
return dict(entry_to_file_map) return dict(entry_to_file_map)