klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-08 05:39:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Log error when fail to index md file. Fix, improve typing in md_to_entries

Browse Source
This commit is contained in:
Debanjum Singh Solanky
2024-02-11 20:40:31 +05:30
parent 44eab74888
commit 2ea8a832a0
1 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/khoj/processor/content/markdown/markdown_to_entries.py
View File

@@ -58,7 +58,7 @@ class MarkdownToEntries(TextToEntries):
def extract_markdown_entries(markdown_files, max_tokens=256) -> List[Entry]: def extract_markdown_entries(markdown_files, max_tokens=256) -> List[Entry]:
"Extract entries by heading from specified Markdown files" "Extract entries by heading from specified Markdown files"
entries: List[str] = [] entries: List[str] = []
entry_to_file_map: List[Tuple[str, Path]] = [] entry_to_file_map: List[Tuple[str, str]] = []
for markdown_file in markdown_files: for markdown_file in markdown_files:
try: try:
markdown_content = markdown_files[markdown_file] markdown_content = markdown_files[markdown_file]
@@ -66,7 +66,7 @@ class MarkdownToEntries(TextToEntries):
markdown_content, markdown_file, entries, entry_to_file_map, max_tokens markdown_content, markdown_file, entries, entry_to_file_map, max_tokens
) )
except Exception as e: except Exception as e:
logger.warning( logger.error(
f"Unable to process file: {markdown_file}. This file will not be indexed.\n{e}", exc_info=True f"Unable to process file: {markdown_file}. This file will not be indexed.\n{e}", exc_info=True
) )
@@ -75,12 +75,12 @@ class MarkdownToEntries(TextToEntries):
@staticmethod @staticmethod
def process_single_markdown_file( def process_single_markdown_file(
markdown_content: str, markdown_content: str,
markdown_file: Path, markdown_file: str,
entries: List[str], entries: List[str],
entry_to_file_map: List[Tuple[str, Path]], entry_to_file_map: List[Tuple[str, str]],
max_tokens=256, max_tokens=256,
ancestry: Dict[int, str] = {}, ancestry: Dict[int, str] = {},
): ) -> Tuple[List[str], List[Tuple[str, str]]]:
# Prepend the markdown section's heading ancestry # Prepend the markdown section's heading ancestry
ancestry_string = "\n".join([f"{'#' * key} {ancestry[key]}" for key in sorted(ancestry.keys())]) ancestry_string = "\n".join([f"{'#' * key} {ancestry[key]}" for key in sorted(ancestry.keys())])
markdown_content_with_ancestry = f"{ancestry_string}{markdown_content}" markdown_content_with_ancestry = f"{ancestry_string}{markdown_content}"