klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-06 05:39:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Process text content files in sorted order for stable indexing

Browse Source 
- Image search already uses a sorted list of images to process
- Prevents index of entries to desync when entries, embeddings
  generated by a separate server/app instance
This commit is contained in:
Debanjum Singh Solanky
2022-09-12 11:02:05 +03:00
parent a701ad08b9
commit 536f03af8f
6 changed files with 12 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/processor/ledger/beancount_to_jsonl.py
View File

@@ -77,12 +77,14 @@ def get_beancount_files(beancount_files=None, beancount_file_filters=None):
for filtered_file in glob.glob(get_absolute_path(beancount_file_filter))
}
all_beancount_files = absolute_beancount_files | filtered_beancount_files
all_beancount_files = sorted(absolute_beancount_files | filtered_beancount_files)
files_with_non_beancount_extensions = {beancount_file
for beancount_file
in all_beancount_files
if not beancount_file.endswith(".bean") and not beancount_file.endswith(".beancount")}
files_with_non_beancount_extensions = {
beancount_file
for beancount_file
in all_beancount_files
if not beancount_file.endswith(".bean") and not beancount_file.endswith(".beancount")
}
if any(files_with_non_beancount_extensions):
print(f"[Warning] There maybe non beancount files in the input set: {files_with_non_beancount_extensions}")