Support multiple input-filters to configure content to index via khoj.yml

- Update existings code, tests to process input-filters as list
  instead of str
- Test `text_to_jsonl' get files methods to work with combination of
  `input-files' and `input-filters'

Resolves #84

tests/conftest.py

@@ -55,7 +55,7 @@ def content_config(tmp_path_factory, search_config: SearchConfig):
     # Generate Notes Embeddings from Test Notes
     content_config.org = TextContentConfig(
         input_files = None,
-        input_filter = 'tests/data/org/*.org',
+        input_filter = ['tests/data/org/*.org'],
         compressed_jsonl = content_dir.joinpath('notes.jsonl.gz'),
         embeddings_file = content_dir.joinpath('note_embeddings.pt'))
 

tests/data/config.yml

@@ -1,9 +1,10 @@
 content-type:
   org:
     input-files: [ "~/first_from_config.org", "~/second_from_config.org" ]
-    input-filter: "*.org"
+    input-filter: ["*.org", "~/notes/*.org"]
     compressed-jsonl: ".notes.json.gz"
     embeddings-file: ".note_embeddings.pt"
+    index-header-entries: true
 
 search-type:
   asymmetric:

tests/test_beancount_to_jsonl.py

@@ -2,7 +2,7 @@
 import json
 
 # Internal Packages
-from src.processor.ledger.beancount_to_jsonl import extract_beancount_transactions, convert_transactions_to_maps, convert_transaction_maps_to_jsonl
+from src.processor.ledger.beancount_to_jsonl import extract_beancount_transactions, convert_transactions_to_maps, convert_transaction_maps_to_jsonl, get_beancount_files
 
 
 def test_no_transactions_in_file(tmp_path):
@@ -75,10 +75,38 @@ Assets:Test:Test  -1.00 KES
     assert len(jsonl_data) == 2
 
 
+def test_get_beancount_files(tmp_path):
+    "Ensure Beancount files specified via input-filter, input-files extracted"
+    # Arrange
+    # Include via input-filter globs
+    group1_file1 = create_file(tmp_path, filename="group1-file1.bean")
+    group1_file2 = create_file(tmp_path, filename="group1-file2.bean")
+    group2_file1 = create_file(tmp_path, filename="group2-file1.beancount")
+    group2_file2 = create_file(tmp_path, filename="group2-file2.beancount")
+    # Include via input-file field
+    file1 = create_file(tmp_path, filename="ledger.bean")
+    # Not included by any filter
+    create_file(tmp_path, filename="not-included-ledger.bean")
+    create_file(tmp_path, filename="not-included-text.txt")
+
+    expected_files = set(map(str, [group1_file1, group1_file2, group2_file1, group2_file2, file1]))
+
+    # Setup input-files, input-filters
+    input_files = [tmp_path / 'ledger.bean']
+    input_filter = [tmp_path / 'group1*.bean', tmp_path / 'group2*.beancount']
+
+    # Act
+    extracted_org_files = get_beancount_files(input_files, input_filter)
+
+    # Assert
+    assert len(extracted_org_files) == 5
+    assert extracted_org_files == expected_files
+
+
 # Helper Functions
-def create_file(tmp_path, entry, filename="ledger.beancount"):
-    beancount_file = tmp_path / f"notes/{filename}"
-    beancount_file.parent.mkdir()
+def create_file(tmp_path, entry=None, filename="ledger.beancount"):
+    beancount_file = tmp_path / filename
     beancount_file.touch()
-    beancount_file.write_text(entry)
+    if entry:
+        beancount_file.write_text(entry)
     return beancount_file

tests/test_markdown_to_jsonl.py

@@ -2,7 +2,7 @@
 import json
 
 # Internal Packages
-from src.processor.markdown.markdown_to_jsonl import extract_markdown_entries, convert_markdown_maps_to_jsonl, convert_markdown_entries_to_maps
+from src.processor.markdown.markdown_to_jsonl import extract_markdown_entries, convert_markdown_maps_to_jsonl, convert_markdown_entries_to_maps, get_markdown_files
 
 
 def test_markdown_file_with_no_headings_to_jsonl(tmp_path):
@@ -72,10 +72,38 @@ def test_multiple_markdown_entries_to_jsonl(tmp_path):
     assert len(jsonl_data) == 2
 
 
+def test_get_markdown_files(tmp_path):
+    "Ensure Markdown files specified via input-filter, input-files extracted"
+    # Arrange
+    # Include via input-filter globs
+    group1_file1 = create_file(tmp_path, filename="group1-file1.md")
+    group1_file2 = create_file(tmp_path, filename="group1-file2.md")
+    group2_file1 = create_file(tmp_path, filename="group2-file1.markdown")
+    group2_file2 = create_file(tmp_path, filename="group2-file2.markdown")
+    # Include via input-file field
+    file1 = create_file(tmp_path, filename="notes.md")
+    # Not included by any filter
+    create_file(tmp_path, filename="not-included-markdown.md")
+    create_file(tmp_path, filename="not-included-text.txt")
+
+    expected_files = set(map(str, [group1_file1, group1_file2, group2_file1, group2_file2, file1]))
+
+    # Setup input-files, input-filters
+    input_files = [tmp_path / 'notes.md']
+    input_filter = [tmp_path / 'group1*.md', tmp_path / 'group2*.markdown']
+
+    # Act
+    extracted_org_files = get_markdown_files(input_files, input_filter)
+
+    # Assert
+    assert len(extracted_org_files) == 5
+    assert extracted_org_files == expected_files
+
+
 # Helper Functions
-def create_file(tmp_path, entry, filename="test.md"):
-    markdown_file = tmp_path / f"notes/{filename}"
-    markdown_file.parent.mkdir()
+def create_file(tmp_path, entry=None, filename="test.md"):
+    markdown_file = tmp_path / filename
     markdown_file.touch()
-    markdown_file.write_text(entry)
+    if entry:
+        markdown_file.write_text(entry)
     return markdown_file

tests/test_org_to_jsonl.py

@@ -2,7 +2,7 @@
 import json
 
 # Internal Packages
-from src.processor.org_mode.org_to_jsonl import convert_org_entries_to_jsonl, convert_org_nodes_to_entries, extract_org_entries
+from src.processor.org_mode.org_to_jsonl import convert_org_entries_to_jsonl, convert_org_nodes_to_entries, extract_org_entries, get_org_files
 from src.utils.helpers import is_none_or_empty
 
 
@@ -81,10 +81,38 @@ def test_file_with_no_headings_to_jsonl(tmp_path):
     assert len(jsonl_data) == 1
 
 
+def test_get_org_files(tmp_path):
+    "Ensure Org files specified via input-filter, input-files extracted"
+    # Arrange
+    # Include via input-filter globs
+    group1_file1 = create_file(tmp_path, filename="group1-file1.org")
+    group1_file2 = create_file(tmp_path, filename="group1-file2.org")
+    group2_file1 = create_file(tmp_path, filename="group2-file1.org")
+    group2_file2 = create_file(tmp_path, filename="group2-file2.org")
+    # Include via input-file field
+    orgfile1 = create_file(tmp_path, filename="orgfile1.org")
+    # Not included by any filter
+    create_file(tmp_path, filename="orgfile2.org")
+    create_file(tmp_path, filename="text1.txt")
+
+    expected_files = set(map(str, [group1_file1, group1_file2, group2_file1, group2_file2, orgfile1]))
+
+    # Setup input-files, input-filters
+    input_files = [tmp_path / 'orgfile1.org']
+    input_filter = [tmp_path / 'group1*.org', tmp_path / 'group2*.org']
+
+    # Act
+    extracted_org_files = get_org_files(input_files, input_filter)
+
+    # Assert
+    assert len(extracted_org_files) == 5
+    assert extracted_org_files == expected_files
+
+
 # Helper Functions
-def create_file(tmp_path, entry, filename="test.org"):
-    org_file = tmp_path / f"notes/{filename}"
-    org_file.parent.mkdir()
+def create_file(tmp_path, entry=None, filename="test.org"):
+    org_file = tmp_path / filename
     org_file.touch()
-    org_file.write_text(entry)
-    return org_file
+    if entry:
+        org_file.write_text(entry)
+    return org_file

tests/test_text_search.py

@@ -16,7 +16,7 @@ from src.processor.org_mode.org_to_jsonl import org_to_jsonl
 # ----------------------------------------------------------------------------------------------------
 def test_asymmetric_setup_with_missing_file_raises_error(content_config: ContentConfig, search_config: SearchConfig):
     # Arrange
-    file_to_index = Path(content_config.org.input_filter).parent / "new_file_to_index.org"
+    file_to_index = Path(content_config.org.input_filter[0]).parent / "new_file_to_index.org"
     new_org_content_config = deepcopy(content_config.org)
     new_org_content_config.input_files = [f'{file_to_index}']
     new_org_content_config.input_filter = None
@@ -30,7 +30,7 @@ def test_asymmetric_setup_with_missing_file_raises_error(content_config: Content
 # ----------------------------------------------------------------------------------------------------
 def test_asymmetric_setup_with_empty_file_raises_error(content_config: ContentConfig, search_config: SearchConfig):
     # Arrange
-    file_to_index = Path(content_config.org.input_filter).parent / "new_file_to_index.org"
+    file_to_index = Path(content_config.org.input_filter[0]).parent / "new_file_to_index.org"
     file_to_index.touch()
     new_org_content_config = deepcopy(content_config.org)
     new_org_content_config.input_files = [f'{file_to_index}']
@@ -88,7 +88,7 @@ def test_asymmetric_reload(content_config: ContentConfig, search_config: SearchC
     assert len(initial_notes_model.entries) == 10
     assert len(initial_notes_model.corpus_embeddings) == 10
 
-    file_to_add_on_reload = Path(content_config.org.input_filter).parent / "reload.org"
+    file_to_add_on_reload = Path(content_config.org.input_filter[0]).parent / "reload.org"
     content_config.org.input_files = [f'{file_to_add_on_reload}']
 
     # append Org-Mode Entry to first Org Input File in Config
@@ -124,7 +124,7 @@ def test_incremental_update(content_config: ContentConfig, search_config: Search
     assert len(initial_notes_model.entries) == 10
     assert len(initial_notes_model.corpus_embeddings) == 10
 
-    file_to_add_on_update = Path(content_config.org.input_filter).parent / "update.org"
+    file_to_add_on_update = Path(content_config.org.input_filter[0]).parent / "update.org"
     content_config.org.input_files = [f'{file_to_add_on_update}']
 
     # append Org-Mode Entry to first Org Input File in Config