Make search filters return entry ids satisfying filter

- Filter entries, embeddings by ids satisfying all filters in query
  func, after each filter has returned entry ids satisfying their
  individual acceptance criteria

- Previously each filter would return a filtered list of entries.
  Each filter would be applied on entries filtered by previous filters.
  This made the filtering order dependent

- Benefits
  - Filters can be applied independent of their order of execution
  - Precomputed indexes for each filter is not in danger of running
    into index out of bound errors, as filters run on original entries
    instead of on entries filtered by filters that have run before it
  - Extract entries satisfying filter only once instead of doing
    this for each filter

- Costs
  - Each filter has to process all entries even if previous filters
    may have already marked them as non-satisfactory

tests/test_date_filter.py

@@ -18,40 +18,34 @@ def test_date_filter():
         {'compiled': '', 'raw': 'Entry with date:1984-04-02'}]
 
     q_with_no_date_filter = 'head tail'
-    ret_query, ret_entries, ret_emb = DateFilter().apply(q_with_no_date_filter, entries.copy(), embeddings)
+    ret_query, entry_indices = DateFilter().apply(q_with_no_date_filter, entries, embeddings)
     assert ret_query == 'head tail'
-    assert len(ret_emb) == 3
-    assert ret_entries == entries
+    assert entry_indices == {0, 1, 2}
 
     q_with_dtrange_non_overlapping_at_boundary = 'head dt>"1984-04-01" dt<"1984-04-02" tail'
-    ret_query, ret_entries, ret_emb = DateFilter().apply(q_with_dtrange_non_overlapping_at_boundary, entries.copy(), embeddings)
+    ret_query, entry_indices = DateFilter().apply(q_with_dtrange_non_overlapping_at_boundary, entries, embeddings)
     assert ret_query == 'head tail'
-    assert len(ret_emb) == 0
-    assert ret_entries == []
+    assert entry_indices == set()
 
     query_with_overlapping_dtrange = 'head dt>"1984-04-01" dt<"1984-04-03" tail'
-    ret_query, ret_entries, ret_emb = DateFilter().apply(query_with_overlapping_dtrange, entries.copy(), embeddings)
+    ret_query, entry_indices = DateFilter().apply(query_with_overlapping_dtrange, entries, embeddings)
     assert ret_query == 'head tail'
-    assert ret_entries == [entries[2]]
-    assert len(ret_emb) == 1
+    assert entry_indices == {2}
 
     query_with_overlapping_dtrange = 'head dt>="1984-04-01" dt<"1984-04-02" tail'
-    ret_query, ret_entries, ret_emb = DateFilter().apply(query_with_overlapping_dtrange, entries.copy(), embeddings)
+    ret_query, entry_indices = DateFilter().apply(query_with_overlapping_dtrange, entries, embeddings)
     assert ret_query == 'head tail'
-    assert ret_entries == [entries[1]]
-    assert len(ret_emb) == 1
+    assert entry_indices == {1}
 
     query_with_overlapping_dtrange = 'head dt>"1984-04-01" dt<="1984-04-02" tail'
-    ret_query, ret_entries, ret_emb = DateFilter().apply(query_with_overlapping_dtrange, entries.copy(), embeddings)
+    ret_query, entry_indices = DateFilter().apply(query_with_overlapping_dtrange, entries, embeddings)
     assert ret_query == 'head tail'
-    assert ret_entries == [entries[2]]
-    assert len(ret_emb) == 1
+    assert entry_indices == {2}
 
     query_with_overlapping_dtrange = 'head dt>="1984-04-01" dt<="1984-04-02" tail'
-    ret_query, ret_entries, ret_emb = DateFilter().apply(query_with_overlapping_dtrange, entries.copy(), embeddings)
+    ret_query, entry_indices = DateFilter().apply(query_with_overlapping_dtrange, entries, embeddings)
     assert ret_query == 'head tail'
-    assert ret_entries == [entries[1], entries[2]]
-    assert len(ret_emb) == 2
+    assert entry_indices == {1, 2}
 
 
 def test_extract_date_range():

tests/test_file_filter.py

@@ -13,13 +13,12 @@ def test_no_file_filter():
 
     # Act
     can_filter = file_filter.can_filter(q_with_no_filter)
-    ret_query, ret_entries, ret_emb = file_filter.apply(q_with_no_filter, entries.copy(), embeddings)
+    ret_query, entry_indices = file_filter.apply(q_with_no_filter, entries.copy(), embeddings)
 
     # Assert
     assert can_filter == False
     assert ret_query == 'head tail'
-    assert len(ret_emb) == 4
-    assert ret_entries == entries
+    assert entry_indices == {0, 1, 2, 3}
 
 
 def test_file_filter_with_non_existent_file():
@@ -30,13 +29,12 @@ def test_file_filter_with_non_existent_file():
 
     # Act
     can_filter = file_filter.can_filter(q_with_no_filter)
-    ret_query, ret_entries, ret_emb = file_filter.apply(q_with_no_filter, entries.copy(), embeddings)
+    ret_query, entry_indices = file_filter.apply(q_with_no_filter, entries.copy(), embeddings)
 
     # Assert
     assert can_filter == True
     assert ret_query == 'head tail'
-    assert len(ret_emb) == 0
-    assert ret_entries == []
+    assert entry_indices == {}
 
 
 def test_single_file_filter():
@@ -47,13 +45,12 @@ def test_single_file_filter():
 
     # Act
     can_filter = file_filter.can_filter(q_with_no_filter)
-    ret_query, ret_entries, ret_emb = file_filter.apply(q_with_no_filter, entries.copy(), embeddings)
+    ret_query, entry_indices = file_filter.apply(q_with_no_filter, entries.copy(), embeddings)
 
     # Assert
     assert can_filter == True
     assert ret_query == 'head tail'
-    assert len(ret_emb) == 2
-    assert ret_entries == [entries[0], entries[2]]
+    assert entry_indices == {0, 2}
 
 
 def test_file_filter_with_partial_match():
@@ -64,13 +61,12 @@ def test_file_filter_with_partial_match():
 
     # Act
     can_filter = file_filter.can_filter(q_with_no_filter)
-    ret_query, ret_entries, ret_emb = file_filter.apply(q_with_no_filter, entries.copy(), embeddings)
+    ret_query, entry_indices = file_filter.apply(q_with_no_filter, entries.copy(), embeddings)
 
     # Assert
     assert can_filter == True
     assert ret_query == 'head tail'
-    assert len(ret_emb) == 2
-    assert ret_entries == [entries[0], entries[2]]
+    assert entry_indices == {0, 2}
 
 
 def test_file_filter_with_regex_match():
@@ -81,13 +77,12 @@ def test_file_filter_with_regex_match():
 
     # Act
     can_filter = file_filter.can_filter(q_with_no_filter)
-    ret_query, ret_entries, ret_emb = file_filter.apply(q_with_no_filter, entries.copy(), embeddings)
+    ret_query, entry_indices = file_filter.apply(q_with_no_filter, entries.copy(), embeddings)
 
     # Assert
     assert can_filter == True
     assert ret_query == 'head tail'
-    assert len(ret_emb) == 4
-    assert ret_entries == entries
+    assert entry_indices == {0, 1, 2, 3}
 
 
 def test_multiple_file_filter():
@@ -98,13 +93,12 @@ def test_multiple_file_filter():
 
     # Act
     can_filter = file_filter.can_filter(q_with_no_filter)
-    ret_query, ret_entries, ret_emb = file_filter.apply(q_with_no_filter, entries.copy(), embeddings)
+    ret_query, entry_indices = file_filter.apply(q_with_no_filter, entries.copy(), embeddings)
 
     # Assert
     assert can_filter == True
     assert ret_query == 'head tail'
-    assert len(ret_emb) == 4
-    assert ret_entries == entries
+    assert entry_indices == {0, 1, 2, 3}
 
 
 def arrange_content():

tests/test_word_filter.py

@@ -14,13 +14,12 @@ def test_no_word_filter(tmp_path):
 
     # Act
     can_filter = word_filter.can_filter(q_with_no_filter)
-    ret_query, ret_entries, ret_emb = word_filter.apply(q_with_no_filter, entries.copy(), embeddings)
+    ret_query, entry_indices = word_filter.apply(q_with_no_filter, entries.copy(), embeddings)
 
     # Assert
     assert can_filter == False
     assert ret_query == 'head tail'
-    assert len(ret_emb) == 4
-    assert ret_entries == entries
+    assert entry_indices == {0, 1, 2, 3}
 
 
 def test_word_exclude_filter(tmp_path):
@@ -31,13 +30,12 @@ def test_word_exclude_filter(tmp_path):
 
     # Act
     can_filter = word_filter.can_filter(q_with_exclude_filter)
-    ret_query, ret_entries, ret_emb = word_filter.apply(q_with_exclude_filter, entries.copy(), embeddings)
+    ret_query, entry_indices = word_filter.apply(q_with_exclude_filter, entries.copy(), embeddings)
 
     # Assert
     assert can_filter == True
     assert ret_query == 'head tail'
-    assert len(ret_emb) == 2
-    assert ret_entries == [entries[0], entries[2]]
+    assert entry_indices == {0, 2}
 
 
 def test_word_include_filter(tmp_path):
@@ -48,13 +46,12 @@ def test_word_include_filter(tmp_path):
 
     # Act
     can_filter = word_filter.can_filter(query_with_include_filter)
-    ret_query, ret_entries, ret_emb = word_filter.apply(query_with_include_filter, entries.copy(), embeddings)
+    ret_query, entry_indices = word_filter.apply(query_with_include_filter, entries.copy(), embeddings)
 
     # Assert
     assert can_filter == True
     assert ret_query == 'head tail'
-    assert len(ret_emb) == 2
-    assert ret_entries == [entries[2], entries[3]]
+    assert entry_indices == {2, 3}
 
 
 def test_word_include_and_exclude_filter(tmp_path):
@@ -65,13 +62,12 @@ def test_word_include_and_exclude_filter(tmp_path):
 
     # Act
     can_filter = word_filter.can_filter(query_with_include_and_exclude_filter)
-    ret_query, ret_entries, ret_emb = word_filter.apply(query_with_include_and_exclude_filter, entries.copy(), embeddings)
+    ret_query, entry_indices = word_filter.apply(query_with_include_and_exclude_filter, entries.copy(), embeddings)
 
     # Assert
     assert can_filter == True
     assert ret_query == 'head tail'
-    assert len(ret_emb) == 1
-    assert ret_entries == [entries[2]]
+    assert entry_indices == {2}
 
 
 def arrange_content():