klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-06 05:39:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Test index is stable sorted on incremental update with new entry

Browse Source 
Ensure order of new embedding, entry insertion on incremental update
is stable
This commit is contained in:
Debanjum Singh Solanky
2023-07-16 01:24:03 -07:00
parent b02323ade6
commit 1482fd4d4d
1 changed files with 11 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
tests/test_text_search.py
View File

@@ -199,24 +199,27 @@ def test_update_index_with_new_entry(content_config: ContentConfig, search_model
OrgToJsonl, content_config.org, search_models.text_search.bi_encoder, regenerate=True OrgToJsonl, content_config.org, search_models.text_search.bi_encoder, regenerate=True
) )
assert len(initial_notes_model.entries) == 10
assert len(initial_notes_model.corpus_embeddings) == 10
# append org-mode entry to first org input file in config # append org-mode entry to first org input file in config
with open(new_org_file, "w") as f: with open(new_org_file, "w") as f:
f.write("\n* A Chihuahua doing Tango\n- Saw a super cute video of a chihuahua doing the Tango on Youtube\n") new_entry = "\n* A Chihuahua doing Tango\n- Saw a super cute video of a chihuahua doing the Tango on Youtube\n"
f.write(new_entry)
# Act # Act
# update embeddings, entries with the newly added note # update embeddings, entries with the newly added note
content_config.org.input_files = [f"{new_org_file}"] content_config.org.input_files = [f"{new_org_file}"]
initial_notes_model = text_search.setup( final_notes_model = text_search.setup(
OrgToJsonl, content_config.org, search_models.text_search.bi_encoder, regenerate=False OrgToJsonl, content_config.org, search_models.text_search.bi_encoder, regenerate=False
) )
# Assert # Assert
# verify new entry added in updated embeddings, entries assert len(final_notes_model.entries) == len(initial_notes_model.entries) + 1
assert len(initial_notes_model.entries) == 11 assert len(final_notes_model.corpus_embeddings) == len(initial_notes_model.corpus_embeddings) + 1
assert len(initial_notes_model.corpus_embeddings) == 11
# verify new entry appended to index, without disrupting order or content of existing entries
error_details = compare_index(initial_notes_model, final_notes_model)
if error_details:
# fails at embeddings index 4, 7. These are not swapped with the new entry embedding or each other
pytest.fail(error_details, False)
# Cleanup # Cleanup
# reset input_files in config to empty list # reset input_files in config to empty list