klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-02 13:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rename RawConfig Types for Consistency

Browse Source 
- Naming convention - [ContentType][ConfigType]Config
  - Where [ConfigType] ~ Content, Search, Processor
  - Where [ContentType] ~ Text, Image, Asymmetric, Symmetric, Conversation

- Current Configs:
  - Content:
    - Org Notes
    - Org Music
    - Image
    - Ledger/Beancount

  - Search:
     - Asymmetric
     - Symmetric
     - Image

  - Processor:
    - Conversation
This commit is contained in:
Debanjum Singh Solanky
2022-01-14 20:54:38 -05:00
parent ed7c2901f5
commit 179153dc5a
8 changed files with 62 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
tests/conftest.py
View File

@@ -1,32 +1,30 @@
# Standard Packages
import pytest
from pathlib import Path
from src import search_type
# Internal Packages
from src.search_type import asymmetric, image_search
from src.utils.rawconfig import AsymmetricConfig, ContentTypeConfig, ImageSearchConfig, ImageSearchTypeConfig, SearchTypeConfig, SymmetricConfig, TextSearchConfig
from src.utils.rawconfig import ContentConfig, TextContentConfig, ImageContentConfig, SearchConfig, SymmetricSearchConfig, AsymmetricSearchConfig, ImageSearchConfig
@pytest.fixture(scope='session')
def search_config(tmp_path_factory):
model_dir = tmp_path_factory.mktemp('data')
search_config = SearchTypeConfig()
search_config = SearchConfig()
search_config.asymmetric = SymmetricConfig(
search_config.asymmetric = SymmetricSearchConfig(
encoder = "sentence-transformers/paraphrase-MiniLM-L6-v2",
cross_encoder = "cross-encoder/ms-marco-MiniLM-L-6-v2",
model_directory = model_dir
)
search_config.asymmetric = AsymmetricConfig(
search_config.asymmetric = AsymmetricSearchConfig(
encoder = "sentence-transformers/msmarco-MiniLM-L-6-v3",
cross_encoder = "cross-encoder/ms-marco-MiniLM-L-6-v2",
model_directory = model_dir
)
search_config.image = ImageSearchTypeConfig(
search_config.image = ImageSearchConfig(
encoder = "clip-ViT-B-32",
model_directory = model_dir
)
@@ -39,8 +37,8 @@ def model_dir(search_config):
model_dir = search_config.asymmetric.model_directory
# Generate Image Embeddings from Test Images
content_config = ContentTypeConfig()
content_config.image = ImageSearchConfig(
content_config = ContentConfig()
content_config.image = ImageContentConfig(
input_directory = 'tests/data',
embeddings_file = model_dir.joinpath('.image_embeddings.pt'),
batch_size = 10,
@@ -49,7 +47,7 @@ def model_dir(search_config):
image_search.setup(content_config.image, search_config.image, regenerate=False, verbose=True)
# Generate Notes Embeddings from Test Notes
content_config.org = TextSearchConfig(
content_config.org = TextContentConfig(
input_files = ['tests/data/main_readme.org', 'tests/data/interface_emacs_readme.org'],
input_filter = None,
compressed_jsonl = model_dir.joinpath('.notes.jsonl.gz'),
@@ -62,14 +60,14 @@ def model_dir(search_config):
@pytest.fixture(scope='session')
def content_config(model_dir):
content_config = ContentTypeConfig()
content_config.org = TextSearchConfig(
content_config = ContentConfig()
content_config.org = TextContentConfig(
input_files = ['tests/data/main_readme.org', 'tests/data/interface_emacs_readme.org'],
input_filter = None,
compressed_jsonl = model_dir.joinpath('.notes.jsonl.gz'),
embeddings_file = model_dir.joinpath('.note_embeddings.pt'))
content_config.image = ImageSearchConfig(
content_config.image = ImageContentConfig(
input_directory = 'tests/data',
embeddings_file = model_dir.joinpath('.image_embeddings.pt'),
batch_size = 10,

6
tests/test_asymmetric_search.py
View File

@@ -1,12 +1,12 @@
# Internal Packages
from src.main import model
from src.search_type import asymmetric
from src.utils.rawconfig import ContentTypeConfig, SearchTypeConfig
from src.utils.rawconfig import ContentConfig, SearchConfig
# Test
# ----------------------------------------------------------------------------------------------------
def test_asymmetric_setup(content_config: ContentTypeConfig, search_config: SearchTypeConfig):
def test_asymmetric_setup(content_config: ContentConfig, search_config: SearchConfig):
# Act
# Regenerate notes embeddings during asymmetric setup
notes_model = asymmetric.setup(content_config.org, search_config.asymmetric, regenerate=True)
@@ -17,7 +17,7 @@ def test_asymmetric_setup(content_config: ContentTypeConfig, search_config: Sear
# ----------------------------------------------------------------------------------------------------
def test_asymmetric_search(content_config: ContentTypeConfig, search_config: SearchTypeConfig):
def test_asymmetric_search(content_config: ContentConfig, search_config: SearchConfig):
# Arrange
model.notes_search = asymmetric.setup(content_config.org, search_config.asymmetric, regenerate=False)
query = "How to git install application?"

14
tests/test_client.py
View File

@@ -9,7 +9,7 @@ import pytest
from src.main import app, model, config
from src.search_type import asymmetric, image_search
from src.utils.helpers import resolve_absolute_path
from src.utils.rawconfig import ContentTypeConfig, SearchTypeConfig
from src.utils.rawconfig import ContentConfig, SearchConfig
# Arrange
@@ -30,7 +30,7 @@ def test_search_with_invalid_content_type():
# ----------------------------------------------------------------------------------------------------
def test_search_with_valid_content_type(content_config: ContentTypeConfig, search_config: SearchTypeConfig):
def test_search_with_valid_content_type(content_config: ContentConfig, search_config: SearchConfig):
# Arrange
config.content_type = content_config
config.search_type = search_config
@@ -53,7 +53,7 @@ def test_regenerate_with_invalid_content_type():
# ----------------------------------------------------------------------------------------------------
def test_regenerate_with_valid_content_type(content_config: ContentTypeConfig, search_config: SearchTypeConfig):
def test_regenerate_with_valid_content_type(content_config: ContentConfig, search_config: SearchConfig):
# Arrange
config.content_type = content_config
config.search_type = search_config
@@ -67,7 +67,7 @@ def test_regenerate_with_valid_content_type(content_config: ContentTypeConfig, s
# ----------------------------------------------------------------------------------------------------
@pytest.mark.skip(reason="Flaky test. Search doesn't always return expected image path.")
def test_image_search(content_config: ContentTypeConfig, search_config: SearchTypeConfig):
def test_image_search(content_config: ContentConfig, search_config: SearchConfig):
# Arrange
config.content_type = content_config
config.search_type = search_config
@@ -90,7 +90,7 @@ def test_image_search(content_config: ContentTypeConfig, search_config: SearchTy
# ----------------------------------------------------------------------------------------------------
def test_notes_search(content_config: ContentTypeConfig, search_config: SearchTypeConfig):
def test_notes_search(content_config: ContentConfig, search_config: SearchConfig):
# Arrange
model.notes_search = asymmetric.setup(content_config.org, search_config.asymmetric, regenerate=False)
user_query = "How to git install application?"
@@ -106,7 +106,7 @@ def test_notes_search(content_config: ContentTypeConfig, search_config: SearchTy
# ----------------------------------------------------------------------------------------------------
def test_notes_search_with_include_filter(content_config: ContentTypeConfig, search_config: SearchTypeConfig):
def test_notes_search_with_include_filter(content_config: ContentConfig, search_config: SearchConfig):
# Arrange
model.notes_search = asymmetric.setup(content_config.org, search_config.asymmetric, regenerate=False)
user_query = "How to git install application? +Emacs"
@@ -122,7 +122,7 @@ def test_notes_search_with_include_filter(content_config: ContentTypeConfig, sea
# ----------------------------------------------------------------------------------------------------
def test_notes_search_with_exclude_filter(content_config: ContentTypeConfig, search_config: SearchTypeConfig):
def test_notes_search_with_exclude_filter(content_config: ContentConfig, search_config: SearchConfig):
# Arrange
model.notes_search = asymmetric.setup(content_config.org, search_config.asymmetric, regenerate=False)
user_query = "How to git install application? -clone"

6
tests/test_image_search.py
View File

@@ -5,12 +5,12 @@ import pytest
from src.main import model
from src.search_type import image_search
from src.utils.helpers import resolve_absolute_path
from src.utils.rawconfig import ContentTypeConfig, SearchTypeConfig
from src.utils.rawconfig import ContentConfig, SearchConfig
# Test
# ----------------------------------------------------------------------------------------------------
def test_image_search_setup(content_config: ContentTypeConfig, search_config: SearchTypeConfig):
def test_image_search_setup(content_config: ContentConfig, search_config: SearchConfig):
# Act
# Regenerate image search embeddings during image setup
image_search_model = image_search.setup(content_config.image, search_config.image, regenerate=True)
@@ -22,7 +22,7 @@ def test_image_search_setup(content_config: ContentTypeConfig, search_config: Se
# ----------------------------------------------------------------------------------------------------
@pytest.mark.skip(reason="results inconsistent currently")
def test_image_search(content_config: ContentTypeConfig, search_config: SearchTypeConfig):
def test_image_search(content_config: ContentConfig, search_config: SearchConfig):
# Arrange
model.image_search = image_search.setup(content_config.image, search_config.image, regenerate=False)
query_expected_image_pairs = [("brown kitten next to plant", "kitten_park.jpg"),