mirror of
https://github.com/khoaliber/khoj.git
synced 2026-03-09 05:39:12 +00:00
Rename RawConfig Types for Consistency
- Naming convention - [ContentType][ConfigType]Config
- Where [ConfigType] ~ Content, Search, Processor
- Where [ContentType] ~ Text, Image, Asymmetric, Symmetric, Conversation
- Current Configs:
- Content:
- Org Notes
- Org Music
- Image
- Ledger/Beancount
- Search:
- Asymmetric
- Symmetric
- Image
- Processor:
- Conversation
This commit is contained in:
@@ -15,10 +15,10 @@ from sentence_transformers import SentenceTransformer, CrossEncoder, util
|
||||
from src.utils.helpers import get_absolute_path, resolve_absolute_path, load_model
|
||||
from src.processor.org_mode.org_to_jsonl import org_to_jsonl
|
||||
from src.utils.config import TextSearchModel
|
||||
from src.utils.rawconfig import AsymmetricConfig, TextSearchConfig
|
||||
from src.utils.rawconfig import AsymmetricSearchConfig, TextContentConfig
|
||||
|
||||
|
||||
def initialize_model(search_config: AsymmetricConfig):
|
||||
def initialize_model(search_config: AsymmetricSearchConfig):
|
||||
"Initialize model for assymetric semantic search. That is, where query smaller than results"
|
||||
torch.set_num_threads(4)
|
||||
|
||||
@@ -162,7 +162,7 @@ def collate_results(hits, entries, count=5):
|
||||
in hits[0:count]]
|
||||
|
||||
|
||||
def setup(config: TextSearchConfig, search_config: AsymmetricConfig, regenerate: bool, verbose: bool=False) -> TextSearchModel:
|
||||
def setup(config: TextContentConfig, search_config: AsymmetricSearchConfig, regenerate: bool, verbose: bool=False) -> TextSearchModel:
|
||||
# Initialize Model
|
||||
bi_encoder, cross_encoder, top_k = initialize_model(search_config)
|
||||
|
||||
|
||||
@@ -13,10 +13,10 @@ import torch
|
||||
from src.utils.helpers import resolve_absolute_path, load_model
|
||||
import src.utils.exiftool as exiftool
|
||||
from src.utils.config import ImageSearchModel
|
||||
from src.utils.rawconfig import ImageSearchConfig, ImageSearchTypeConfig
|
||||
from src.utils.rawconfig import ImageContentConfig, ImageSearchConfig
|
||||
|
||||
|
||||
def initialize_model(search_config: ImageSearchTypeConfig):
|
||||
def initialize_model(search_config: ImageSearchConfig):
|
||||
# Initialize Model
|
||||
torch.set_num_threads(4)
|
||||
|
||||
@@ -160,7 +160,7 @@ def collate_results(hits, image_names, image_directory, count=5):
|
||||
in hits[0:count]]
|
||||
|
||||
|
||||
def setup(config: ImageSearchConfig, search_config: ImageSearchTypeConfig, regenerate: bool, verbose: bool=False) -> ImageSearchModel:
|
||||
def setup(config: ImageContentConfig, search_config: ImageSearchConfig, regenerate: bool, verbose: bool=False) -> ImageSearchModel:
|
||||
# Initialize Model
|
||||
encoder = initialize_model(search_config)
|
||||
|
||||
|
||||
@@ -13,10 +13,10 @@ from sentence_transformers import SentenceTransformer, CrossEncoder, util
|
||||
from src.utils.helpers import get_absolute_path, resolve_absolute_path, load_model
|
||||
from src.processor.ledger.beancount_to_jsonl import beancount_to_jsonl
|
||||
from src.utils.config import TextSearchModel
|
||||
from src.utils.rawconfig import SymmetricConfig, TextSearchConfig
|
||||
from src.utils.rawconfig import SymmetricSearchConfig, TextContentConfig
|
||||
|
||||
|
||||
def initialize_model(search_config: SymmetricConfig):
|
||||
def initialize_model(search_config: SymmetricSearchConfig):
|
||||
"Initialize model for symmetric semantic search. That is, where query of similar size to results"
|
||||
torch.set_num_threads(4)
|
||||
|
||||
@@ -154,7 +154,7 @@ def collate_results(hits, entries, count=5):
|
||||
in hits[0:count]]
|
||||
|
||||
|
||||
def setup(config: TextSearchConfig, search_config: SymmetricConfig, regenerate: bool, verbose: bool) -> TextSearchModel:
|
||||
def setup(config: TextContentConfig, search_config: SymmetricSearchConfig, regenerate: bool, verbose: bool) -> TextSearchModel:
|
||||
# Initialize Model
|
||||
bi_encoder, cross_encoder, top_k = initialize_model(search_config)
|
||||
|
||||
|
||||
@@ -13,57 +13,52 @@ class ConfigBase(BaseModel):
|
||||
alias_generator = to_snake_case_from_dash
|
||||
allow_population_by_field_name = True
|
||||
|
||||
class SearchConfig(ConfigBase):
|
||||
input_files: Optional[List[str]]
|
||||
input_filter: Optional[str]
|
||||
embeddings_file: Optional[Path]
|
||||
|
||||
class TextSearchConfig(ConfigBase):
|
||||
class TextContentConfig(ConfigBase):
|
||||
compressed_jsonl: Optional[Path]
|
||||
input_files: Optional[List[str]]
|
||||
input_filter: Optional[str]
|
||||
embeddings_file: Optional[Path]
|
||||
|
||||
class ImageSearchConfig(ConfigBase):
|
||||
class ImageContentConfig(ConfigBase):
|
||||
use_xmp_metadata: Optional[str]
|
||||
batch_size: Optional[int]
|
||||
input_directory: Optional[Path]
|
||||
input_filter: Optional[str]
|
||||
embeddings_file: Optional[Path]
|
||||
|
||||
class ContentTypeConfig(ConfigBase):
|
||||
org: Optional[TextSearchConfig]
|
||||
ledger: Optional[TextSearchConfig]
|
||||
class ContentConfig(ConfigBase):
|
||||
org: Optional[TextContentConfig]
|
||||
ledger: Optional[TextContentConfig]
|
||||
image: Optional[ImageContentConfig]
|
||||
music: Optional[TextContentConfig]
|
||||
|
||||
class SymmetricSearchConfig(ConfigBase):
|
||||
encoder: Optional[str]
|
||||
cross_encoder: Optional[str]
|
||||
model_directory: Optional[Path]
|
||||
|
||||
class AsymmetricSearchConfig(ConfigBase):
|
||||
encoder: Optional[str]
|
||||
cross_encoder: Optional[str]
|
||||
model_directory: Optional[Path]
|
||||
|
||||
class ImageSearchConfig(ConfigBase):
|
||||
encoder: Optional[str]
|
||||
model_directory: Optional[Path]
|
||||
|
||||
class SearchConfig(ConfigBase):
|
||||
asymmetric: Optional[AsymmetricSearchConfig]
|
||||
symmetric: Optional[SymmetricSearchConfig]
|
||||
image: Optional[ImageSearchConfig]
|
||||
music: Optional[TextSearchConfig]
|
||||
|
||||
class SymmetricConfig(ConfigBase):
|
||||
encoder: Optional[str]
|
||||
cross_encoder: Optional[str]
|
||||
model_directory: Optional[Path]
|
||||
|
||||
class AsymmetricConfig(ConfigBase):
|
||||
encoder: Optional[str]
|
||||
cross_encoder: Optional[str]
|
||||
model_directory: Optional[Path]
|
||||
|
||||
class ImageSearchTypeConfig(ConfigBase):
|
||||
encoder: Optional[str]
|
||||
model_directory: Optional[Path]
|
||||
|
||||
class SearchTypeConfig(ConfigBase):
|
||||
asymmetric: Optional[AsymmetricConfig]
|
||||
symmetric: Optional[SymmetricConfig]
|
||||
image: Optional[ImageSearchTypeConfig]
|
||||
|
||||
class ConversationProcessorConfig(ConfigBase):
|
||||
openai_api_key: Optional[str]
|
||||
conversation_logfile: Optional[str]
|
||||
|
||||
class ProcessorConfigModel(ConfigBase):
|
||||
class ProcessorConfig(ConfigBase):
|
||||
conversation: Optional[ConversationProcessorConfig]
|
||||
|
||||
class FullConfig(ConfigBase):
|
||||
content_type: Optional[ContentTypeConfig]
|
||||
search_type: Optional[SearchTypeConfig]
|
||||
processor: Optional[ProcessorConfigModel]
|
||||
content_type: Optional[ContentConfig]
|
||||
search_type: Optional[SearchConfig]
|
||||
processor: Optional[ProcessorConfig]
|
||||
|
||||
Reference in New Issue
Block a user