Merge branch 'master' of github.com:debanjum/semantic-search into add-summarize-capability-to-chat-bot

- Fix openai_api_key being set in ConfigProcessorConfig
- Merge addition of config UI and config instantiation updates

src/utils/cli.py

@@ -1,12 +1,14 @@
 # Standard Packages
 import argparse
 import pathlib
+import json
 
 # External Packages
 import yaml
 
 # Internal Packages
-from src.utils.helpers import is_none_or_empty, get_absolute_path, resolve_absolute_path, get_from_dict, merge_dicts
+from src.utils.helpers import is_none_or_empty, get_absolute_path, resolve_absolute_path, merge_dicts
+from src.utils.rawconfig import FullConfig
 
 def cli(args=None):
     if is_none_or_empty(args):
@@ -35,12 +37,15 @@ def cli(args=None):
         with open(get_absolute_path(args.config_file), 'r', encoding='utf-8') as config_file:
             config_from_file = yaml.safe_load(config_file)
             args.config = merge_dicts(priority_dict=config_from_file, default_dict=args.config)
+            args.config = FullConfig.parse_obj(args.config)
+    else:
+        args.config = FullConfig.parse_obj(args.config)
 
     if args.org_files:
-        args.config['content-type']['org']['input-files'] = args.org_files
+        args.config.content_type.org.input_files = args.org_files
 
     if args.org_filter:
-        args.config['content-type']['org']['input-filter'] = args.org_filter
+        args.config.content_type.org.input_filter = args.org_filter
 
     return args
 

src/utils/config.py

@@ -4,7 +4,7 @@ from dataclasses import dataclass
 from pathlib import Path
 
 # Internal Packages
-from src.utils.helpers import get_from_dict
+from src.utils.rawconfig import ConversationProcessorConfig
 
 
 class SearchType(str, Enum):
@@ -42,80 +42,15 @@ class SearchModels():
     image_search: ImageSearchModel = None
 
 
-class TextSearchConfig():
-    def __init__(self, input_files, input_filter, compressed_jsonl, embeddings_file, verbose):
-        self.input_files = input_files
-        self.input_filter = input_filter
-        self.compressed_jsonl = Path(compressed_jsonl)
-        self.embeddings_file = Path(embeddings_file)
+class ConversationProcessorConfigModel():
+    def __init__(self, processor_config: ConversationProcessorConfig, verbose: bool):
+        self.openai_api_key = processor_config.openai_api_key
+        self.conversation_logfile = Path(processor_config.conversation_logfile)
+        self.chat_session = ''
+        self.meta_log = []
         self.verbose = verbose
 
 
-    def create_from_dictionary(config, key_tree, verbose):
-        text_config = get_from_dict(config, *key_tree)
-        search_enabled = text_config and ('input-files' in text_config or 'input-filter' in text_config)
-        if not search_enabled:
-            return None
-
-        return TextSearchConfig(
-            input_files = text_config['input-files'],
-            input_filter = text_config['input-filter'],
-            compressed_jsonl = Path(text_config['compressed-jsonl']),
-            embeddings_file = Path(text_config['embeddings-file']),
-            verbose = verbose)
-
-
-class ImageSearchConfig():
-    def __init__(self, input_directory, embeddings_file, batch_size, use_xmp_metadata, verbose):
-        self.input_directory = input_directory
-        self.embeddings_file = Path(embeddings_file)
-        self.batch_size = batch_size
-        self.use_xmp_metadata = use_xmp_metadata
-        self.verbose = verbose
-
-    def create_from_dictionary(config, key_tree, verbose):
-        image_config = get_from_dict(config, *key_tree)
-        search_enabled = image_config and 'input-directory' in image_config
-        if not search_enabled:
-            return None
-
-        return ImageSearchConfig(
-            input_directory = Path(image_config['input-directory']),
-            embeddings_file = Path(image_config['embeddings-file']),
-            batch_size = image_config['batch-size'],
-            use_xmp_metadata = {'yes': True, 'no': False}[image_config['use-xmp-metadata']],
-            verbose = verbose)
-
-
 @dataclass
-class SearchConfig():
-    notes: TextSearchConfig = None
-    ledger: TextSearchConfig = None
-    music: TextSearchConfig = None
-    image: ImageSearchConfig = None
-
-
-class ConversationProcessorConfig():
-    def __init__(self, conversation_logfile, chat_session, meta_log, openai_api_key, verbose):
-        self.openai_api_key = openai_api_key
-        self.conversation_logfile = conversation_logfile
-        self.chat_session = chat_session
-        self.meta_log = meta_log
-        self.verbose = verbose
-
-    def create_from_dictionary(config, key_tree, verbose):
-        conversation_config = get_from_dict(config, *key_tree)
-        if not conversation_config:
-            return None
-
-        return ConversationProcessorConfig(
-            openai_api_key = conversation_config['openai-api-key'],
-            chat_session = '',
-            meta_log = [],
-            conversation_logfile = Path(conversation_config['conversation-logfile']),
-            verbose = verbose)
-
-
-@dataclass
-class ProcessorConfig():
-    conversation: ConversationProcessorConfig = None
+class ProcessorConfigModel():
+    conversation: ConversationProcessorConfigModel = None

src/utils/helpers.py

@@ -4,6 +4,8 @@ import pathlib
 def is_none_or_empty(item):
     return item == None or (hasattr(item, '__iter__') and len(item) == 0)
 
+def to_snake_case_from_dash(item: str):
+    return item.replace('_', '-')
 
 def get_absolute_path(filepath):
     return str(pathlib.Path(filepath).expanduser().absolute())

src/utils/rawconfig.py

@@ -0,0 +1,62 @@
+# System Packages
+from pathlib import Path
+from typing import List, Optional
+
+# External Packages
+from pydantic import BaseModel
+
+# Internal Packages
+from src.utils.helpers import to_snake_case_from_dash
+
+class ConfigBase(BaseModel):
+    class Config:
+        alias_generator = to_snake_case_from_dash
+        allow_population_by_field_name = True
+
+class SearchConfig(ConfigBase):
+    input_files: Optional[List[str]]
+    input_filter: Optional[str]
+    embeddings_file: Optional[Path]
+
+class TextSearchConfig(ConfigBase):
+    compressed_jsonl: Optional[Path]
+    input_files: Optional[List[str]]
+    input_filter: Optional[str]
+    embeddings_file: Optional[Path]
+
+class ImageSearchConfig(ConfigBase):
+    use_xmp_metadata: Optional[str]
+    batch_size: Optional[int]
+    input_directory: Optional[Path]
+    input_filter: Optional[str]
+    embeddings_file: Optional[Path]
+
+class ContentTypeConfig(ConfigBase):
+    org: Optional[TextSearchConfig]
+    ledger: Optional[TextSearchConfig]
+    image: Optional[ImageSearchConfig]
+    music: Optional[TextSearchConfig]
+
+class AsymmetricConfig(ConfigBase):
+    encoder: Optional[str]
+    cross_encoder: Optional[str]
+
+class ImageSearchTypeConfig(ConfigBase):
+    encoder: Optional[str]
+
+class SearchTypeConfig(ConfigBase):
+    asymmetric: Optional[AsymmetricConfig]
+    image: Optional[ImageSearchTypeConfig]
+
+class ConversationProcessorConfig(ConfigBase):
+    openai_api_key: Optional[str]
+    conversation_logfile: Optional[str]
+    conversation_history: Optional[str]
+
+class ProcessorConfigModel(ConfigBase):
+    conversation: Optional[ConversationProcessorConfig]
+
+class FullConfig(ConfigBase):
+    content_type: Optional[ContentTypeConfig]
+    search_type: Optional[SearchTypeConfig]
+    processor: Optional[ProcessorConfigModel]