Modularize Code. Wrap Search, Model Config in Classes. Add Tests

Details
  - Rename method query_* to query in search_types for standardization
  - Wrapping Config code in classes simplified mocking test config
  - Reduce args beings passed to a function by passing it as single
    argument wrapped in a class
  - Minimize setup in main.py:__main__. Put most of it into functions
    These functions can be mocked if required in tests later too

Setup Flow:
  CLI_Args|Config_YAML -> (Text|Image)SearchConfig -> (Text|Image)SearchModel

src/main.py

@@ -11,12 +11,12 @@ from fastapi import FastAPI
 from search_type import asymmetric, symmetric_ledger, image_search
 from utils.helpers import get_from_dict
 from utils.cli import cli
-from utils.config import SearchType, SearchSettings, SearchModels
+from utils.config import SearchType, SearchModels, TextSearchConfig, ImageSearchConfig, SearchConfig
 
 
 # Application Global State
 model = SearchModels()
-search_settings = SearchSettings()
+search_config = SearchConfig()
 app = FastAPI()
 
 
@@ -29,36 +29,36 @@ def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None):
     user_query = q
     results_count = n
 
-    if (t == SearchType.Notes or t == None) and search_settings.notes_search_enabled:
+    if (t == SearchType.Notes or t == None) and model.notes_search:
         # query notes
-        hits = asymmetric.query_notes(user_query, model.notes_search)
+        hits = asymmetric.query(user_query, model.notes_search)
 
         # collate and return results
         return asymmetric.collate_results(hits, model.notes_search.entries, results_count)
 
-    if (t == SearchType.Music or t == None) and search_settings.music_search_enabled:
+    if (t == SearchType.Music or t == None) and model.music_search:
         # query music library
-        hits = asymmetric.query_notes(user_query, model.music_search)
+        hits = asymmetric.query(user_query, model.music_search)
 
         # collate and return results
         return asymmetric.collate_results(hits, model.music_search.entries, results_count)
 
-    if (t == SearchType.Ledger or t == None) and search_settings.ledger_search_enabled:
+    if (t == SearchType.Ledger or t == None) and model.ledger_search:
         # query transactions
-        hits = symmetric_ledger.query_transactions(user_query, model.ledger_search)
+        hits = symmetric_ledger.query(user_query, model.ledger_search)
 
         # collate and return results
         return symmetric_ledger.collate_results(hits, model.ledger_search.entries, results_count)
 
-    if (t == SearchType.Image or t == None) and search_settings.image_search_enabled:
+    if (t == SearchType.Image or t == None) and model.image_search:
         # query transactions
-        hits = image_search.query_images(user_query, model.image_search, args.verbose)
+        hits = image_search.query(user_query, results_count, model.image_search)
 
         # collate and return results
         return image_search.collate_results(
             hits,
             model.image_search.image_names,
-            image_config['input-directory'],
+            search_config.image.input_directory,
             results_count)
 
     else:
@@ -67,98 +67,58 @@ def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None):
 
 @app.get('/regenerate')
 def regenerate(t: Optional[SearchType] = None):
-    if (t == SearchType.Notes or t == None) and search_settings.notes_search_enabled:
+    if (t == SearchType.Notes or t == None) and search_config.notes:
         # Extract Entries, Generate Embeddings
-        models.notes_search = asymmetric.setup(
-            org_config['input-files'],
-            org_config['input-filter'],
-            pathlib.Path(org_config['compressed-jsonl']),
-            pathlib.Path(org_config['embeddings-file']),
-            regenerate=True,
-            verbose=args.verbose)
+        model.notes_search = asymmetric.setup(search_config.notes, regenerate=True)
 
-    if (t == SearchType.Music or t == None) and search_settings.music_search_enabled:
+    if (t == SearchType.Music or t == None) and search_config.music:
         # Extract Entries, Generate Song Embeddings
-        model.music_search = asymmetric.setup(
-            song_config['input-files'],
-            song_config['input-filter'],
-            pathlib.Path(song_config['compressed-jsonl']),
-            pathlib.Path(song_config['embeddings-file']),
-            regenerate=True,
-            verbose=args.verbose)
+        model.music_search = asymmetric.setup(search_config.music, regenerate=True)
 
-    if (t == SearchType.Ledger or t == None) and search_settings.ledger_search_enabled:
+    if (t == SearchType.Ledger or t == None) and search_config.ledger:
         # Extract Entries, Generate Embeddings
-        model.ledger_search = symmetric_ledger.setup(
-            ledger_config['input-files'],
-            ledger_config['input-filter'],
-            pathlib.Path(ledger_config['compressed-jsonl']),
-            pathlib.Path(ledger_config['embeddings-file']),
-            regenerate=True,
-            verbose=args.verbose)
+        model.ledger_search = symmetric_ledger.setup(search_config.ledger, regenerate=True)
 
-    if (t == SearchType.Image or t == None) and search_settings.image_search_enabled:
+    if (t == SearchType.Image or t == None) and search_config.image:
         # Extract Images, Generate Embeddings
-        model.image_search = image_search.setup(
-            pathlib.Path(image_config['input-directory']),
-            pathlib.Path(image_config['embeddings-file']),
-            regenerate=True,
-            verbose=args.verbose)
+        model.image_search = image_search.setup(search_config.image, regenerate=True)
 
     return {'status': 'ok', 'message': 'regeneration completed'}
 
 
-if __name__ == '__main__':
-    args = cli(sys.argv[1:])
+def initialize_search(config, regenerate, verbose):
+    model = SearchModels()
+    search_config = SearchConfig()
 
     # Initialize Org Notes Search
-    org_config = get_from_dict(args.config, 'content-type', 'org')
-    if org_config and ('input-files' in org_config or 'input-filter' in org_config):
-        search_settings.notes_search_enabled = True
-        model.notes_search = asymmetric.setup(
-            org_config['input-files'],
-            org_config['input-filter'],
-            pathlib.Path(org_config['compressed-jsonl']),
-            pathlib.Path(org_config['embeddings-file']),
-            args.regenerate,
-            args.verbose)
+    search_config.notes = TextSearchConfig.create_from_dictionary(config, ('content-type', 'org'), verbose)
+    if search_config.notes:
+        model.notes_search = asymmetric.setup(search_config.notes, regenerate=regenerate)
 
     # Initialize Org Music Search
-    song_config = get_from_dict(args.config, 'content-type', 'music')
-    music_search_enabled = False
-    if song_config and ('input-files' in song_config or 'input-filter' in song_config):
-        search_settings.music_search_enabled = True
-        model.music_search = asymmetric.setup(
-            song_config['input-files'],
-            song_config['input-filter'],
-            pathlib.Path(song_config['compressed-jsonl']),
-            pathlib.Path(song_config['embeddings-file']),
-            args.regenerate,
-            args.verbose)
+    search_config.music = TextSearchConfig.create_from_dictionary(config, ('content-type', 'music'), verbose)
+    if search_config.music:
+        model.music_search = asymmetric.setup(search_config.music, regenerate=regenerate)
 
     # Initialize Ledger Search
-    ledger_config = get_from_dict(args.config, 'content-type', 'ledger')
-    if ledger_config and ('input-files' in ledger_config or 'input-filter' in ledger_config):
-        search_settings.ledger_search_enabled = True
-        model.ledger_search = symmetric_ledger.setup(
-            ledger_config['input-files'],
-            ledger_config['input-filter'],
-            pathlib.Path(ledger_config['compressed-jsonl']),
-            pathlib.Path(ledger_config['embeddings-file']),
-            args.regenerate,
-            args.verbose)
+    search_config.ledger = TextSearchConfig.create_from_dictionary(config, ('content-type', 'ledger'), verbose)
+    if search_config.ledger:
+        model.ledger_search = symmetric_ledger.setup(search_config.ledger, regenerate=regenerate)
 
     # Initialize Image Search
-    image_config = get_from_dict(args.config, 'content-type', 'image')
-    if image_config and 'input-directory' in image_config:
-        search_settings.image_search_enabled = True
-        model.image_search = image_search.setup(
-            pathlib.Path(image_config['input-directory']),
-            pathlib.Path(image_config['embeddings-file']),
-            batch_size=image_config['batch-size'],
-            regenerate=args.regenerate,
-            use_xmp_metadata={'yes': True, 'no': False}[image_config['use-xmp-metadata']],
-            verbose=args.verbose)
+    search_config.image = ImageSearchConfig.create_from_dictionary(config, ('content-type', 'image'), verbose)
+    if search_config.image:
+        model.image_search = image_search.setup(search_config.image, regenerate=regenerate)
+
+    return model, search_config
+
+
+if __name__ == '__main__':
+    # Load config from CLI
+    args = cli(sys.argv[1:])
+
+    # Initialize Search from Config
+    model, search_config = initialize_search(args.config, args.regenerate, args.verbose)
 
     # Start Application Server
     uvicorn.run(app)