klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-03 21:29:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update to re-use the raw config base models in config.py as well

Browse Source
This commit is contained in:
Saba
2021-11-28 11:57:33 -05:00
parent 6292fe4481
commit da52433d89
3 changed files with 25 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
src/utils/config.py
View File

@@ -6,6 +6,8 @@ from pathlib import Path
# Internal Packages
from src.utils.helpers import get_from_dict
from src.utils.rawconfig import TextSearchConfigTest, ImageSearchConfigTest, ProcessorConversationConfig
class SearchType(str, Enum):
Notes = "notes"
@@ -43,49 +45,22 @@ class SearchModels():
class TextSearchConfig():
def __init__(self, input_files, input_filter, compressed_jsonl, embeddings_file, verbose):
self.input_files = input_files
self.input_filter = input_filter
self.compressed_jsonl = Path(compressed_jsonl)
self.embeddings_file = Path(embeddings_file)
def __init__(self, text_search_config: TextSearchConfigTest, verbose: bool):
self.input_files = text_search_config.input_files
self.input_filter = text_search_config.input_filter
self.compressed_jsonl = Path(text_search_config.compressed_jsonl)
self.embeddings_file = Path(text_search_config.embeddings_file)
self.verbose = verbose
def create_from_dictionary(config, key_tree, verbose):
text_config = get_from_dict(config, *key_tree)
search_enabled = text_config and ('input-files' in text_config or 'input-filter' in text_config)
if not search_enabled:
return None
return TextSearchConfig(
input_files = text_config['input-files'],
input_filter = text_config['input-filter'],
compressed_jsonl = Path(text_config['compressed-jsonl']),
embeddings_file = Path(text_config['embeddings-file']),
verbose = verbose)
class ImageSearchConfig():
def __init__(self, input_directory, embeddings_file, batch_size, use_xmp_metadata, verbose):
self.input_directory = input_directory
self.embeddings_file = Path(embeddings_file)
self.batch_size = batch_size
self.use_xmp_metadata = use_xmp_metadata
def __init__(self, image_search_config: ImageSearchConfigTest, verbose):
self.input_directory = Path(image_search_config.input_directory)
self.embeddings_file = Path(image_search_config.embeddings_file)
self.batch_size = image_search_config.batch_size
self.use_xmp_metadata = image_search_config.use_xmp_metadata
self.verbose = verbose
def create_from_dictionary(config, key_tree, verbose):
image_config = get_from_dict(config, *key_tree)
search_enabled = image_config and 'input-directory' in image_config
if not search_enabled:
return None
return ImageSearchConfig(
input_directory = Path(image_config['input-directory']),
embeddings_file = Path(image_config['embeddings-file']),
batch_size = image_config['batch-size'],
use_xmp_metadata = {'yes': True, 'no': False}[image_config['use-xmp-metadata']],
verbose = verbose)
@dataclass
class SearchConfig():
@@ -96,24 +71,12 @@ class SearchConfig():
class ConversationProcessorConfig():
def __init__(self, conversation_logfile, conversation_history, openai_api_key, verbose):
self.openai_api_key = openai_api_key
self.conversation_logfile = conversation_logfile
self.conversation_history = conversation_history
def __init__(self, processor_config: ProcessorConversationConfig, verbose: bool):
self.openai_api_key = processor_config.open_api_key
self.conversation_logfile = Path(processor_config.conversation_logfile)
self.conversation_history = Path(processor_config.conversation_history)
self.verbose = verbose
def create_from_dictionary(config, key_tree, verbose):
conversation_config = get_from_dict(config, *key_tree)
if not conversation_config:
return None
return ConversationProcessorConfig(
openai_api_key = conversation_config['openai-api-key'],
conversation_history = '',
conversation_logfile = Path(conversation_config['conversation-logfile']),
verbose = verbose)
@dataclass
class ProcessorConfig():
conversation: ConversationProcessorConfig = None