diff --git a/src/configure.py b/src/configure.py index 57517883..d920a614 100644 --- a/src/configure.py +++ b/src/configure.py @@ -12,7 +12,7 @@ from src.processor.org_mode.org_to_jsonl import org_to_jsonl from src.search_type import image_search, text_search from src.utils.config import SearchType, SearchModels, ProcessorConfigModel, ConversationProcessorConfigModel from src.utils import state -from src.utils.helpers import resolve_absolute_path +from src.utils.helpers import LRU, resolve_absolute_path from src.utils.rawconfig import FullConfig, ProcessorConfig from src.search_filter.date_filter import DateFilter from src.search_filter.word_filter import WordFilter @@ -89,7 +89,7 @@ def configure_search(model: SearchModels, config: FullConfig, regenerate: bool, regenerate=regenerate) # Invalidate Query Cache - state.query_cache = {} + state.query_cache = LRU() return model diff --git a/src/interface/desktop/system_tray.py b/src/interface/desktop/system_tray.py index 7b230c54..26df260f 100644 --- a/src/interface/desktop/system_tray.py +++ b/src/interface/desktop/system_tray.py @@ -6,9 +6,10 @@ from PyQt6 import QtGui, QtWidgets # Internal Packages from src.utils import constants, state +from src.interface.desktop.main_window import MainWindow -def create_system_tray(gui: QtWidgets.QApplication, main_window: QtWidgets.QMainWindow): +def create_system_tray(gui: QtWidgets.QApplication, main_window: MainWindow): """Create System Tray with Menu. Menu contain options to 1. Open Search Page on the Web Interface 2. Open App Configuration Screen diff --git a/src/main.py b/src/main.py index e7dbba5d..dc3b12c3 100644 --- a/src/main.py +++ b/src/main.py @@ -42,14 +42,14 @@ class CustomFormatter(logging.Formatter): red = "\x1b[31;20m" bold_red = "\x1b[31;1m" reset = "\x1b[0m" - format = "%(levelname)s: %(asctime)s: %(name)s | %(message)s" + format_str = "%(levelname)s: %(asctime)s: %(name)s | %(message)s" FORMATS = { - logging.DEBUG: blue + format + reset, - logging.INFO: green + format + reset, - logging.WARNING: yellow + format + reset, - logging.ERROR: red + format + reset, - logging.CRITICAL: bold_red + format + reset + logging.DEBUG: blue + format_str + reset, + logging.INFO: green + format_str + reset, + logging.WARNING: yellow + format_str + reset, + logging.ERROR: red + format_str + reset, + logging.CRITICAL: bold_red + format_str + reset } def format(self, record): diff --git a/src/processor/ledger/beancount_to_jsonl.py b/src/processor/ledger/beancount_to_jsonl.py index b80fece6..7b8b9bba 100644 --- a/src/processor/ledger/beancount_to_jsonl.py +++ b/src/processor/ledger/beancount_to_jsonl.py @@ -2,8 +2,6 @@ # Standard Packages import json -import argparse -import pathlib import glob import re import logging diff --git a/src/processor/markdown/markdown_to_jsonl.py b/src/processor/markdown/markdown_to_jsonl.py index 80562a9f..22f5ea17 100644 --- a/src/processor/markdown/markdown_to_jsonl.py +++ b/src/processor/markdown/markdown_to_jsonl.py @@ -2,8 +2,6 @@ # Standard Packages import json -import argparse -import pathlib import glob import re import logging diff --git a/src/processor/org_mode/org_to_jsonl.py b/src/processor/org_mode/org_to_jsonl.py index 699ceaa9..43f4acef 100644 --- a/src/processor/org_mode/org_to_jsonl.py +++ b/src/processor/org_mode/org_to_jsonl.py @@ -1,13 +1,11 @@ #!/usr/bin/env python3 # Standard Packages -import re import json -import argparse -import pathlib import glob import logging import time +from typing import Iterable # Internal Packages from src.processor.org_mode import orgnode @@ -50,7 +48,7 @@ def org_to_jsonl(config: TextContentConfig, previous_entries=None): if not previous_entries: entries_with_ids = list(enumerate(current_entries)) else: - entries_with_ids = mark_entries_for_update(current_entries, previous_entries, key='compiled', logger=logger) + entries_with_ids = mark_entries_for_update(current_entries, previous_entries, key='compiled', logger=logger) # Process Each Entry from All Notes Files start = time.time() @@ -72,9 +70,11 @@ def get_org_files(org_files=None, org_file_filters=None): "Get Org files to process" absolute_org_files, filtered_org_files = set(), set() if org_files: - absolute_org_files = {get_absolute_path(org_file) - for org_file - in org_files} + absolute_org_files = { + get_absolute_path(org_file) + for org_file + in org_files + } if org_file_filters: filtered_org_files = { filtered_file @@ -150,6 +150,6 @@ def convert_org_nodes_to_entries(entries: list[orgnode.Orgnode], entry_to_file_m return entry_maps -def convert_org_entries_to_jsonl(entries: list[dict]) -> str: +def convert_org_entries_to_jsonl(entries: Iterable[dict]) -> str: "Convert each Org-Mode entry to JSON and collate as JSONL" return ''.join([f'{json.dumps(entry_dict, ensure_ascii=False)}\n' for entry_dict in entries]) diff --git a/src/router.py b/src/router.py index 9d53a00e..d73ca331 100644 --- a/src/router.py +++ b/src/router.py @@ -17,14 +17,14 @@ from src.search_type import image_search, text_search from src.processor.conversation.gpt import converse, extract_search_type, message_to_log, message_to_prompt, understand, summarize from src.utils.rawconfig import FullConfig from src.utils.config import SearchType -from src.utils.helpers import get_absolute_path, get_from_dict +from src.utils.helpers import LRU, get_absolute_path, get_from_dict from src.utils import state, constants router = APIRouter() templates = Jinja2Templates(directory=constants.web_directory) logger = logging.getLogger(__name__) -query_cache = {} +query_cache = LRU() @router.get("/", response_class=FileResponse) diff --git a/src/search_filter/base_filter.py b/src/search_filter/base_filter.py index 735b6915..2550b32e 100644 --- a/src/search_filter/base_filter.py +++ b/src/search_filter/base_filter.py @@ -1,9 +1,5 @@ # Standard Packages from abc import ABC, abstractmethod -from typing import List, Set, Tuple - -# External Packages -import torch class BaseFilter(ABC): @@ -16,5 +12,5 @@ class BaseFilter(ABC): pass @abstractmethod - def apply(self, query:str, raw_entries:List[str]) -> Tuple[str, Set[int]]: + def apply(self, query:str, raw_entries:list[str]) -> tuple[str, set[int]]: pass \ No newline at end of file diff --git a/src/search_type/image_search.py b/src/search_type/image_search.py index 9ac8aa9a..c9dcdd6b 100644 --- a/src/search_type/image_search.py +++ b/src/search_type/image_search.py @@ -1,5 +1,4 @@ # Standard Packages -import argparse import glob import pathlib import copy diff --git a/src/search_type/text_search.py b/src/search_type/text_search.py index 922fab1a..d4d8a9d4 100644 --- a/src/search_type/text_search.py +++ b/src/search_type/text_search.py @@ -1,6 +1,4 @@ # Standard Packages -import argparse -import pathlib import logging import time diff --git a/src/utils/config.py b/src/utils/config.py index c163e22f..c417b2bf 100644 --- a/src/utils/config.py +++ b/src/utils/config.py @@ -2,7 +2,6 @@ from enum import Enum from dataclasses import dataclass from pathlib import Path -from typing import List # Internal Packages from src.utils.rawconfig import ConversationProcessorConfig @@ -22,7 +21,7 @@ class ProcessorType(str, Enum): class TextSearchModel(): - def __init__(self, entries, corpus_embeddings, bi_encoder, cross_encoder, filters: List[BaseFilter], top_k): + def __init__(self, entries, corpus_embeddings, bi_encoder, cross_encoder, filters: list[BaseFilter], top_k): self.entries = entries self.corpus_embeddings = corpus_embeddings self.bi_encoder = bi_encoder @@ -54,7 +53,7 @@ class ConversationProcessorConfigModel(): self.openai_api_key = processor_config.openai_api_key self.conversation_logfile = Path(processor_config.conversation_logfile) self.chat_session = '' - self.meta_log = [] + self.meta_log: dict = {} @dataclass diff --git a/src/utils/helpers.py b/src/utils/helpers.py index e8b2b8ca..df1899f9 100644 --- a/src/utils/helpers.py +++ b/src/utils/helpers.py @@ -1,10 +1,11 @@ # Standard Packages -import pathlib +from pathlib import Path import sys import time import hashlib from os.path import join from collections import OrderedDict +from typing import Optional, Union def is_none_or_empty(item): @@ -15,12 +16,12 @@ def to_snake_case_from_dash(item: str): return item.replace('_', '-') -def get_absolute_path(filepath): - return str(pathlib.Path(filepath).expanduser().absolute()) +def get_absolute_path(filepath: Union[str, Path]) -> str: + return str(Path(filepath).expanduser().absolute()) -def resolve_absolute_path(filepath, strict=False): - return pathlib.Path(filepath).expanduser().absolute().resolve(strict=strict) +def resolve_absolute_path(filepath: Union[str, Optional[Path]], strict=False) -> Path: + return Path(filepath).expanduser().absolute().resolve(strict=strict) def get_from_dict(dictionary, *args): diff --git a/src/utils/state.py b/src/utils/state.py index f3ceda00..283d2b5a 100644 --- a/src/utils/state.py +++ b/src/utils/state.py @@ -1,5 +1,6 @@ # Standard Packages from packaging import version + # External Packages import torch from pathlib import Path @@ -13,11 +14,11 @@ from src.utils.rawconfig import FullConfig config = FullConfig() model = SearchModels() processor_config = ProcessorConfigModel() -config_file: Path = "" +config_file: Path = None verbose: int = 0 host: str = None port: int = None -cli_args = None +cli_args: list[str] = None query_cache = LRU() if torch.cuda.is_available(): diff --git a/src/utils/yaml.py b/src/utils/yaml.py index 46ddb788..a70c6f76 100644 --- a/src/utils/yaml.py +++ b/src/utils/yaml.py @@ -5,12 +5,13 @@ from pathlib import Path import yaml # Internal Packages -from src.utils.helpers import get_absolute_path, resolve_absolute_path from src.utils.rawconfig import FullConfig + # Do not emit tags when dumping to YAML yaml.emitter.Emitter.process_tag = lambda self, *args, **kwargs: None + def save_config_to_file(yaml_config: dict, yaml_config_file: Path): "Write config to YML file" # Create output directory, if it doesn't exist