klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-07 05:40:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge Symmetric, Asymmetric Search Types into a single Text Search Type

Browse Source 
- The code for both the text search types were mostly the same
  It was earlier done this way for expedience while experimenting
- The minor differences were reconciled and merged into a single
  text_search type
- This simplifies the app and making it easier to process other
  text types
This commit is contained in:
Debanjum Singh Solanky
2022-07-21 18:05:43 +04:00
parent 0917f1574d
commit 0602d018c0
9 changed files with 52 additions and 324 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/main.py
View File

@@ -11,7 +11,9 @@ from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
# Internal Packages
from src.search_type import asymmetric, symmetric_ledger, image_search
from src.search_type import image_search, text_search
from src.processor.org_mode.org_to_jsonl import org_to_jsonl
from src.processor.ledger.beancount_to_jsonl import beancount_to_jsonl
from src.utils.helpers import get_absolute_path, get_from_dict
from src.utils.cli import cli
from src.utils.config import SearchType, SearchModels, ProcessorConfigModel, ConversationProcessorConfigModel
@@ -66,24 +68,24 @@ def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None):
if (t == SearchType.Notes or t == None) and model.notes_search:
# query notes
hits, entries = asymmetric.query(user_query, model.notes_search, device=device, filters=[explicit_filter, date_filter])
hits, entries = text_search.query(user_query, model.notes_search, device=device, filters=[explicit_filter, date_filter])
# collate and return results
return asymmetric.collate_results(hits, entries, results_count)
return text_search.collate_results(hits, entries, results_count)
if (t == SearchType.Music or t == None) and model.music_search:
# query music library
hits, entries = asymmetric.query(user_query, model.music_search, device=device, filters=[explicit_filter, date_filter])
hits, entries = text_search.query(user_query, model.music_search, device=device, filters=[explicit_filter, date_filter])
# collate and return results
return asymmetric.collate_results(hits, entries, results_count)
return text_search.collate_results(hits, entries, results_count)
if (t == SearchType.Ledger or t == None) and model.ledger_search:
# query transactions
hits, entries = symmetric_ledger.query(user_query, model.ledger_search, filters=[explicit_filter, date_filter])
hits, entries = text_search.query(user_query, model.ledger_search, filters=[explicit_filter, date_filter])
# collate and return results
return symmetric_ledger.collate_results(hits, entries, results_count)
return text_search.collate_results(hits, entries, results_count)
if (t == SearchType.Image or t == None) and model.image_search:
# query transactions
@@ -163,17 +165,17 @@ def initialize_search(config: FullConfig, regenerate: bool, t: SearchType = None
# Initialize Org Notes Search
if (t == SearchType.Notes or t == None) and config.content_type.org:
# Extract Entries, Generate Notes Embeddings
model.notes_search = asymmetric.setup(config.content_type.org, search_config=config.search_type.asymmetric, regenerate=regenerate, device=device, verbose=verbose)
model.notes_search = text_search.setup(org_to_jsonl, config.content_type.org, search_config=config.search_type.asymmetric, regenerate=regenerate, device=device, verbose=verbose)
# Initialize Org Music Search
if (t == SearchType.Music or t == None) and config.content_type.music:
# Extract Entries, Generate Music Embeddings
model.music_search = asymmetric.setup(config.content_type.music, search_config=config.search_type.asymmetric, regenerate=regenerate, device=device, verbose=verbose)
model.music_search = text_search.setup(org_to_jsonl, config.content_type.music, search_config=config.search_type.asymmetric, regenerate=regenerate, device=device, verbose=verbose)
# Initialize Ledger Search
if (t == SearchType.Ledger or t == None) and config.content_type.ledger:
# Extract Entries, Generate Ledger Embeddings
model.ledger_search = symmetric_ledger.setup(config.content_type.ledger, search_config=config.search_type.symmetric, regenerate=regenerate, verbose=verbose)
model.ledger_search = text_search.setup(beancount_to_jsonl, config.content_type.ledger, search_config=config.search_type.symmetric, regenerate=regenerate, verbose=verbose)
# Initialize Image Search
if (t == SearchType.Image or t == None) and config.content_type.image: