diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5b67d2f4..dec6e5ec 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,7 +5,7 @@ on: branches: - master paths: - - src/** + - src/khoj/** - config/** - setup.py - Dockerfile diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 78e7bb66..c797b7bc 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,14 +7,14 @@ on: branches: - 'master' paths: - - src/** + - src/khoj/** - setup.py - .github/workflows/publish.yml pull_request: branches: - 'master' paths: - - src/** + - src/khoj/** - setup.py - .github/workflows/publish.yml @@ -33,7 +33,7 @@ jobs: - name: Install Dependencies run: | python -m pip install --upgrade pip - pip install build twine + pip install build twine check-wheel-contents - name: Install Application run: | @@ -49,10 +49,15 @@ jobs: export PYTHONHASHSEED=42 export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) - # Build and Upload PyPi Package + # Build PyPi Package rm -rf dist python -m build + + # Validate PyPi Package + check-wheel-contents dist/*.whl twine check dist/* + + # Upload PyPi Package twine upload --verbose dist/* - name: Publish Master to PyPI @@ -68,10 +73,15 @@ jobs: export PYTHONHASHSEED=42 export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) - # Build and Upload PyPi Package + # Build PyPi Package rm -rf dist python -m build + + # Validate PyPi Package + check-wheel-contents dist/*.whl twine check dist/* + + # Upload PyPi Package twine upload --verbose dist/* - name: Publish Repo PR to Test PyPI @@ -88,8 +98,13 @@ jobs: export PYTHONHASHSEED=42 export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) - # Build and Upload PyPi Package + # Build PyPi Package rm -rf dist python -m build + + # Validate PyPi Package + check-wheel-contents dist/*.whl twine check dist/* + + # Upload PyPi Package twine upload -r testpypi --verbose dist/* \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7b9213de..0e83970b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -66,7 +66,7 @@ jobs: # Create disk image with the app create-dmg \ --volname "Khoj" \ - --volicon "src/interface/web/assets/icons/favicon.icns" \ + --volicon "src/khoj/interface/web/assets/icons/favicon.icns" \ --window-pos 200 120 \ --window-size 600 300 \ --icon-size 100 \ @@ -92,7 +92,7 @@ jobs: # Copy app files into expected output directory structure mkdir -p package/opt package/usr/share/applications package/usr/share/icons/hicolor/128x128/apps cp -r dist/Khoj package/opt/Khoj - cp src/interface/web/assets/icons/favicon-128x128.png package/usr/share/icons/hicolor/128x128/apps/Khoj.png + cp src/khoj/interface/web/assets/icons/favicon-128x128.png package/usr/share/icons/hicolor/128x128/apps/Khoj.png cp Khoj.desktop package/usr/share/applications # Fix permissions to be usable by non-root users diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1e0e6ef0..fed245ee 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,7 +5,7 @@ on: branches: - 'master' paths: - - src/** + - src/khoj/** - tests/** - config/** - setup.py @@ -14,7 +14,7 @@ on: branches: - 'master' paths: - - src/** + - src/khoj/** - tests/** - config/** - setup.py diff --git a/.gitignore b/.gitignore index 3b1cfd35..555aba27 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ # Khoj artifacts *.gz *.pt -src/.data tests/data/models tests/data/embeddings @@ -13,10 +12,10 @@ __pycache__ .vscode # Build artifacts -/src/interface/web/images +/src/khoj/interface/web/images /build/ /dist/ -/khoj_assistant.egg-info/ +khoj_assistant.egg-info /config/khoj*.yml .pytest_cache khoj.log diff --git a/.mypy.ini b/.mypy.ini index 205d50d6..fd43bd47 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -5,9 +5,9 @@ install_types = True non_interactive = True show_error_codes = True exclude = (?x)( - src/interface/desktop/main_window.py - | src/interface/desktop/file_browser.py - | src/interface/desktop/system_tray.py + src/khoj/interface/desktop/main_window.py + | src/khoj/interface/desktop/file_browser.py + | src/khoj/interface/desktop/system_tray.py | build/* | tests/* ) diff --git a/Khoj.spec b/Khoj.spec index b191acd9..0e025ab8 100644 --- a/Khoj.spec +++ b/Khoj.spec @@ -5,7 +5,7 @@ from PyInstaller.utils.hooks import copy_metadata import sysconfig datas = [ - ('src/interface/web', 'src/interface/web'), + ('src/khoj/interface/web', 'src/khoj/interface/web'), (f'{sysconfig.get_paths()["purelib"]}/transformers', 'transformers') ] datas += copy_metadata('tqdm') @@ -19,7 +19,7 @@ datas += copy_metadata('tokenizers') block_cipher = None a = Analysis( - ['src/main.py'], + ['src/khoj/main.py'], pathex=[], binaries=[], datas=datas, @@ -50,7 +50,7 @@ pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) if system() != 'Darwin': # Add Splash screen to show on app launch splash = Splash( - 'src/interface/web/assets/icons/favicon-144x144.png', + 'src/khoj/interface/web/assets/icons/favicon-144x144.png', binaries=a.binaries, datas=a.datas, text_pos=(10, 160), @@ -82,7 +82,7 @@ if system() != 'Darwin': target_arch='x86_64', codesign_identity=None, entitlements_file=None, - icon='src/interface/web/assets/icons/favicon-144x144.ico', + icon='src/khoj/interface/web/assets/icons/favicon-144x144.ico', ) else: exe = EXE( @@ -105,11 +105,11 @@ else: target_arch='x86_64', codesign_identity=None, entitlements_file=None, - icon='src/interface/web/assets/icons/favicon.icns', + icon='src/khoj/interface/web/assets/icons/favicon.icns', ) app = BUNDLE( exe, name='Khoj.app', - icon='src/interface/web/assets/icons/favicon.icns', + icon='src/khoj/interface/web/assets/icons/favicon.icns', bundle_identifier=None, ) diff --git a/MANIFEST.in b/MANIFEST.in index 5628f3d3..d4301ef1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include README.md -graft src/interface/* -prune src/interface/web/images* +graft src/khoj/interface/* +prune src/khoj/interface/web/images* prune docs* global-exclude .DS_Store *.py[cod] \ No newline at end of file diff --git a/README.md b/README.md index 0b879b51..55c269fc 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ - **Incremental**: Incremental search for a fast, search-as-you-type experience - **Pluggable**: Modular architecture makes it easy to plug in new data sources, frontends and ML models - **Multiple Sources**: Search your Org-mode and Markdown notes, Beancount transactions and Photos -- **Multiple Interfaces**: Search using a [Web Browser](./src/interface/web/index.html), [Emacs](./src/interface/emacs/khoj.el) or the [API](http://localhost:8000/docs) +- **Multiple Interfaces**: Search from your [Web Browser](./src/khoj/interface/web/index.html), [Emacs](./src/interface/emacs/khoj.el) or [Obsidian](./src/interface/obsidian/) ## Demos ### Khoj in Obsidian @@ -238,7 +238,7 @@ pip install --upgrade khoj-assistant asymmetric: - encoder: "sentence-transformers/multi-qa-MiniLM-L6-cos-v1" + encoder: text-embedding-ada-002 - + encoder-type: src.utils.models.OpenAI + + encoder-type: src.khoj.utils.models.OpenAI cross-encoder: "cross-encoder/ms-marco-MiniLM-L-6-v2" - encoder-type: sentence_transformers.SentenceTransformer - model_directory: "~/.khoj/search/asymmetric/" @@ -408,7 +408,7 @@ python3 -m pip install pyqt6 # As conda does not support pyqt6 yet ##### 4. Run ```shell -python3 -m src.main -vv +python3 -m src.khoj.main -vv ``` Load ML model, generate embeddings and expose API to query notes, images, transactions etc specified in config YAML diff --git a/setup.py b/setup.py index 1f0f7fe3..2ea55970 100644 --- a/setup.py +++ b/setup.py @@ -17,10 +17,10 @@ setup( license="GPLv3", keywords="search semantic-search productivity NLP org-mode markdown beancount images", python_requires=">=3.8, <3.11", + package_dir={"": "src"}, packages=find_packages( - where=".", - exclude=["tests*"], - include=["src*"] + where="src", + include=["khoj*"] ), install_requires=[ "torch == 1.13.1", @@ -39,7 +39,7 @@ setup( 'schedule == 1.1.0', ], include_package_data=True, - entry_points={"console_scripts": ["khoj = src.main:run"]}, + entry_points={"console_scripts": ["khoj = khoj.main:run"]}, classifiers=[ "Development Status :: 4 - Beta", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", diff --git a/src/__init__.py b/src/khoj/__init__.py similarity index 100% rename from src/__init__.py rename to src/khoj/__init__.py diff --git a/src/configure.py b/src/khoj/configure.py similarity index 86% rename from src/configure.py rename to src/khoj/configure.py index c1132e8c..c776a4a0 100644 --- a/src/configure.py +++ b/src/khoj/configure.py @@ -7,17 +7,17 @@ import json import schedule # Internal Packages -from src.processor.ledger.beancount_to_jsonl import BeancountToJsonl -from src.processor.markdown.markdown_to_jsonl import MarkdownToJsonl -from src.processor.org_mode.org_to_jsonl import OrgToJsonl -from src.search_type import image_search, text_search -from src.utils.config import SearchType, SearchModels, ProcessorConfigModel, ConversationProcessorConfigModel -from src.utils import state -from src.utils.helpers import LRU, resolve_absolute_path -from src.utils.rawconfig import FullConfig, ProcessorConfig -from src.search_filter.date_filter import DateFilter -from src.search_filter.word_filter import WordFilter -from src.search_filter.file_filter import FileFilter +from khoj.processor.ledger.beancount_to_jsonl import BeancountToJsonl +from khoj.processor.markdown.markdown_to_jsonl import MarkdownToJsonl +from khoj.processor.org_mode.org_to_jsonl import OrgToJsonl +from khoj.search_type import image_search, text_search +from khoj.utils.config import SearchType, SearchModels, ProcessorConfigModel, ConversationProcessorConfigModel +from khoj.utils import state +from khoj.utils.helpers import LRU, resolve_absolute_path +from khoj.utils.rawconfig import FullConfig, ProcessorConfig +from khoj.search_filter.date_filter import DateFilter +from khoj.search_filter.word_filter import WordFilter +from khoj.search_filter.file_filter import FileFilter logger = logging.getLogger(__name__) diff --git a/src/interface/desktop/__init__.py b/src/khoj/interface/desktop/__init__.py similarity index 100% rename from src/interface/desktop/__init__.py rename to src/khoj/interface/desktop/__init__.py diff --git a/src/interface/desktop/file_browser.py b/src/khoj/interface/desktop/file_browser.py similarity index 96% rename from src/interface/desktop/file_browser.py rename to src/khoj/interface/desktop/file_browser.py index dca60bdf..70e6b8d7 100644 --- a/src/interface/desktop/file_browser.py +++ b/src/khoj/interface/desktop/file_browser.py @@ -3,8 +3,8 @@ from PyQt6 import QtWidgets from PyQt6.QtCore import QDir # Internal Packages -from src.utils.config import SearchType -from src.utils.helpers import is_none_or_empty +from khoj.utils.config import SearchType +from khoj.utils.helpers import is_none_or_empty class FileBrowser(QtWidgets.QWidget): diff --git a/src/interface/desktop/labelled_text_field.py b/src/khoj/interface/desktop/labelled_text_field.py similarity index 94% rename from src/interface/desktop/labelled_text_field.py rename to src/khoj/interface/desktop/labelled_text_field.py index 34634efd..3248c21c 100644 --- a/src/interface/desktop/labelled_text_field.py +++ b/src/khoj/interface/desktop/labelled_text_field.py @@ -2,7 +2,7 @@ from PyQt6 import QtWidgets # Internal Packages -from src.utils.config import ProcessorType +from khoj.utils.config import ProcessorType class LabelledTextField(QtWidgets.QWidget): diff --git a/src/interface/desktop/main_window.py b/src/khoj/interface/desktop/main_window.py similarity index 97% rename from src/interface/desktop/main_window.py rename to src/khoj/interface/desktop/main_window.py index 25287d27..6ee45fb6 100644 --- a/src/interface/desktop/main_window.py +++ b/src/khoj/interface/desktop/main_window.py @@ -9,13 +9,13 @@ from PyQt6 import QtGui, QtWidgets from PyQt6.QtCore import Qt, QThread, QObject, pyqtSignal # Internal Packages -from src.configure import configure_server -from src.interface.desktop.file_browser import FileBrowser -from src.interface.desktop.labelled_text_field import LabelledTextField -from src.utils import constants, state, yaml as yaml_utils -from src.utils.cli import cli -from src.utils.config import SearchType, ProcessorType -from src.utils.helpers import merge_dicts, resolve_absolute_path +from khoj.configure import configure_server +from khoj.interface.desktop.file_browser import FileBrowser +from khoj.interface.desktop.labelled_text_field import LabelledTextField +from khoj.utils import constants, state, yaml as yaml_utils +from khoj.utils.cli import cli +from khoj.utils.config import SearchType, ProcessorType +from khoj.utils.helpers import merge_dicts, resolve_absolute_path class MainWindow(QtWidgets.QMainWindow): diff --git a/src/interface/desktop/system_tray.py b/src/khoj/interface/desktop/system_tray.py similarity index 92% rename from src/interface/desktop/system_tray.py rename to src/khoj/interface/desktop/system_tray.py index 26df260f..c8559527 100644 --- a/src/interface/desktop/system_tray.py +++ b/src/khoj/interface/desktop/system_tray.py @@ -5,8 +5,8 @@ import webbrowser from PyQt6 import QtGui, QtWidgets # Internal Packages -from src.utils import constants, state -from src.interface.desktop.main_window import MainWindow +from khoj.utils import constants, state +from khoj.interface.desktop.main_window import MainWindow def create_system_tray(gui: QtWidgets.QApplication, main_window: MainWindow): diff --git a/src/interface/web/assets/config.css b/src/khoj/interface/web/assets/config.css similarity index 100% rename from src/interface/web/assets/config.css rename to src/khoj/interface/web/assets/config.css diff --git a/src/interface/web/assets/config.js b/src/khoj/interface/web/assets/config.js similarity index 100% rename from src/interface/web/assets/config.js rename to src/khoj/interface/web/assets/config.js diff --git a/src/interface/web/assets/icons/favicon-128x128.png b/src/khoj/interface/web/assets/icons/favicon-128x128.png similarity index 100% rename from src/interface/web/assets/icons/favicon-128x128.png rename to src/khoj/interface/web/assets/icons/favicon-128x128.png diff --git a/src/interface/web/assets/icons/favicon-144x144.ico b/src/khoj/interface/web/assets/icons/favicon-144x144.ico similarity index 100% rename from src/interface/web/assets/icons/favicon-144x144.ico rename to src/khoj/interface/web/assets/icons/favicon-144x144.ico diff --git a/src/interface/web/assets/icons/favicon-144x144.png b/src/khoj/interface/web/assets/icons/favicon-144x144.png similarity index 100% rename from src/interface/web/assets/icons/favicon-144x144.png rename to src/khoj/interface/web/assets/icons/favicon-144x144.png diff --git a/src/interface/web/assets/icons/favicon.icns b/src/khoj/interface/web/assets/icons/favicon.icns similarity index 100% rename from src/interface/web/assets/icons/favicon.icns rename to src/khoj/interface/web/assets/icons/favicon.icns diff --git a/src/interface/web/assets/markdown-it.min.js b/src/khoj/interface/web/assets/markdown-it.min.js similarity index 100% rename from src/interface/web/assets/markdown-it.min.js rename to src/khoj/interface/web/assets/markdown-it.min.js diff --git a/src/interface/web/assets/org.min.js b/src/khoj/interface/web/assets/org.min.js similarity index 100% rename from src/interface/web/assets/org.min.js rename to src/khoj/interface/web/assets/org.min.js diff --git a/src/interface/web/chat.html b/src/khoj/interface/web/chat.html similarity index 100% rename from src/interface/web/chat.html rename to src/khoj/interface/web/chat.html diff --git a/src/interface/web/config.html b/src/khoj/interface/web/config.html similarity index 100% rename from src/interface/web/config.html rename to src/khoj/interface/web/config.html diff --git a/src/interface/web/index.html b/src/khoj/interface/web/index.html similarity index 100% rename from src/interface/web/index.html rename to src/khoj/interface/web/index.html diff --git a/src/interface/web/khoj.webmanifest b/src/khoj/interface/web/khoj.webmanifest similarity index 100% rename from src/interface/web/khoj.webmanifest rename to src/khoj/interface/web/khoj.webmanifest diff --git a/src/main.py b/src/khoj/main.py similarity index 91% rename from src/main.py rename to src/khoj/main.py index 0c5937f3..8461f1b2 100644 --- a/src/main.py +++ b/src/khoj/main.py @@ -20,15 +20,15 @@ from PyQt6.QtCore import QThread, QTimer import schedule # Internal Packages -from src.configure import configure_server -from src.routers.api import api -from src.routers.api_beta import api_beta -from src.routers.web_client import web_client -from src.utils import constants, state -from src.utils.cli import cli -from src.utils.helpers import CustomFormatter -from src.interface.desktop.main_window import MainWindow -from src.interface.desktop.system_tray import create_system_tray +from khoj.configure import configure_server +from khoj.routers.api import api +from khoj.routers.api_beta import api_beta +from khoj.routers.web_client import web_client +from khoj.utils import constants, state +from khoj.utils.cli import cli +from khoj.utils.helpers import CustomFormatter +from khoj.interface.desktop.main_window import MainWindow +from khoj.interface.desktop.system_tray import create_system_tray # Initialize the Application Server @@ -38,7 +38,7 @@ app.include_router(api, prefix="/api") app.include_router(api_beta, prefix="/api/beta") app.include_router(web_client) -logger = logging.getLogger('src') +logger = logging.getLogger('khoj') def run(): diff --git a/src/processor/__init__.py b/src/khoj/processor/__init__.py similarity index 100% rename from src/processor/__init__.py rename to src/khoj/processor/__init__.py diff --git a/src/processor/conversation/__init__.py b/src/khoj/processor/conversation/__init__.py similarity index 100% rename from src/processor/conversation/__init__.py rename to src/khoj/processor/conversation/__init__.py diff --git a/src/processor/conversation/gpt.py b/src/khoj/processor/conversation/gpt.py similarity index 99% rename from src/processor/conversation/gpt.py rename to src/khoj/processor/conversation/gpt.py index 8a813a1c..a274074f 100644 --- a/src/processor/conversation/gpt.py +++ b/src/khoj/processor/conversation/gpt.py @@ -7,7 +7,7 @@ from datetime import datetime import openai # Internal Packages -from src.utils.constants import empty_escape_sequences +from khoj.utils.constants import empty_escape_sequences def summarize(text, summary_type, model, user_query=None, api_key=None, temperature=0.5, max_tokens=200): diff --git a/src/processor/ledger/__init__.py b/src/khoj/processor/ledger/__init__.py similarity index 100% rename from src/processor/ledger/__init__.py rename to src/khoj/processor/ledger/__init__.py diff --git a/src/processor/ledger/beancount_to_jsonl.py b/src/khoj/processor/ledger/beancount_to_jsonl.py similarity index 94% rename from src/processor/ledger/beancount_to_jsonl.py rename to src/khoj/processor/ledger/beancount_to_jsonl.py index c88b66c2..2703304f 100644 --- a/src/processor/ledger/beancount_to_jsonl.py +++ b/src/khoj/processor/ledger/beancount_to_jsonl.py @@ -5,11 +5,11 @@ import logging from typing import List # Internal Packages -from src.processor.text_to_jsonl import TextToJsonl -from src.utils.helpers import get_absolute_path, is_none_or_empty, timer -from src.utils.constants import empty_escape_sequences -from src.utils.jsonl import dump_jsonl, compress_jsonl_data -from src.utils.rawconfig import Entry +from khoj.processor.text_to_jsonl import TextToJsonl +from khoj.utils.helpers import get_absolute_path, is_none_or_empty, timer +from khoj.utils.constants import empty_escape_sequences +from khoj.utils.jsonl import dump_jsonl, compress_jsonl_data +from khoj.utils.rawconfig import Entry logger = logging.getLogger(__name__) diff --git a/src/processor/markdown/__init__.py b/src/khoj/processor/markdown/__init__.py similarity index 100% rename from src/processor/markdown/__init__.py rename to src/khoj/processor/markdown/__init__.py diff --git a/src/processor/markdown/markdown_to_jsonl.py b/src/khoj/processor/markdown/markdown_to_jsonl.py similarity index 94% rename from src/processor/markdown/markdown_to_jsonl.py rename to src/khoj/processor/markdown/markdown_to_jsonl.py index 822cad0c..98e5d924 100644 --- a/src/processor/markdown/markdown_to_jsonl.py +++ b/src/khoj/processor/markdown/markdown_to_jsonl.py @@ -6,11 +6,11 @@ import time from typing import List # Internal Packages -from src.processor.text_to_jsonl import TextToJsonl -from src.utils.helpers import get_absolute_path, is_none_or_empty, timer -from src.utils.constants import empty_escape_sequences -from src.utils.jsonl import dump_jsonl, compress_jsonl_data -from src.utils.rawconfig import Entry +from khoj.processor.text_to_jsonl import TextToJsonl +from khoj.utils.helpers import get_absolute_path, is_none_or_empty, timer +from khoj.utils.constants import empty_escape_sequences +from khoj.utils.jsonl import dump_jsonl, compress_jsonl_data +from khoj.utils.rawconfig import Entry logger = logging.getLogger(__name__) diff --git a/src/processor/org_mode/__init__.py b/src/khoj/processor/org_mode/__init__.py similarity index 100% rename from src/processor/org_mode/__init__.py rename to src/khoj/processor/org_mode/__init__.py diff --git a/src/processor/org_mode/org_to_jsonl.py b/src/khoj/processor/org_mode/org_to_jsonl.py similarity index 95% rename from src/processor/org_mode/org_to_jsonl.py rename to src/khoj/processor/org_mode/org_to_jsonl.py index 2e227de5..2938413f 100644 --- a/src/processor/org_mode/org_to_jsonl.py +++ b/src/khoj/processor/org_mode/org_to_jsonl.py @@ -5,12 +5,12 @@ import time from typing import Iterable, List # Internal Packages -from src.processor.org_mode import orgnode -from src.processor.text_to_jsonl import TextToJsonl -from src.utils.helpers import get_absolute_path, is_none_or_empty, timer -from src.utils.jsonl import dump_jsonl, compress_jsonl_data -from src.utils.rawconfig import Entry -from src.utils import state +from khoj.processor.org_mode import orgnode +from khoj.processor.text_to_jsonl import TextToJsonl +from khoj.utils.helpers import get_absolute_path, is_none_or_empty, timer +from khoj.utils.jsonl import dump_jsonl, compress_jsonl_data +from khoj.utils.rawconfig import Entry +from khoj.utils import state logger = logging.getLogger(__name__) diff --git a/src/processor/org_mode/orgnode.py b/src/khoj/processor/org_mode/orgnode.py similarity index 100% rename from src/processor/org_mode/orgnode.py rename to src/khoj/processor/org_mode/orgnode.py diff --git a/src/processor/text_to_jsonl.py b/src/khoj/processor/text_to_jsonl.py similarity index 97% rename from src/processor/text_to_jsonl.py rename to src/khoj/processor/text_to_jsonl.py index 33a6a515..80063128 100644 --- a/src/processor/text_to_jsonl.py +++ b/src/khoj/processor/text_to_jsonl.py @@ -3,10 +3,10 @@ from abc import ABC, abstractmethod import hashlib import logging from typing import Callable, List, Tuple -from src.utils.helpers import timer +from khoj.utils.helpers import timer # Internal Packages -from src.utils.rawconfig import Entry, TextContentConfig +from khoj.utils.rawconfig import Entry, TextContentConfig logger = logging.getLogger(__name__) diff --git a/src/routers/__init__.py b/src/khoj/routers/__init__.py similarity index 100% rename from src/routers/__init__.py rename to src/khoj/routers/__init__.py diff --git a/src/routers/api.py b/src/khoj/routers/api.py similarity index 93% rename from src/routers/api.py rename to src/khoj/routers/api.py index ca9b08d4..ea9b3334 100644 --- a/src/routers/api.py +++ b/src/khoj/routers/api.py @@ -8,12 +8,12 @@ from fastapi import APIRouter from fastapi import HTTPException # Internal Packages -from src.configure import configure_processor, configure_search -from src.search_type import image_search, text_search -from src.utils.helpers import timer -from src.utils.rawconfig import FullConfig, SearchResponse -from src.utils.config import SearchType -from src.utils import state, constants +from khoj.configure import configure_processor, configure_search +from khoj.search_type import image_search, text_search +from khoj.utils.helpers import timer +from khoj.utils.rawconfig import FullConfig, SearchResponse +from khoj.utils.config import SearchType +from khoj.utils import state, constants # Initialize Router diff --git a/src/routers/api_beta.py b/src/khoj/routers/api_beta.py similarity index 94% rename from src/routers/api_beta.py rename to src/khoj/routers/api_beta.py index d0555c04..7e4d4c7a 100644 --- a/src/routers/api_beta.py +++ b/src/khoj/routers/api_beta.py @@ -8,11 +8,11 @@ import schedule from fastapi import APIRouter # Internal Packages -from src.routers.api import search -from src.processor.conversation.gpt import converse, extract_search_type, message_to_log, message_to_prompt, understand, summarize -from src.utils.config import SearchType -from src.utils.helpers import get_from_dict, resolve_absolute_path -from src.utils import state +from khoj.routers.api import search +from khoj.processor.conversation.gpt import converse, extract_search_type, message_to_log, message_to_prompt, understand, summarize +from khoj.utils.config import SearchType +from khoj.utils.helpers import get_from_dict, resolve_absolute_path +from khoj.utils import state # Initialize Router diff --git a/src/routers/web_client.py b/src/khoj/routers/web_client.py similarity index 95% rename from src/routers/web_client.py rename to src/khoj/routers/web_client.py index 4a22e7f7..79e47375 100644 --- a/src/routers/web_client.py +++ b/src/khoj/routers/web_client.py @@ -5,7 +5,7 @@ from fastapi.responses import HTMLResponse, FileResponse from fastapi.templating import Jinja2Templates # Internal Packages -from src.utils import constants +from khoj.utils import constants # Initialize Router diff --git a/src/search_filter/__init__.py b/src/khoj/search_filter/__init__.py similarity index 100% rename from src/search_filter/__init__.py rename to src/khoj/search_filter/__init__.py diff --git a/src/search_filter/base_filter.py b/src/khoj/search_filter/base_filter.py similarity index 91% rename from src/search_filter/base_filter.py rename to src/khoj/search_filter/base_filter.py index 5b7c7f60..c27a051c 100644 --- a/src/search_filter/base_filter.py +++ b/src/khoj/search_filter/base_filter.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod from typing import List, Set, Tuple # Internal Packages -from src.utils.rawconfig import Entry +from khoj.utils.rawconfig import Entry class BaseFilter(ABC): diff --git a/src/search_filter/date_filter.py b/src/khoj/search_filter/date_filter.py similarity index 98% rename from src/search_filter/date_filter.py rename to src/khoj/search_filter/date_filter.py index bcacc190..13067562 100644 --- a/src/search_filter/date_filter.py +++ b/src/khoj/search_filter/date_filter.py @@ -11,8 +11,8 @@ from math import inf import dateparser as dtparse # Internal Packages -from src.search_filter.base_filter import BaseFilter -from src.utils.helpers import LRU, timer +from khoj.search_filter.base_filter import BaseFilter +from khoj.utils.helpers import LRU, timer logger = logging.getLogger(__name__) diff --git a/src/search_filter/file_filter.py b/src/khoj/search_filter/file_filter.py similarity index 96% rename from src/search_filter/file_filter.py rename to src/khoj/search_filter/file_filter.py index 35fb078a..970da150 100644 --- a/src/search_filter/file_filter.py +++ b/src/khoj/search_filter/file_filter.py @@ -6,8 +6,8 @@ import logging from collections import defaultdict # Internal Packages -from src.search_filter.base_filter import BaseFilter -from src.utils.helpers import LRU, timer +from khoj.search_filter.base_filter import BaseFilter +from khoj.utils.helpers import LRU, timer logger = logging.getLogger(__name__) diff --git a/src/search_filter/word_filter.py b/src/khoj/search_filter/word_filter.py similarity index 97% rename from src/search_filter/word_filter.py rename to src/khoj/search_filter/word_filter.py index 684847b3..16ed633d 100644 --- a/src/search_filter/word_filter.py +++ b/src/khoj/search_filter/word_filter.py @@ -5,8 +5,8 @@ import logging from collections import defaultdict # Internal Packages -from src.search_filter.base_filter import BaseFilter -from src.utils.helpers import LRU, timer +from khoj.search_filter.base_filter import BaseFilter +from khoj.utils.helpers import LRU, timer logger = logging.getLogger(__name__) diff --git a/src/search_type/__init__.py b/src/khoj/search_type/__init__.py similarity index 100% rename from src/search_type/__init__.py rename to src/khoj/search_type/__init__.py diff --git a/src/search_type/image_search.py b/src/khoj/search_type/image_search.py similarity index 97% rename from src/search_type/image_search.py rename to src/khoj/search_type/image_search.py index db358988..d50b52ec 100644 --- a/src/search_type/image_search.py +++ b/src/khoj/search_type/image_search.py @@ -14,9 +14,9 @@ from tqdm import trange import torch # Internal Packages -from src.utils.helpers import get_absolute_path, get_from_dict, resolve_absolute_path, load_model, timer -from src.utils.config import ImageSearchModel -from src.utils.rawconfig import ImageContentConfig, ImageSearchConfig, SearchResponse +from khoj.utils.helpers import get_absolute_path, get_from_dict, resolve_absolute_path, load_model, timer +from khoj.utils.config import ImageSearchModel +from khoj.utils.rawconfig import ImageContentConfig, ImageSearchConfig, SearchResponse # Create Logger diff --git a/src/search_type/text_search.py b/src/khoj/search_type/text_search.py similarity index 95% rename from src/search_type/text_search.py rename to src/khoj/search_type/text_search.py index 46f83638..bd5715b6 100644 --- a/src/search_type/text_search.py +++ b/src/khoj/search_type/text_search.py @@ -7,16 +7,16 @@ from typing import List, Tuple, Type # External Packages import torch from sentence_transformers import SentenceTransformer, CrossEncoder, util -from src.processor.text_to_jsonl import TextToJsonl -from src.search_filter.base_filter import BaseFilter +from khoj.processor.text_to_jsonl import TextToJsonl +from khoj.search_filter.base_filter import BaseFilter # Internal Packages -from src.utils import state -from src.utils.helpers import get_absolute_path, is_none_or_empty, resolve_absolute_path, load_model, timer -from src.utils.config import TextSearchModel -from src.utils.models import BaseEncoder -from src.utils.rawconfig import SearchResponse, TextSearchConfig, TextContentConfig, Entry -from src.utils.jsonl import load_jsonl +from khoj.utils import state +from khoj.utils.helpers import get_absolute_path, is_none_or_empty, resolve_absolute_path, load_model, timer +from khoj.utils.config import TextSearchModel +from khoj.utils.models import BaseEncoder +from khoj.utils.rawconfig import SearchResponse, TextSearchConfig, TextContentConfig, Entry +from khoj.utils.jsonl import load_jsonl logger = logging.getLogger(__name__) diff --git a/src/utils/__init__.py b/src/khoj/utils/__init__.py similarity index 100% rename from src/utils/__init__.py rename to src/khoj/utils/__init__.py diff --git a/src/utils/cli.py b/src/khoj/utils/cli.py similarity index 94% rename from src/utils/cli.py rename to src/khoj/utils/cli.py index 1f66a757..ab2e7af9 100644 --- a/src/utils/cli.py +++ b/src/khoj/utils/cli.py @@ -4,8 +4,8 @@ import pathlib from importlib.metadata import version # Internal Packages -from src.utils.helpers import resolve_absolute_path -from src.utils.yaml import parse_config_from_file +from khoj.utils.helpers import resolve_absolute_path +from khoj.utils.yaml import parse_config_from_file def cli(args=None): diff --git a/src/utils/config.py b/src/khoj/utils/config.py similarity index 92% rename from src/utils/config.py rename to src/khoj/utils/config.py index e1f3ddb4..22193f9e 100644 --- a/src/utils/config.py +++ b/src/khoj/utils/config.py @@ -11,9 +11,9 @@ import torch # Internal Packages if TYPE_CHECKING: from sentence_transformers import CrossEncoder - from src.search_filter.base_filter import BaseFilter - from src.utils.models import BaseEncoder - from src.utils.rawconfig import ConversationProcessorConfig, Entry + from khoj.search_filter.base_filter import BaseFilter + from khoj.utils.models import BaseEncoder + from khoj.utils.rawconfig import ConversationProcessorConfig, Entry class SearchType(str, Enum): diff --git a/src/utils/constants.py b/src/khoj/utils/constants.py similarity index 97% rename from src/utils/constants.py rename to src/khoj/utils/constants.py index e4840134..86686e93 100644 --- a/src/utils/constants.py +++ b/src/khoj/utils/constants.py @@ -1,7 +1,7 @@ from pathlib import Path app_root_directory = Path(__file__).parent.parent.parent -web_directory = app_root_directory / 'src/interface/web/' +web_directory = app_root_directory / 'khoj/interface/web/' empty_escape_sequences = '\n|\r|\t| ' # default app config to use diff --git a/src/utils/helpers.py b/src/khoj/utils/helpers.py similarity index 99% rename from src/utils/helpers.py rename to src/khoj/utils/helpers.py index 5ecbfef7..b361efd2 100644 --- a/src/utils/helpers.py +++ b/src/khoj/utils/helpers.py @@ -14,7 +14,7 @@ if TYPE_CHECKING: # External Packages from sentence_transformers import CrossEncoder # Internal Packages - from src.utils.models import BaseEncoder + from khoj.utils.models import BaseEncoder def is_none_or_empty(item): diff --git a/src/utils/jsonl.py b/src/khoj/utils/jsonl.py similarity index 93% rename from src/utils/jsonl.py rename to src/khoj/utils/jsonl.py index 5874c17c..41923186 100644 --- a/src/utils/jsonl.py +++ b/src/khoj/utils/jsonl.py @@ -4,8 +4,8 @@ import gzip import logging # Internal Packages -from src.utils.constants import empty_escape_sequences -from src.utils.helpers import get_absolute_path +from khoj.utils.constants import empty_escape_sequences +from khoj.utils.helpers import get_absolute_path logger = logging.getLogger(__name__) diff --git a/src/utils/models.py b/src/khoj/utils/models.py similarity index 97% rename from src/utils/models.py rename to src/khoj/utils/models.py index 53525566..77e620fd 100644 --- a/src/utils/models.py +++ b/src/khoj/utils/models.py @@ -8,7 +8,7 @@ import torch from tqdm import trange # Internal Packages -from src.utils.state import processor_config, config_file +from khoj.utils.state import processor_config, config_file class BaseEncoder(ABC): diff --git a/src/utils/rawconfig.py b/src/khoj/utils/rawconfig.py similarity index 97% rename from src/utils/rawconfig.py rename to src/khoj/utils/rawconfig.py index c814726e..82715617 100644 --- a/src/utils/rawconfig.py +++ b/src/khoj/utils/rawconfig.py @@ -7,7 +7,7 @@ from typing import List, Optional from pydantic import BaseModel, validator # Internal Packages -from src.utils.helpers import to_snake_case_from_dash, is_none_or_empty +from khoj.utils.helpers import to_snake_case_from_dash, is_none_or_empty class ConfigBase(BaseModel): class Config: diff --git a/src/utils/state.py b/src/khoj/utils/state.py similarity index 83% rename from src/utils/state.py rename to src/khoj/utils/state.py index 8574c92d..7e6abc1e 100644 --- a/src/utils/state.py +++ b/src/khoj/utils/state.py @@ -8,9 +8,9 @@ import torch from pathlib import Path # Internal Packages -from src.utils.config import SearchModels, ProcessorConfigModel -from src.utils.helpers import LRU -from src.utils.rawconfig import FullConfig +from khoj.utils.config import SearchModels, ProcessorConfigModel +from khoj.utils.helpers import LRU +from khoj.utils.rawconfig import FullConfig # Application Global State config = FullConfig() diff --git a/src/utils/yaml.py b/src/khoj/utils/yaml.py similarity index 96% rename from src/utils/yaml.py rename to src/khoj/utils/yaml.py index 07f7cd87..ca2a3aaf 100644 --- a/src/utils/yaml.py +++ b/src/khoj/utils/yaml.py @@ -5,7 +5,7 @@ from pathlib import Path import yaml # Internal Packages -from src.utils.rawconfig import FullConfig +from khoj.utils.rawconfig import FullConfig # Do not emit tags when dumping to YAML diff --git a/tests/conftest.py b/tests/conftest.py index 538e9206..cad2cb58 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,13 +4,13 @@ from pathlib import Path import pytest # Internal Packages -from src.search_type import image_search, text_search -from src.utils.helpers import resolve_absolute_path -from src.utils.rawconfig import ContentConfig, TextContentConfig, ImageContentConfig, SearchConfig, TextSearchConfig, ImageSearchConfig -from src.processor.org_mode.org_to_jsonl import OrgToJsonl -from src.search_filter.date_filter import DateFilter -from src.search_filter.word_filter import WordFilter -from src.search_filter.file_filter import FileFilter +from khoj.search_type import image_search, text_search +from khoj.utils.helpers import resolve_absolute_path +from khoj.utils.rawconfig import ContentConfig, TextContentConfig, ImageContentConfig, SearchConfig, TextSearchConfig, ImageSearchConfig +from khoj.processor.org_mode.org_to_jsonl import OrgToJsonl +from khoj.search_filter.date_filter import DateFilter +from khoj.search_filter.word_filter import WordFilter +from khoj.search_filter.file_filter import FileFilter @pytest.fixture(scope='session') diff --git a/tests/data/markdown/main_readme.md b/tests/data/markdown/main_readme.md index 14d97a97..d2f11e9e 100644 --- a/tests/data/markdown/main_readme.md +++ b/tests/data/markdown/main_readme.md @@ -6,8 +6,8 @@ *Allow natural language search on user content like notes, images, transactions using transformer ML models* -User can interface with Khoj via [Web](./src/interface/web/index.html), -[Emacs](./src/interface/emacs/khoj.el) or the API. All search is done +User can interface with Khoj via [Web](./src/khoj/interface/web/index.html), +[Emacs](./src/khoj/interface/emacs/khoj.el) or the API. All search is done locally[\*](https://github.com/debanjum/khoj#miscellaneous) ## Demo @@ -47,8 +47,8 @@ just generating embeddings* - [Update Index](http://localhost:8000/api/update?t=ledger) - [Configure Application](https://localhost:8000/ui) - **Khoj via Emacs** - - [Install](https://github.com/debanjum/khoj/tree/master/src/interface/emacs#installation) - [khoj.el](./src/interface/emacs/khoj.el) + - [Install](https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#installation) + [khoj.el](./src/khoj/interface/emacs/khoj.el) - Run `M-x khoj ` ## Run Unit tests @@ -118,7 +118,7 @@ docker-compose build --pull images, transactions etc specified in config YAML ``` shell - python3 -m src.main -c=config/khoj_sample.yml -vv + python3 -m src.khoj.main -c=config/khoj_sample.yml -vv ``` ### Upgrade On Local Machine diff --git a/tests/test_beancount_to_jsonl.py b/tests/test_beancount_to_jsonl.py index 2c1cb9e6..7150ea35 100644 --- a/tests/test_beancount_to_jsonl.py +++ b/tests/test_beancount_to_jsonl.py @@ -2,7 +2,7 @@ import json # Internal Packages -from src.processor.ledger.beancount_to_jsonl import BeancountToJsonl +from khoj.processor.ledger.beancount_to_jsonl import BeancountToJsonl def test_no_transactions_in_file(tmp_path): diff --git a/tests/test_chatbot.py b/tests/test_chatbot.py index ea01eb2a..4a135061 100644 --- a/tests/test_chatbot.py +++ b/tests/test_chatbot.py @@ -2,7 +2,7 @@ import pytest # Internal Packages -from src.processor.conversation.gpt import converse, understand, message_to_prompt +from khoj.processor.conversation.gpt import converse, understand, message_to_prompt # Initialize variables for tests diff --git a/tests/test_cli.py b/tests/test_cli.py index 3c99f424..f51f76cb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,8 +3,8 @@ from pathlib import Path from random import random # Internal Packages -from src.utils.cli import cli -from src.utils.helpers import resolve_absolute_path +from khoj.utils.cli import cli +from khoj.utils.helpers import resolve_absolute_path # Test diff --git a/tests/test_client.py b/tests/test_client.py index d3dde245..ac94bc27 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -8,13 +8,13 @@ from urllib.parse import quote from fastapi.testclient import TestClient # Internal Packages -from src.main import app -from src.utils.state import model, config -from src.search_type import text_search, image_search -from src.utils.rawconfig import ContentConfig, SearchConfig -from src.processor.org_mode.org_to_jsonl import OrgToJsonl -from src.search_filter.word_filter import WordFilter -from src.search_filter.file_filter import FileFilter +from khoj.main import app +from khoj.utils.state import model, config +from khoj.search_type import text_search, image_search +from khoj.utils.rawconfig import ContentConfig, SearchConfig +from khoj.processor.org_mode.org_to_jsonl import OrgToJsonl +from khoj.search_filter.word_filter import WordFilter +from khoj.search_filter.file_filter import FileFilter # Arrange diff --git a/tests/test_date_filter.py b/tests/test_date_filter.py index bc656701..480719fd 100644 --- a/tests/test_date_filter.py +++ b/tests/test_date_filter.py @@ -4,8 +4,8 @@ from datetime import datetime from math import inf # Application Packages -from src.search_filter.date_filter import DateFilter -from src.utils.rawconfig import Entry +from khoj.search_filter.date_filter import DateFilter +from khoj.utils.rawconfig import Entry def test_date_filter(): diff --git a/tests/test_file_filter.py b/tests/test_file_filter.py index 28b0367f..7ac98f56 100644 --- a/tests/test_file_filter.py +++ b/tests/test_file_filter.py @@ -1,6 +1,6 @@ # Application Packages -from src.search_filter.file_filter import FileFilter -from src.utils.rawconfig import Entry +from khoj.search_filter.file_filter import FileFilter +from khoj.utils.rawconfig import Entry def test_no_file_filter(): diff --git a/tests/test_helpers.py b/tests/test_helpers.py index c9b1cd75..2ee1569e 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,4 +1,4 @@ -from src.utils import helpers +from khoj.utils import helpers def test_get_from_null_dict(): # null handling diff --git a/tests/test_image_search.py b/tests/test_image_search.py index 77000d60..cb19a55a 100644 --- a/tests/test_image_search.py +++ b/tests/test_image_search.py @@ -4,11 +4,11 @@ from pathlib import Path from PIL import Image # Internal Packages -from src.utils.state import model -from src.utils.constants import web_directory -from src.search_type import image_search -from src.utils.helpers import resolve_absolute_path -from src.utils.rawconfig import ContentConfig, SearchConfig +from khoj.utils.state import model +from khoj.utils.constants import web_directory +from khoj.search_type import image_search +from khoj.utils.helpers import resolve_absolute_path +from khoj.utils.rawconfig import ContentConfig, SearchConfig # Test @@ -91,7 +91,7 @@ def test_image_search_query_truncated(content_config: ContentConfig, search_conf # Act try: - with caplog.at_level(logging.INFO, logger="src.search_type.image_search"): + with caplog.at_level(logging.INFO, logger="khoj.search_type.image_search"): image_search.query( query, count = 1, @@ -114,7 +114,7 @@ def test_image_search_by_filepath(content_config: ContentConfig, search_config: expected_image_path = f"{image_directory.joinpath('kitten_park.jpg')}" # Act - with caplog.at_level(logging.INFO, logger="src.search_type.image_search"): + with caplog.at_level(logging.INFO, logger="khoj.search_type.image_search"): hits = image_search.query( query, count = 1, diff --git a/tests/test_markdown_to_jsonl.py b/tests/test_markdown_to_jsonl.py index e7cae1ae..a019ef5e 100644 --- a/tests/test_markdown_to_jsonl.py +++ b/tests/test_markdown_to_jsonl.py @@ -2,7 +2,7 @@ import json # Internal Packages -from src.processor.markdown.markdown_to_jsonl import MarkdownToJsonl +from khoj.processor.markdown.markdown_to_jsonl import MarkdownToJsonl def test_markdown_file_with_no_headings_to_jsonl(tmp_path): diff --git a/tests/test_org_to_jsonl.py b/tests/test_org_to_jsonl.py index c1e816f4..62cab75b 100644 --- a/tests/test_org_to_jsonl.py +++ b/tests/test_org_to_jsonl.py @@ -2,10 +2,10 @@ import json # Internal Packages -from src.processor.org_mode.org_to_jsonl import OrgToJsonl -from src.processor.text_to_jsonl import TextToJsonl -from src.utils.helpers import is_none_or_empty -from src.utils.rawconfig import Entry +from khoj.processor.org_mode.org_to_jsonl import OrgToJsonl +from khoj.processor.text_to_jsonl import TextToJsonl +from khoj.utils.helpers import is_none_or_empty +from khoj.utils.rawconfig import Entry def test_configure_heading_entry_to_jsonl(tmp_path): diff --git a/tests/test_orgnode.py b/tests/test_orgnode.py index d36cca79..af67deed 100644 --- a/tests/test_orgnode.py +++ b/tests/test_orgnode.py @@ -2,7 +2,7 @@ import datetime # Internal Packages -from src.processor.org_mode import orgnode +from khoj.processor.org_mode import orgnode # Test diff --git a/tests/test_text_search.py b/tests/test_text_search.py index 8b162874..412bceec 100644 --- a/tests/test_text_search.py +++ b/tests/test_text_search.py @@ -6,10 +6,10 @@ from pathlib import Path import pytest # Internal Packages -from src.utils.state import model -from src.search_type import text_search -from src.utils.rawconfig import ContentConfig, SearchConfig, TextContentConfig -from src.processor.org_mode.org_to_jsonl import OrgToJsonl +from khoj.utils.state import model +from khoj.search_type import text_search +from khoj.utils.rawconfig import ContentConfig, SearchConfig, TextContentConfig +from khoj.processor.org_mode.org_to_jsonl import OrgToJsonl # Test diff --git a/tests/test_word_filter.py b/tests/test_word_filter.py index 2e662fd0..4a7c894b 100644 --- a/tests/test_word_filter.py +++ b/tests/test_word_filter.py @@ -1,6 +1,6 @@ # Application Packages -from src.search_filter.word_filter import WordFilter -from src.utils.rawconfig import Entry +from khoj.search_filter.word_filter import WordFilter +from khoj.utils.rawconfig import Entry def test_no_word_filter():