Use src layout to fix packaging khoj for pypi

### Issue
The khoj python package was using a common top level name[1], `src' instead of `khoj' due to incorrect usage of the src layout[2]

### Fix
Put content meant for python packaging from `src/' to `src/khoj/'
Update code, tests, configs and docs to reference new layout

The `khoj' python package should now get unpacked under `khoj' instead of `src' directory

### Details
- 25a749c Use the src/ layout to fix packaging Khoj for PyPi
- bc7477e Move Emacs, Obsidian plugin code out from under src/khoj directory
- f83cf4e Check wheel contents in workflow before publishing Khoj to PyPI

[1]: https://github.com/jwodder/check-wheel-contents#w005--wheel-contains-common-toplevel-name-in-library
[2]: https://packaging.python.org/en/latest/discussions/src-layout-vs-flat-layout/

.github/workflows/build.yml

@@ -5,7 +5,7 @@ on:
     branches:
       - master
     paths:
-      - src/**
+      - src/khoj/**
       - config/**
       - setup.py
       - Dockerfile

.github/workflows/publish.yml

@@ -7,14 +7,14 @@ on:
     branches:
       - 'master'
     paths:
-      - src/**
+      - src/khoj/**
       - setup.py
       - .github/workflows/publish.yml
   pull_request:
     branches:
       - 'master'
     paths:
-      - src/**
+      - src/khoj/**
       - setup.py
       - .github/workflows/publish.yml
 
@@ -33,7 +33,7 @@ jobs:
       - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install build twine
+          pip install build twine check-wheel-contents
 
       - name: Install Application
         run: |
@@ -49,10 +49,15 @@ jobs:
           export PYTHONHASHSEED=42
           export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
 
-          # Build and Upload PyPi Package
+          # Build PyPi Package
           rm -rf dist
           python -m build
+
+          # Validate PyPi Package
+          check-wheel-contents dist/*.whl
           twine check dist/*
+
+          # Upload PyPi Package
           twine upload --verbose dist/*
 
       - name: Publish Master to PyPI
@@ -68,10 +73,15 @@ jobs:
           export PYTHONHASHSEED=42
           export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
 
-          # Build and Upload PyPi Package
+          # Build PyPi Package
           rm -rf dist
           python -m build
+
+          # Validate PyPi Package
+          check-wheel-contents dist/*.whl
           twine check dist/*
+
+          # Upload PyPi Package
           twine upload --verbose dist/*
 
       - name: Publish Repo PR to Test PyPI
@@ -88,8 +98,13 @@ jobs:
           export PYTHONHASHSEED=42
           export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
 
-          # Build and Upload PyPi Package
+          # Build PyPi Package
           rm -rf dist
           python -m build
+
+          # Validate PyPi Package
+          check-wheel-contents dist/*.whl
           twine check dist/*
+
+          # Upload PyPi Package
           twine upload -r testpypi --verbose dist/*

.github/workflows/release.yml

@@ -66,7 +66,7 @@ jobs:
           # Create disk image with the app
           create-dmg \
             --volname "Khoj" \
-            --volicon "src/interface/web/assets/icons/favicon.icns" \
+            --volicon "src/khoj/interface/web/assets/icons/favicon.icns" \
             --window-pos 200 120 \
             --window-size 600 300 \
             --icon-size 100 \
@@ -92,7 +92,7 @@ jobs:
           # Copy app files into expected output directory structure
           mkdir -p package/opt package/usr/share/applications package/usr/share/icons/hicolor/128x128/apps
           cp -r dist/Khoj package/opt/Khoj
-          cp src/interface/web/assets/icons/favicon-128x128.png package/usr/share/icons/hicolor/128x128/apps/Khoj.png
+          cp src/khoj/interface/web/assets/icons/favicon-128x128.png package/usr/share/icons/hicolor/128x128/apps/Khoj.png
           cp Khoj.desktop package/usr/share/applications
 
           # Fix permissions to be usable by non-root users

.github/workflows/test.yml

@@ -5,7 +5,7 @@ on:
     branches:
       - 'master'
     paths:
-      - src/**
+      - src/khoj/**
       - tests/**
       - config/**
       - setup.py
@@ -14,7 +14,7 @@ on:
     branches:
       - 'master'
     paths:
-      - src/**
+      - src/khoj/**
       - tests/**
       - config/**
       - setup.py

.gitignore

@@ -1,7 +1,6 @@
 # Khoj artifacts
 *.gz
 *.pt
-src/.data
 tests/data/models
 tests/data/embeddings
 
@@ -13,10 +12,10 @@ __pycache__
 .vscode
 
 # Build artifacts
-/src/interface/web/images
+/src/khoj/interface/web/images
 /build/
 /dist/
-/khoj_assistant.egg-info/
+khoj_assistant.egg-info
 /config/khoj*.yml
 .pytest_cache
 khoj.log

.mypy.ini

@@ -5,9 +5,9 @@ install_types = True
 non_interactive = True
 show_error_codes = True
 exclude = (?x)(
-    src/interface/desktop/main_window.py
+    src/khoj/interface/desktop/main_window.py
-    | src/interface/desktop/file_browser.py
+    | src/khoj/interface/desktop/file_browser.py
-    | src/interface/desktop/system_tray.py
+    | src/khoj/interface/desktop/system_tray.py
     | build/*
     | tests/*
   )

Khoj.spec

@@ -5,7 +5,7 @@ from PyInstaller.utils.hooks import copy_metadata
 import sysconfig
 
 datas = [
-    ('src/interface/web', 'src/interface/web'),
+    ('src/khoj/interface/web', 'src/khoj/interface/web'),
     (f'{sysconfig.get_paths()["purelib"]}/transformers', 'transformers')
 ]
 datas += copy_metadata('tqdm')
@@ -19,7 +19,7 @@ datas += copy_metadata('tokenizers')
 block_cipher = None
 
 a = Analysis(
-    ['src/main.py'],
+    ['src/khoj/main.py'],
     pathex=[],
     binaries=[],
     datas=datas,
@@ -50,7 +50,7 @@ pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
 if system() != 'Darwin':
     # Add Splash screen to show on app launch
     splash = Splash(
-        'src/interface/web/assets/icons/favicon-144x144.png',
+        'src/khoj/interface/web/assets/icons/favicon-144x144.png',
         binaries=a.binaries,
         datas=a.datas,
         text_pos=(10, 160),
@@ -82,7 +82,7 @@ if system() != 'Darwin':
         target_arch='x86_64',
         codesign_identity=None,
         entitlements_file=None,
-        icon='src/interface/web/assets/icons/favicon-144x144.ico',
+        icon='src/khoj/interface/web/assets/icons/favicon-144x144.ico',
     )
 else:
     exe = EXE(
@@ -105,11 +105,11 @@ else:
         target_arch='x86_64',
         codesign_identity=None,
         entitlements_file=None,
-        icon='src/interface/web/assets/icons/favicon.icns',
+        icon='src/khoj/interface/web/assets/icons/favicon.icns',
     )
     app = BUNDLE(
         exe,
         name='Khoj.app',
-        icon='src/interface/web/assets/icons/favicon.icns',
+        icon='src/khoj/interface/web/assets/icons/favicon.icns',
         bundle_identifier=None,
     )

MANIFEST.in

@@ -1,5 +1,5 @@
 include README.md
-graft src/interface/*
+graft src/khoj/interface/*
-prune src/interface/web/images*
+prune src/khoj/interface/web/images*
 prune docs*
 global-exclude .DS_Store *.py[cod]

README.md

@@ -59,7 +59,7 @@
 - **Incremental**: Incremental search for a fast, search-as-you-type experience
 - **Pluggable**: Modular architecture makes it easy to plug in new data sources, frontends and ML models
 - **Multiple Sources**: Search your Org-mode and Markdown notes, Beancount transactions and Photos
-- **Multiple Interfaces**: Search using a [Web Browser](./src/interface/web/index.html), [Emacs](./src/interface/emacs/khoj.el) or the [API](http://localhost:8000/docs)
+- **Multiple Interfaces**: Search from your [Web Browser](./src/khoj/interface/web/index.html), [Emacs](./src/interface/emacs/khoj.el) or [Obsidian](./src/interface/obsidian/)
 
 ## Demos
 ### Khoj in Obsidian
@@ -238,7 +238,7 @@ pip install --upgrade khoj-assistant
       asymmetric:
    -    encoder: "sentence-transformers/multi-qa-MiniLM-L6-cos-v1"
    +    encoder: text-embedding-ada-002
-   +    encoder-type: src.utils.models.OpenAI
+   +    encoder-type: src.khoj.utils.models.OpenAI
         cross-encoder: "cross-encoder/ms-marco-MiniLM-L-6-v2"
    -    encoder-type: sentence_transformers.SentenceTransformer
    -    model_directory: "~/.khoj/search/asymmetric/"
@@ -408,7 +408,7 @@ python3 -m pip install pyqt6  # As conda does not support pyqt6 yet
 
 ##### 4. Run
 ```shell
-python3 -m src.main -vv
+python3 -m src.khoj.main -vv
 ```
   Load ML model, generate embeddings and expose API to query notes, images, transactions etc specified in config YAML
 

setup.py

@@ -17,10 +17,10 @@ setup(
     license="GPLv3",
     keywords="search semantic-search productivity NLP org-mode markdown beancount images",
     python_requires=">=3.8, <3.11",
+    package_dir={"": "src"},
     packages=find_packages(
-        where=".",
+        where="src",
-        exclude=["tests*"],
+        include=["khoj*"]
-        include=["src*"]
     ),
     install_requires=[
         "torch == 1.13.1",
@@ -39,7 +39,7 @@ setup(
         'schedule == 1.1.0',
     ],
     include_package_data=True,
-    entry_points={"console_scripts": ["khoj = src.main:run"]},
+    entry_points={"console_scripts": ["khoj = khoj.main:run"]},
     classifiers=[
         "Development Status :: 4 - Beta",
         "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",

src/khoj/configure.py

@@ -7,17 +7,17 @@ import json
 import schedule
 
 # Internal Packages
-from src.processor.ledger.beancount_to_jsonl import BeancountToJsonl
+from khoj.processor.ledger.beancount_to_jsonl import BeancountToJsonl
-from src.processor.markdown.markdown_to_jsonl import MarkdownToJsonl
+from khoj.processor.markdown.markdown_to_jsonl import MarkdownToJsonl
-from src.processor.org_mode.org_to_jsonl import OrgToJsonl
+from khoj.processor.org_mode.org_to_jsonl import OrgToJsonl
-from src.search_type import image_search, text_search
+from khoj.search_type import image_search, text_search
-from src.utils.config import SearchType, SearchModels, ProcessorConfigModel, ConversationProcessorConfigModel
+from khoj.utils.config import SearchType, SearchModels, ProcessorConfigModel, ConversationProcessorConfigModel
-from src.utils import state
+from khoj.utils import state
-from src.utils.helpers import LRU, resolve_absolute_path
+from khoj.utils.helpers import LRU, resolve_absolute_path
-from src.utils.rawconfig import FullConfig, ProcessorConfig
+from khoj.utils.rawconfig import FullConfig, ProcessorConfig
-from src.search_filter.date_filter import DateFilter
+from khoj.search_filter.date_filter import DateFilter
-from src.search_filter.word_filter import WordFilter
+from khoj.search_filter.word_filter import WordFilter
-from src.search_filter.file_filter import FileFilter
+from khoj.search_filter.file_filter import FileFilter
 
 
 logger = logging.getLogger(__name__)

src/khoj/interface/desktop/file_browser.py

@@ -3,8 +3,8 @@ from PyQt6 import QtWidgets
 from PyQt6.QtCore import QDir
 
 # Internal Packages
-from src.utils.config import SearchType
+from khoj.utils.config import SearchType
-from src.utils.helpers import is_none_or_empty
+from khoj.utils.helpers import is_none_or_empty
 
 
 class FileBrowser(QtWidgets.QWidget):

src/khoj/interface/desktop/labelled_text_field.py

@@ -2,7 +2,7 @@
 from PyQt6 import QtWidgets
 
 # Internal Packages
-from src.utils.config import ProcessorType
+from khoj.utils.config import ProcessorType
 
 
 class LabelledTextField(QtWidgets.QWidget):

src/khoj/interface/desktop/main_window.py

@@ -9,13 +9,13 @@ from PyQt6 import QtGui, QtWidgets
 from PyQt6.QtCore import Qt, QThread, QObject, pyqtSignal
 
 # Internal Packages
-from src.configure import configure_server
+from khoj.configure import configure_server
-from src.interface.desktop.file_browser import FileBrowser
+from khoj.interface.desktop.file_browser import FileBrowser
-from src.interface.desktop.labelled_text_field import LabelledTextField
+from khoj.interface.desktop.labelled_text_field import LabelledTextField
-from src.utils import constants, state, yaml as yaml_utils
+from khoj.utils import constants, state, yaml as yaml_utils
-from src.utils.cli import cli
+from khoj.utils.cli import cli
-from src.utils.config import SearchType, ProcessorType
+from khoj.utils.config import SearchType, ProcessorType
-from src.utils.helpers import merge_dicts, resolve_absolute_path
+from khoj.utils.helpers import merge_dicts, resolve_absolute_path
 
 
 class MainWindow(QtWidgets.QMainWindow):

src/khoj/interface/desktop/system_tray.py

@@ -5,8 +5,8 @@ import webbrowser
 from PyQt6 import QtGui, QtWidgets
 
 # Internal Packages
-from src.utils import constants, state
+from khoj.utils import constants, state
-from src.interface.desktop.main_window import MainWindow
+from khoj.interface.desktop.main_window import MainWindow
 
 
 def create_system_tray(gui: QtWidgets.QApplication, main_window: MainWindow):

src/khoj/main.py

@@ -20,15 +20,15 @@ from PyQt6.QtCore import QThread, QTimer
 import schedule
 
 # Internal Packages
-from src.configure import configure_server
+from khoj.configure import configure_server
-from src.routers.api import api
+from khoj.routers.api import api
-from src.routers.api_beta import api_beta
+from khoj.routers.api_beta import api_beta
-from src.routers.web_client import web_client
+from khoj.routers.web_client import web_client
-from src.utils import constants, state
+from khoj.utils import constants, state
-from src.utils.cli import cli
+from khoj.utils.cli import cli
-from src.utils.helpers import CustomFormatter
+from khoj.utils.helpers import CustomFormatter
-from src.interface.desktop.main_window import MainWindow
+from khoj.interface.desktop.main_window import MainWindow
-from src.interface.desktop.system_tray import create_system_tray
+from khoj.interface.desktop.system_tray import create_system_tray
 
 
 # Initialize the Application Server
@@ -38,7 +38,7 @@ app.include_router(api, prefix="/api")
 app.include_router(api_beta, prefix="/api/beta")
 app.include_router(web_client)
 
-logger = logging.getLogger('src')
+logger = logging.getLogger('khoj')
 
 
 def run():

src/khoj/processor/conversation/gpt.py

@@ -7,7 +7,7 @@ from datetime import datetime
 import openai
 
 # Internal Packages
-from src.utils.constants import empty_escape_sequences
+from khoj.utils.constants import empty_escape_sequences
 
 
 def summarize(text, summary_type, model, user_query=None, api_key=None, temperature=0.5, max_tokens=200):

src/khoj/processor/ledger/beancount_to_jsonl.py

@@ -5,11 +5,11 @@ import logging
 from typing import List
 
 # Internal Packages
-from src.processor.text_to_jsonl import TextToJsonl
+from khoj.processor.text_to_jsonl import TextToJsonl
-from src.utils.helpers import get_absolute_path, is_none_or_empty, timer
+from khoj.utils.helpers import get_absolute_path, is_none_or_empty, timer
-from src.utils.constants import empty_escape_sequences
+from khoj.utils.constants import empty_escape_sequences
-from src.utils.jsonl import dump_jsonl, compress_jsonl_data
+from khoj.utils.jsonl import dump_jsonl, compress_jsonl_data
-from src.utils.rawconfig import Entry
+from khoj.utils.rawconfig import Entry
 
 
 logger = logging.getLogger(__name__)

src/khoj/processor/markdown/markdown_to_jsonl.py

@@ -6,11 +6,11 @@ import time
 from typing import List
 
 # Internal Packages
-from src.processor.text_to_jsonl import TextToJsonl
+from khoj.processor.text_to_jsonl import TextToJsonl
-from src.utils.helpers import get_absolute_path, is_none_or_empty, timer
+from khoj.utils.helpers import get_absolute_path, is_none_or_empty, timer
-from src.utils.constants import empty_escape_sequences
+from khoj.utils.constants import empty_escape_sequences
-from src.utils.jsonl import dump_jsonl, compress_jsonl_data
+from khoj.utils.jsonl import dump_jsonl, compress_jsonl_data
-from src.utils.rawconfig import Entry
+from khoj.utils.rawconfig import Entry
 
 
 logger = logging.getLogger(__name__)

src/khoj/processor/org_mode/org_to_jsonl.py

@@ -5,12 +5,12 @@ import time
 from typing import Iterable, List
 
 # Internal Packages
-from src.processor.org_mode import orgnode
+from khoj.processor.org_mode import orgnode
-from src.processor.text_to_jsonl import TextToJsonl
+from khoj.processor.text_to_jsonl import TextToJsonl
-from src.utils.helpers import get_absolute_path, is_none_or_empty, timer
+from khoj.utils.helpers import get_absolute_path, is_none_or_empty, timer
-from src.utils.jsonl import dump_jsonl, compress_jsonl_data
+from khoj.utils.jsonl import dump_jsonl, compress_jsonl_data
-from src.utils.rawconfig import Entry
+from khoj.utils.rawconfig import Entry
-from src.utils import state
+from khoj.utils import state
 
 
 logger = logging.getLogger(__name__)

src/khoj/processor/text_to_jsonl.py

@@ -3,10 +3,10 @@ from abc import ABC, abstractmethod
 import hashlib
 import logging
 from typing import Callable, List, Tuple
-from src.utils.helpers import timer
+from khoj.utils.helpers import timer
 
 # Internal Packages
-from src.utils.rawconfig import Entry, TextContentConfig
+from khoj.utils.rawconfig import Entry, TextContentConfig
 
 
 logger = logging.getLogger(__name__)

src/khoj/routers/api.py

@@ -8,12 +8,12 @@ from fastapi import APIRouter
 from fastapi import HTTPException
 
 # Internal Packages
-from src.configure import configure_processor, configure_search
+from khoj.configure import configure_processor, configure_search
-from src.search_type import image_search, text_search
+from khoj.search_type import image_search, text_search
-from src.utils.helpers import timer
+from khoj.utils.helpers import timer
-from src.utils.rawconfig import FullConfig, SearchResponse
+from khoj.utils.rawconfig import FullConfig, SearchResponse
-from src.utils.config import SearchType
+from khoj.utils.config import SearchType
-from src.utils import state, constants
+from khoj.utils import state, constants
 
 
 # Initialize Router

src/khoj/routers/api_beta.py

@@ -8,11 +8,11 @@ import schedule
 from fastapi import APIRouter
 
 # Internal Packages
-from src.routers.api import search
+from khoj.routers.api import search
-from src.processor.conversation.gpt import converse, extract_search_type, message_to_log, message_to_prompt, understand, summarize
+from khoj.processor.conversation.gpt import converse, extract_search_type, message_to_log, message_to_prompt, understand, summarize
-from src.utils.config import SearchType
+from khoj.utils.config import SearchType
-from src.utils.helpers import get_from_dict, resolve_absolute_path
+from khoj.utils.helpers import get_from_dict, resolve_absolute_path
-from src.utils import state
+from khoj.utils import state
 
 
 # Initialize Router

src/khoj/routers/web_client.py

@@ -5,7 +5,7 @@ from fastapi.responses import HTMLResponse, FileResponse
 from fastapi.templating import Jinja2Templates
 
 # Internal Packages
-from src.utils import constants
+from khoj.utils import constants
 
 
 # Initialize Router

src/khoj/search_filter/base_filter.py

@@ -3,7 +3,7 @@ from abc import ABC, abstractmethod
 from typing import List, Set, Tuple
 
 # Internal Packages
-from src.utils.rawconfig import Entry
+from khoj.utils.rawconfig import Entry
 
 
 class BaseFilter(ABC):

src/khoj/search_filter/date_filter.py

@@ -11,8 +11,8 @@ from math import inf
 import dateparser as dtparse
 
 # Internal Packages
-from src.search_filter.base_filter import BaseFilter
+from khoj.search_filter.base_filter import BaseFilter
-from src.utils.helpers import LRU, timer
+from khoj.utils.helpers import LRU, timer
 
 
 logger = logging.getLogger(__name__)

src/khoj/search_filter/file_filter.py

@@ -6,8 +6,8 @@ import logging
 from collections import defaultdict
 
 # Internal Packages
-from src.search_filter.base_filter import BaseFilter
+from khoj.search_filter.base_filter import BaseFilter
-from src.utils.helpers import LRU, timer
+from khoj.utils.helpers import LRU, timer
 
 
 logger = logging.getLogger(__name__)

src/khoj/search_filter/word_filter.py

@@ -5,8 +5,8 @@ import logging
 from collections import defaultdict
 
 # Internal Packages
-from src.search_filter.base_filter import BaseFilter
+from khoj.search_filter.base_filter import BaseFilter
-from src.utils.helpers import LRU, timer
+from khoj.utils.helpers import LRU, timer
 
 
 logger = logging.getLogger(__name__)

src/khoj/search_type/image_search.py

@@ -14,9 +14,9 @@ from tqdm import trange
 import torch
 
 # Internal Packages
-from src.utils.helpers import get_absolute_path, get_from_dict, resolve_absolute_path, load_model, timer
+from khoj.utils.helpers import get_absolute_path, get_from_dict, resolve_absolute_path, load_model, timer
-from src.utils.config import ImageSearchModel
+from khoj.utils.config import ImageSearchModel
-from src.utils.rawconfig import ImageContentConfig, ImageSearchConfig, SearchResponse
+from khoj.utils.rawconfig import ImageContentConfig, ImageSearchConfig, SearchResponse
 
 
 # Create Logger

src/khoj/search_type/text_search.py

@@ -7,16 +7,16 @@ from typing import List, Tuple, Type
 # External Packages
 import torch
 from sentence_transformers import SentenceTransformer, CrossEncoder, util
-from src.processor.text_to_jsonl import TextToJsonl
+from khoj.processor.text_to_jsonl import TextToJsonl
-from src.search_filter.base_filter import BaseFilter
+from khoj.search_filter.base_filter import BaseFilter
 
 # Internal Packages
-from src.utils import state
+from khoj.utils import state
-from src.utils.helpers import get_absolute_path, is_none_or_empty, resolve_absolute_path, load_model, timer
+from khoj.utils.helpers import get_absolute_path, is_none_or_empty, resolve_absolute_path, load_model, timer
-from src.utils.config import TextSearchModel
+from khoj.utils.config import TextSearchModel
-from src.utils.models import BaseEncoder
+from khoj.utils.models import BaseEncoder
-from src.utils.rawconfig import SearchResponse, TextSearchConfig, TextContentConfig, Entry
+from khoj.utils.rawconfig import SearchResponse, TextSearchConfig, TextContentConfig, Entry
-from src.utils.jsonl import load_jsonl
+from khoj.utils.jsonl import load_jsonl
 
 
 logger = logging.getLogger(__name__)

src/khoj/utils/cli.py

@@ -4,8 +4,8 @@ import pathlib
 from importlib.metadata import version
 
 # Internal Packages
-from src.utils.helpers import resolve_absolute_path
+from khoj.utils.helpers import resolve_absolute_path
-from src.utils.yaml import parse_config_from_file
+from khoj.utils.yaml import parse_config_from_file
 
 
 def cli(args=None):

src/khoj/utils/config.py

@@ -11,9 +11,9 @@ import torch
 # Internal Packages
 if TYPE_CHECKING:
     from sentence_transformers import CrossEncoder
-    from src.search_filter.base_filter import BaseFilter
+    from khoj.search_filter.base_filter import BaseFilter
-    from src.utils.models import BaseEncoder
+    from khoj.utils.models import BaseEncoder
-    from src.utils.rawconfig import ConversationProcessorConfig, Entry
+    from khoj.utils.rawconfig import ConversationProcessorConfig, Entry
 
 
 class SearchType(str, Enum):

src/khoj/utils/constants.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 
 app_root_directory = Path(__file__).parent.parent.parent
-web_directory = app_root_directory / 'src/interface/web/'
+web_directory = app_root_directory / 'khoj/interface/web/'
 empty_escape_sequences = '\n|\r|\t| '
 
 # default app config to use

src/khoj/utils/helpers.py

@@ -14,7 +14,7 @@ if TYPE_CHECKING:
     # External Packages
     from sentence_transformers import CrossEncoder
     # Internal Packages
-    from src.utils.models import BaseEncoder
+    from khoj.utils.models import BaseEncoder
 
 
 def is_none_or_empty(item):

src/khoj/utils/jsonl.py

@@ -4,8 +4,8 @@ import gzip
 import logging
 
 # Internal Packages
-from src.utils.constants import empty_escape_sequences
+from khoj.utils.constants import empty_escape_sequences
-from src.utils.helpers import get_absolute_path
+from khoj.utils.helpers import get_absolute_path
 
 
 logger = logging.getLogger(__name__)

src/khoj/utils/models.py

@@ -8,7 +8,7 @@ import torch
 from tqdm import trange
 
 # Internal Packages
-from src.utils.state import processor_config, config_file
+from khoj.utils.state import processor_config, config_file
 
 
 class BaseEncoder(ABC):

src/khoj/utils/rawconfig.py

@@ -7,7 +7,7 @@ from typing import List, Optional
 from pydantic import BaseModel, validator
 
 # Internal Packages
-from src.utils.helpers import to_snake_case_from_dash, is_none_or_empty
+from khoj.utils.helpers import to_snake_case_from_dash, is_none_or_empty
 
 class ConfigBase(BaseModel):
     class Config:

src/khoj/utils/state.py

@@ -8,9 +8,9 @@ import torch
 from pathlib import Path
 
 # Internal Packages
-from src.utils.config import SearchModels, ProcessorConfigModel
+from khoj.utils.config import SearchModels, ProcessorConfigModel
-from src.utils.helpers import LRU
+from khoj.utils.helpers import LRU
-from src.utils.rawconfig import FullConfig
+from khoj.utils.rawconfig import FullConfig
 
 # Application Global State
 config = FullConfig()

src/khoj/utils/yaml.py

@@ -5,7 +5,7 @@ from pathlib import Path
 import yaml
 
 # Internal Packages
-from src.utils.rawconfig import FullConfig
+from khoj.utils.rawconfig import FullConfig
 
 
 # Do not emit tags when dumping to YAML

tests/conftest.py

@@ -4,13 +4,13 @@ from pathlib import Path
 import pytest
 
 # Internal Packages
-from src.search_type import image_search, text_search
+from khoj.search_type import image_search, text_search
-from src.utils.helpers import resolve_absolute_path
+from khoj.utils.helpers import resolve_absolute_path
-from src.utils.rawconfig import ContentConfig, TextContentConfig, ImageContentConfig, SearchConfig, TextSearchConfig, ImageSearchConfig
+from khoj.utils.rawconfig import ContentConfig, TextContentConfig, ImageContentConfig, SearchConfig, TextSearchConfig, ImageSearchConfig
-from src.processor.org_mode.org_to_jsonl import OrgToJsonl
+from khoj.processor.org_mode.org_to_jsonl import OrgToJsonl
-from src.search_filter.date_filter import DateFilter
+from khoj.search_filter.date_filter import DateFilter
-from src.search_filter.word_filter import WordFilter
+from khoj.search_filter.word_filter import WordFilter
-from src.search_filter.file_filter import FileFilter
+from khoj.search_filter.file_filter import FileFilter
 
 
 @pytest.fixture(scope='session')

tests/data/markdown/main_readme.md

@@ -6,8 +6,8 @@
 *Allow natural language search on user content like notes, images,
 transactions using transformer ML models*
 
-User can interface with Khoj via [Web](./src/interface/web/index.html),
+User can interface with Khoj via [Web](./src/khoj/interface/web/index.html),
-[Emacs](./src/interface/emacs/khoj.el) or the API. All search is done
+[Emacs](./src/khoj/interface/emacs/khoj.el) or the API. All search is done
 locally[\*](https://github.com/debanjum/khoj#miscellaneous)
 
 ## Demo
@@ -47,8 +47,8 @@ just generating embeddings*
     -   [Update Index](http://localhost:8000/api/update?t=ledger)
     -   [Configure Application](https://localhost:8000/ui)
 -   **Khoj via Emacs**
-    -   [Install](https://github.com/debanjum/khoj/tree/master/src/interface/emacs#installation)
+    -   [Install](https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#installation)
-        [khoj.el](./src/interface/emacs/khoj.el)
+        [khoj.el](./src/khoj/interface/emacs/khoj.el)
     -   Run `M-x khoj <user-query>`
 
 ## Run Unit tests
@@ -118,7 +118,7 @@ docker-compose build --pull
     images, transactions etc specified in config YAML
 
     ``` shell
-    python3 -m src.main -c=config/khoj_sample.yml -vv
+    python3 -m src.khoj.main -c=config/khoj_sample.yml -vv
     ```
 
 ### Upgrade On Local Machine

tests/test_beancount_to_jsonl.py

@@ -2,7 +2,7 @@
 import json
 
 # Internal Packages
-from src.processor.ledger.beancount_to_jsonl import BeancountToJsonl
+from khoj.processor.ledger.beancount_to_jsonl import BeancountToJsonl
 
 
 def test_no_transactions_in_file(tmp_path):

tests/test_chatbot.py

@@ -2,7 +2,7 @@
 import pytest
 
 # Internal Packages
-from src.processor.conversation.gpt import converse, understand, message_to_prompt
+from khoj.processor.conversation.gpt import converse, understand, message_to_prompt
 
 
 # Initialize variables for tests

tests/test_cli.py

@@ -3,8 +3,8 @@ from pathlib import Path
 from random import random
 
 # Internal Packages
-from src.utils.cli import cli
+from khoj.utils.cli import cli
-from src.utils.helpers import resolve_absolute_path
+from khoj.utils.helpers import resolve_absolute_path
 
 
 # Test

tests/test_client.py

@@ -8,13 +8,13 @@ from urllib.parse import quote
 from fastapi.testclient import TestClient
 
 # Internal Packages
-from src.main import app
+from khoj.main import app
-from src.utils.state import model, config
+from khoj.utils.state import model, config
-from src.search_type import text_search, image_search
+from khoj.search_type import text_search, image_search
-from src.utils.rawconfig import ContentConfig, SearchConfig
+from khoj.utils.rawconfig import ContentConfig, SearchConfig
-from src.processor.org_mode.org_to_jsonl import OrgToJsonl
+from khoj.processor.org_mode.org_to_jsonl import OrgToJsonl
-from src.search_filter.word_filter import WordFilter
+from khoj.search_filter.word_filter import WordFilter
-from src.search_filter.file_filter import FileFilter
+from khoj.search_filter.file_filter import FileFilter
 
 
 # Arrange

tests/test_date_filter.py

@@ -4,8 +4,8 @@ from datetime import datetime
 from math import inf
 
 # Application Packages
-from src.search_filter.date_filter import DateFilter
+from khoj.search_filter.date_filter import DateFilter
-from src.utils.rawconfig import Entry
+from khoj.utils.rawconfig import Entry
 
 
 def test_date_filter():

tests/test_file_filter.py

@@ -1,6 +1,6 @@
 # Application Packages
-from src.search_filter.file_filter import FileFilter
+from khoj.search_filter.file_filter import FileFilter
-from src.utils.rawconfig import Entry
+from khoj.utils.rawconfig import Entry
 
 
 def test_no_file_filter():

tests/test_helpers.py

@@ -1,4 +1,4 @@
-from src.utils import helpers
+from khoj.utils import helpers
 
 def test_get_from_null_dict():
     # null handling

tests/test_image_search.py

@@ -4,11 +4,11 @@ from pathlib import Path
 from PIL import Image
 
 # Internal Packages
-from src.utils.state import model
+from khoj.utils.state import model
-from src.utils.constants import web_directory
+from khoj.utils.constants import web_directory
-from src.search_type import image_search
+from khoj.search_type import image_search
-from src.utils.helpers import resolve_absolute_path
+from khoj.utils.helpers import resolve_absolute_path
-from src.utils.rawconfig import ContentConfig, SearchConfig
+from khoj.utils.rawconfig import ContentConfig, SearchConfig
 
 
 # Test
@@ -91,7 +91,7 @@ def test_image_search_query_truncated(content_config: ContentConfig, search_conf
 
     # Act
     try:
-        with caplog.at_level(logging.INFO, logger="src.search_type.image_search"):
+        with caplog.at_level(logging.INFO, logger="khoj.search_type.image_search"):
             image_search.query(
                 query,
                 count = 1,
@@ -114,7 +114,7 @@ def test_image_search_by_filepath(content_config: ContentConfig, search_config:
     expected_image_path = f"{image_directory.joinpath('kitten_park.jpg')}"
 
     # Act
-    with caplog.at_level(logging.INFO, logger="src.search_type.image_search"):
+    with caplog.at_level(logging.INFO, logger="khoj.search_type.image_search"):
         hits = image_search.query(
             query,
             count = 1,

tests/test_markdown_to_jsonl.py

@@ -2,7 +2,7 @@
 import json
 
 # Internal Packages
-from src.processor.markdown.markdown_to_jsonl import MarkdownToJsonl
+from khoj.processor.markdown.markdown_to_jsonl import MarkdownToJsonl
 
 
 def test_markdown_file_with_no_headings_to_jsonl(tmp_path):

tests/test_org_to_jsonl.py

@@ -2,10 +2,10 @@
 import json
 
 # Internal Packages
-from src.processor.org_mode.org_to_jsonl import OrgToJsonl
+from khoj.processor.org_mode.org_to_jsonl import OrgToJsonl
-from src.processor.text_to_jsonl import TextToJsonl
+from khoj.processor.text_to_jsonl import TextToJsonl
-from src.utils.helpers import is_none_or_empty
+from khoj.utils.helpers import is_none_or_empty
-from src.utils.rawconfig import Entry
+from khoj.utils.rawconfig import Entry
 
 
 def test_configure_heading_entry_to_jsonl(tmp_path):

tests/test_orgnode.py

@@ -2,7 +2,7 @@
 import datetime
 
 # Internal Packages
-from src.processor.org_mode import orgnode
+from khoj.processor.org_mode import orgnode
 
 
 # Test

tests/test_text_search.py

@@ -6,10 +6,10 @@ from pathlib import Path
 import pytest
 
 # Internal Packages
-from src.utils.state import model
+from khoj.utils.state import model
-from src.search_type import text_search
+from khoj.search_type import text_search
-from src.utils.rawconfig import ContentConfig, SearchConfig, TextContentConfig
+from khoj.utils.rawconfig import ContentConfig, SearchConfig, TextContentConfig
-from src.processor.org_mode.org_to_jsonl import OrgToJsonl
+from khoj.processor.org_mode.org_to_jsonl import OrgToJsonl
 
 
 # Test

tests/test_word_filter.py

@@ -1,6 +1,6 @@
 # Application Packages
-from src.search_filter.word_filter import WordFilter
+from khoj.search_filter.word_filter import WordFilter
-from src.utils.rawconfig import Entry
+from khoj.utils.rawconfig import Entry
 
 
 def test_no_word_filter():