Use the src/ layout to fix packaging Khoj for PyPi

- Why
  The khoj pypi packages should be installed in `khoj' directory.
  Previously it was being installed into `src' directory, which is a
  generic top level directory name that is discouraged from being used

- Changes
 - move src/* to src/khoj/*
 - update `setup.py' to `find_packages' in `src' instead of project root
 - rename imports to form `from khoj.*' in complete project
 - update `constants.web_directory' path to use `khoj' directory
 - rename root logger to `khoj' in `main.py'
 - fix image_search tests to use the newly rename `khoj' logger
 - update config, docs, workflows to reference new path `src/khoj'

src/khoj/configure.py

@@ -7,17 +7,17 @@ import json
 import schedule
 
 # Internal Packages
-from src.processor.ledger.beancount_to_jsonl import BeancountToJsonl
-from src.processor.markdown.markdown_to_jsonl import MarkdownToJsonl
-from src.processor.org_mode.org_to_jsonl import OrgToJsonl
-from src.search_type import image_search, text_search
-from src.utils.config import SearchType, SearchModels, ProcessorConfigModel, ConversationProcessorConfigModel
-from src.utils import state
-from src.utils.helpers import LRU, resolve_absolute_path
-from src.utils.rawconfig import FullConfig, ProcessorConfig
-from src.search_filter.date_filter import DateFilter
-from src.search_filter.word_filter import WordFilter
-from src.search_filter.file_filter import FileFilter
+from khoj.processor.ledger.beancount_to_jsonl import BeancountToJsonl
+from khoj.processor.markdown.markdown_to_jsonl import MarkdownToJsonl
+from khoj.processor.org_mode.org_to_jsonl import OrgToJsonl
+from khoj.search_type import image_search, text_search
+from khoj.utils.config import SearchType, SearchModels, ProcessorConfigModel, ConversationProcessorConfigModel
+from khoj.utils import state
+from khoj.utils.helpers import LRU, resolve_absolute_path
+from khoj.utils.rawconfig import FullConfig, ProcessorConfig
+from khoj.search_filter.date_filter import DateFilter
+from khoj.search_filter.word_filter import WordFilter
+from khoj.search_filter.file_filter import FileFilter
 
 
 logger = logging.getLogger(__name__)

src/khoj/interface/desktop/file_browser.py

@@ -3,8 +3,8 @@ from PyQt6 import QtWidgets
 from PyQt6.QtCore import QDir
 
 # Internal Packages
-from src.utils.config import SearchType
-from src.utils.helpers import is_none_or_empty
+from khoj.utils.config import SearchType
+from khoj.utils.helpers import is_none_or_empty
 
 
 class FileBrowser(QtWidgets.QWidget):

src/khoj/interface/desktop/labelled_text_field.py

@@ -2,7 +2,7 @@
 from PyQt6 import QtWidgets
 
 # Internal Packages
-from src.utils.config import ProcessorType
+from khoj.utils.config import ProcessorType
 
 
 class LabelledTextField(QtWidgets.QWidget):

src/khoj/interface/desktop/main_window.py

@@ -9,13 +9,13 @@ from PyQt6 import QtGui, QtWidgets
 from PyQt6.QtCore import Qt, QThread, QObject, pyqtSignal
 
 # Internal Packages
-from src.configure import configure_server
-from src.interface.desktop.file_browser import FileBrowser
-from src.interface.desktop.labelled_text_field import LabelledTextField
-from src.utils import constants, state, yaml as yaml_utils
-from src.utils.cli import cli
-from src.utils.config import SearchType, ProcessorType
-from src.utils.helpers import merge_dicts, resolve_absolute_path
+from khoj.configure import configure_server
+from khoj.interface.desktop.file_browser import FileBrowser
+from khoj.interface.desktop.labelled_text_field import LabelledTextField
+from khoj.utils import constants, state, yaml as yaml_utils
+from khoj.utils.cli import cli
+from khoj.utils.config import SearchType, ProcessorType
+from khoj.utils.helpers import merge_dicts, resolve_absolute_path
 
 
 class MainWindow(QtWidgets.QMainWindow):

src/khoj/interface/desktop/system_tray.py

@@ -5,8 +5,8 @@ import webbrowser
 from PyQt6 import QtGui, QtWidgets
 
 # Internal Packages
-from src.utils import constants, state
-from src.interface.desktop.main_window import MainWindow
+from khoj.utils import constants, state
+from khoj.interface.desktop.main_window import MainWindow
 
 
 def create_system_tray(gui: QtWidgets.QApplication, main_window: MainWindow):

src/khoj/interface/emacs/README.org

@@ -4,19 +4,19 @@
   /Natural, Incremental Search for your Second Brain/
 
 ** Table of Contents
-  - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#features][Features]]
-  - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Interface][Interface]]
-  - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Setup][Setup]]
-    - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#1-Setup-Backend][Setup Backend]]
-    - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#2-Install-Khojel][Install Khoj.el]]
-  - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Use][Use]]
-    - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Search][Search]]
-    - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Find-similar-entries][Find Similar Entries]]
-    - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Advanced-usage][Advanced Usage]]
-    - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Khoj-menu][Khoj Menu]]
-  - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Upgrade][Upgrade]]
-    - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Upgrade-Khoj-Backend][Upgrade Backend]]
-    - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Upgrade-Khojel][Upgrade Khoj.el]]
+  - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#features][Features]]
+  - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Interface][Interface]]
+  - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Setup][Setup]]
+    - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#1-Setup-Backend][Setup Backend]]
+    - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#2-Install-Khojel][Install Khoj.el]]
+  - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Use][Use]]
+    - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Search][Search]]
+    - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Find-similar-entries][Find Similar Entries]]
+    - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Advanced-usage][Advanced Usage]]
+    - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Khoj-menu][Khoj Menu]]
+  - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Upgrade][Upgrade]]
+    - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Upgrade-Khoj-Backend][Upgrade Backend]]
+    - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Upgrade-Khojel][Upgrade Khoj.el]]
 
 ** Features
   - *Natural*: Advanced natural language understanding using Transformer based ML Models
@@ -58,7 +58,7 @@
     ;; Install Khoj Package using Straight.el
     (use-package khoj
       :after org
-      :straight (khoj :type git :host github :repo "debanjum/khoj" :files (:defaults "src/interface/emacs/khoj.el"))
+      :straight (khoj :type git :host github :repo "debanjum/khoj" :files (:defaults "src/khoj/interface/emacs/khoj.el"))
       :bind ("C-c s" . 'khoj))
   #+end_src
 

src/khoj/interface/emacs/khoj.el

@@ -7,7 +7,7 @@
 ;; Keywords: search, org-mode, outlines, markdown, beancount, ledger, image
 ;; Version: 0.3.0
 ;; Package-Requires: ((emacs "27.1") (transient "0.3.0"))
-;; URL: https://github.com/debanjum/khoj/tree/master/src/interface/emacs
+;; URL: https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs
 
 ;; This file is NOT part of GNU Emacs.
 

src/khoj/interface/emacs/tests/khoj-tests.el

@@ -5,7 +5,7 @@
 ;; Author: Debanjum Singh Solanky <debanjum@gmail.com>
 ;; Version: 0.0.0
 ;; Package-Requires: ((emacs "27.1") (transient "0.3.0"))
-;; URL: https://github.com/debanjum/khoj/tree/master/src/interface/emacs
+;; URL: https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs
 
 ;;; License:
 

src/khoj/main.py

@@ -20,15 +20,15 @@ from PyQt6.QtCore import QThread, QTimer
 import schedule
 
 # Internal Packages
-from src.configure import configure_server
-from src.routers.api import api
-from src.routers.api_beta import api_beta
-from src.routers.web_client import web_client
-from src.utils import constants, state
-from src.utils.cli import cli
-from src.utils.helpers import CustomFormatter
-from src.interface.desktop.main_window import MainWindow
-from src.interface.desktop.system_tray import create_system_tray
+from khoj.configure import configure_server
+from khoj.routers.api import api
+from khoj.routers.api_beta import api_beta
+from khoj.routers.web_client import web_client
+from khoj.utils import constants, state
+from khoj.utils.cli import cli
+from khoj.utils.helpers import CustomFormatter
+from khoj.interface.desktop.main_window import MainWindow
+from khoj.interface.desktop.system_tray import create_system_tray
 
 
 # Initialize the Application Server
@@ -38,7 +38,7 @@ app.include_router(api, prefix="/api")
 app.include_router(api_beta, prefix="/api/beta")
 app.include_router(web_client)
 
-logger = logging.getLogger('src')
+logger = logging.getLogger('khoj')
 
 
 def run():

src/khoj/processor/conversation/gpt.py

@@ -7,7 +7,7 @@ from datetime import datetime
 import openai
 
 # Internal Packages
-from src.utils.constants import empty_escape_sequences
+from khoj.utils.constants import empty_escape_sequences
 
 
 def summarize(text, summary_type, model, user_query=None, api_key=None, temperature=0.5, max_tokens=200):

src/khoj/processor/ledger/beancount_to_jsonl.py

@@ -5,11 +5,11 @@ import logging
 from typing import List
 
 # Internal Packages
-from src.processor.text_to_jsonl import TextToJsonl
-from src.utils.helpers import get_absolute_path, is_none_or_empty, timer
-from src.utils.constants import empty_escape_sequences
-from src.utils.jsonl import dump_jsonl, compress_jsonl_data
-from src.utils.rawconfig import Entry
+from khoj.processor.text_to_jsonl import TextToJsonl
+from khoj.utils.helpers import get_absolute_path, is_none_or_empty, timer
+from khoj.utils.constants import empty_escape_sequences
+from khoj.utils.jsonl import dump_jsonl, compress_jsonl_data
+from khoj.utils.rawconfig import Entry
 
 
 logger = logging.getLogger(__name__)

src/khoj/processor/markdown/markdown_to_jsonl.py

@@ -6,11 +6,11 @@ import time
 from typing import List
 
 # Internal Packages
-from src.processor.text_to_jsonl import TextToJsonl
-from src.utils.helpers import get_absolute_path, is_none_or_empty, timer
-from src.utils.constants import empty_escape_sequences
-from src.utils.jsonl import dump_jsonl, compress_jsonl_data
-from src.utils.rawconfig import Entry
+from khoj.processor.text_to_jsonl import TextToJsonl
+from khoj.utils.helpers import get_absolute_path, is_none_or_empty, timer
+from khoj.utils.constants import empty_escape_sequences
+from khoj.utils.jsonl import dump_jsonl, compress_jsonl_data
+from khoj.utils.rawconfig import Entry
 
 
 logger = logging.getLogger(__name__)

src/khoj/processor/org_mode/org_to_jsonl.py

@@ -5,12 +5,12 @@ import time
 from typing import Iterable, List
 
 # Internal Packages
-from src.processor.org_mode import orgnode
-from src.processor.text_to_jsonl import TextToJsonl
-from src.utils.helpers import get_absolute_path, is_none_or_empty, timer
-from src.utils.jsonl import dump_jsonl, compress_jsonl_data
-from src.utils.rawconfig import Entry
-from src.utils import state
+from khoj.processor.org_mode import orgnode
+from khoj.processor.text_to_jsonl import TextToJsonl
+from khoj.utils.helpers import get_absolute_path, is_none_or_empty, timer
+from khoj.utils.jsonl import dump_jsonl, compress_jsonl_data
+from khoj.utils.rawconfig import Entry
+from khoj.utils import state
 
 
 logger = logging.getLogger(__name__)

src/khoj/processor/text_to_jsonl.py

@@ -3,10 +3,10 @@ from abc import ABC, abstractmethod
 import hashlib
 import logging
 from typing import Callable, List, Tuple
-from src.utils.helpers import timer
+from khoj.utils.helpers import timer
 
 # Internal Packages
-from src.utils.rawconfig import Entry, TextContentConfig
+from khoj.utils.rawconfig import Entry, TextContentConfig
 
 
 logger = logging.getLogger(__name__)

src/khoj/routers/api.py

@@ -8,12 +8,12 @@ from fastapi import APIRouter
 from fastapi import HTTPException
 
 # Internal Packages
-from src.configure import configure_processor, configure_search
-from src.search_type import image_search, text_search
-from src.utils.helpers import timer
-from src.utils.rawconfig import FullConfig, SearchResponse
-from src.utils.config import SearchType
-from src.utils import state, constants
+from khoj.configure import configure_processor, configure_search
+from khoj.search_type import image_search, text_search
+from khoj.utils.helpers import timer
+from khoj.utils.rawconfig import FullConfig, SearchResponse
+from khoj.utils.config import SearchType
+from khoj.utils import state, constants
 
 
 # Initialize Router

src/khoj/routers/api_beta.py

@@ -8,11 +8,11 @@ import schedule
 from fastapi import APIRouter
 
 # Internal Packages
-from src.routers.api import search
-from src.processor.conversation.gpt import converse, extract_search_type, message_to_log, message_to_prompt, understand, summarize
-from src.utils.config import SearchType
-from src.utils.helpers import get_from_dict, resolve_absolute_path
-from src.utils import state
+from khoj.routers.api import search
+from khoj.processor.conversation.gpt import converse, extract_search_type, message_to_log, message_to_prompt, understand, summarize
+from khoj.utils.config import SearchType
+from khoj.utils.helpers import get_from_dict, resolve_absolute_path
+from khoj.utils import state
 
 
 # Initialize Router

src/khoj/routers/web_client.py

@@ -5,7 +5,7 @@ from fastapi.responses import HTMLResponse, FileResponse
 from fastapi.templating import Jinja2Templates
 
 # Internal Packages
-from src.utils import constants
+from khoj.utils import constants
 
 
 # Initialize Router

src/khoj/search_filter/base_filter.py

@@ -3,7 +3,7 @@ from abc import ABC, abstractmethod
 from typing import List, Set, Tuple
 
 # Internal Packages
-from src.utils.rawconfig import Entry
+from khoj.utils.rawconfig import Entry
 
 
 class BaseFilter(ABC):

src/khoj/search_filter/date_filter.py

@@ -11,8 +11,8 @@ from math import inf
 import dateparser as dtparse
 
 # Internal Packages
-from src.search_filter.base_filter import BaseFilter
-from src.utils.helpers import LRU, timer
+from khoj.search_filter.base_filter import BaseFilter
+from khoj.utils.helpers import LRU, timer
 
 
 logger = logging.getLogger(__name__)

src/khoj/search_filter/file_filter.py

@@ -6,8 +6,8 @@ import logging
 from collections import defaultdict
 
 # Internal Packages
-from src.search_filter.base_filter import BaseFilter
-from src.utils.helpers import LRU, timer
+from khoj.search_filter.base_filter import BaseFilter
+from khoj.utils.helpers import LRU, timer
 
 
 logger = logging.getLogger(__name__)

src/khoj/search_filter/word_filter.py

@@ -5,8 +5,8 @@ import logging
 from collections import defaultdict
 
 # Internal Packages
-from src.search_filter.base_filter import BaseFilter
-from src.utils.helpers import LRU, timer
+from khoj.search_filter.base_filter import BaseFilter
+from khoj.utils.helpers import LRU, timer
 
 
 logger = logging.getLogger(__name__)

src/khoj/search_type/image_search.py

@@ -14,9 +14,9 @@ from tqdm import trange
 import torch
 
 # Internal Packages
-from src.utils.helpers import get_absolute_path, get_from_dict, resolve_absolute_path, load_model, timer
-from src.utils.config import ImageSearchModel
-from src.utils.rawconfig import ImageContentConfig, ImageSearchConfig, SearchResponse
+from khoj.utils.helpers import get_absolute_path, get_from_dict, resolve_absolute_path, load_model, timer
+from khoj.utils.config import ImageSearchModel
+from khoj.utils.rawconfig import ImageContentConfig, ImageSearchConfig, SearchResponse
 
 
 # Create Logger

src/khoj/search_type/text_search.py

@@ -7,16 +7,16 @@ from typing import List, Tuple, Type
 # External Packages
 import torch
 from sentence_transformers import SentenceTransformer, CrossEncoder, util
-from src.processor.text_to_jsonl import TextToJsonl
-from src.search_filter.base_filter import BaseFilter
+from khoj.processor.text_to_jsonl import TextToJsonl
+from khoj.search_filter.base_filter import BaseFilter
 
 # Internal Packages
-from src.utils import state
-from src.utils.helpers import get_absolute_path, is_none_or_empty, resolve_absolute_path, load_model, timer
-from src.utils.config import TextSearchModel
-from src.utils.models import BaseEncoder
-from src.utils.rawconfig import SearchResponse, TextSearchConfig, TextContentConfig, Entry
-from src.utils.jsonl import load_jsonl
+from khoj.utils import state
+from khoj.utils.helpers import get_absolute_path, is_none_or_empty, resolve_absolute_path, load_model, timer
+from khoj.utils.config import TextSearchModel
+from khoj.utils.models import BaseEncoder
+from khoj.utils.rawconfig import SearchResponse, TextSearchConfig, TextContentConfig, Entry
+from khoj.utils.jsonl import load_jsonl
 
 
 logger = logging.getLogger(__name__)

src/khoj/utils/cli.py

@@ -4,8 +4,8 @@ import pathlib
 from importlib.metadata import version
 
 # Internal Packages
-from src.utils.helpers import resolve_absolute_path
-from src.utils.yaml import parse_config_from_file
+from khoj.utils.helpers import resolve_absolute_path
+from khoj.utils.yaml import parse_config_from_file
 
 
 def cli(args=None):

src/khoj/utils/config.py

@@ -11,9 +11,9 @@ import torch
 # Internal Packages
 if TYPE_CHECKING:
     from sentence_transformers import CrossEncoder
-    from src.search_filter.base_filter import BaseFilter
-    from src.utils.models import BaseEncoder
-    from src.utils.rawconfig import ConversationProcessorConfig, Entry
+    from khoj.search_filter.base_filter import BaseFilter
+    from khoj.utils.models import BaseEncoder
+    from khoj.utils.rawconfig import ConversationProcessorConfig, Entry
 
 
 class SearchType(str, Enum):

src/khoj/utils/constants.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 
 app_root_directory = Path(__file__).parent.parent.parent
-web_directory = app_root_directory / 'src/interface/web/'
+web_directory = app_root_directory / 'khoj/interface/web/'
 empty_escape_sequences = '\n|\r|\t| '
 
 # default app config to use

src/khoj/utils/helpers.py

@@ -14,7 +14,7 @@ if TYPE_CHECKING:
     # External Packages
     from sentence_transformers import CrossEncoder
     # Internal Packages
-    from src.utils.models import BaseEncoder
+    from khoj.utils.models import BaseEncoder
 
 
 def is_none_or_empty(item):

src/khoj/utils/jsonl.py

@@ -4,8 +4,8 @@ import gzip
 import logging
 
 # Internal Packages
-from src.utils.constants import empty_escape_sequences
-from src.utils.helpers import get_absolute_path
+from khoj.utils.constants import empty_escape_sequences
+from khoj.utils.helpers import get_absolute_path
 
 
 logger = logging.getLogger(__name__)

src/khoj/utils/models.py

@@ -8,7 +8,7 @@ import torch
 from tqdm import trange
 
 # Internal Packages
-from src.utils.state import processor_config, config_file
+from khoj.utils.state import processor_config, config_file
 
 
 class BaseEncoder(ABC):

src/khoj/utils/rawconfig.py

@@ -7,7 +7,7 @@ from typing import List, Optional
 from pydantic import BaseModel, validator
 
 # Internal Packages
-from src.utils.helpers import to_snake_case_from_dash, is_none_or_empty
+from khoj.utils.helpers import to_snake_case_from_dash, is_none_or_empty
 
 class ConfigBase(BaseModel):
     class Config:

src/khoj/utils/state.py

@@ -8,9 +8,9 @@ import torch
 from pathlib import Path
 
 # Internal Packages
-from src.utils.config import SearchModels, ProcessorConfigModel
-from src.utils.helpers import LRU
-from src.utils.rawconfig import FullConfig
+from khoj.utils.config import SearchModels, ProcessorConfigModel
+from khoj.utils.helpers import LRU
+from khoj.utils.rawconfig import FullConfig
 
 # Application Global State
 config = FullConfig()

src/khoj/utils/yaml.py

@@ -5,7 +5,7 @@ from pathlib import Path
 import yaml
 
 # Internal Packages
-from src.utils.rawconfig import FullConfig
+from khoj.utils.rawconfig import FullConfig
 
 
 # Do not emit tags when dumping to YAML