diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 21792f51..1ac735aa 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -48,7 +48,7 @@ jobs: - name: 🌡️ Validate Python Package run: | # Validate PyPi Package - pipx run check-wheel-contents dist/*.whl + pipx run check-wheel-contents dist/*.whl --ignore W004 pipx run twine check dist/* - name: ⏫ Upload Python Package Artifacts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 697579da..51301491 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,7 +3,7 @@ name: test on: pull_request: paths: - - src/** + - src/khoj/** - tests/** - config/** - pyproject.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 77d4ec6c..b2442280 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,6 +4,12 @@ repos: hooks: - id: black +# - repo: https://github.com/pycqa/isort +# rev: 5.12.0 +# hooks: +# - id: isort +# name: isort (python) + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: diff --git a/Dockerfile b/Dockerfile index 9882a236..4512e884 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ RUN sed -i 's/dynamic = \["version"\]/version = "0.0.0"/' pyproject.toml && \ COPY . . # Set the PYTHONPATH environment variable in order for it to find the Django app. -ENV PYTHONPATH=/app/src:$PYTHONPATH +ENV PYTHONPATH=/app/src/khoj:$PYTHONPATH # Run the Application # There are more arguments required for the application to run, diff --git a/prod.Dockerfile b/prod.Dockerfile index 693a3a8b..a935f3c6 100644 --- a/prod.Dockerfile +++ b/prod.Dockerfile @@ -20,7 +20,7 @@ COPY . . RUN apt install vim -y # Set the PYTHONPATH environment variable in order for it to find the Django app. -ENV PYTHONPATH=/app/src:$PYTHONPATH +ENV PYTHONPATH=/app/src/khoj:$PYTHONPATH # Run the Application # There are more arguments required for the application to run, diff --git a/pyproject.toml b/pyproject.toml index a457aec6..17c11e77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ dependencies = [ "tiktoken >= 0.3.2", "tenacity >= 8.2.2", "pillow == 9.3.0", - "pydantic >= 1.10.10", + "pydantic >= 2.0.0", "pyyaml == 6.0", "rich >= 13.3.1", "schedule == 1.1.0", diff --git a/pytest.ini b/pytest.ini index b3e418d0..36d842ae 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -DJANGO_SETTINGS_MODULE = app.settings +DJANGO_SETTINGS_MODULE = khoj.app.settings pythonpath = . src testpaths = tests markers = diff --git a/src/interface/obsidian/src/main.ts b/src/interface/obsidian/src/main.ts index 26b0a5a1..b45a7fb5 100644 --- a/src/interface/obsidian/src/main.ts +++ b/src/interface/obsidian/src/main.ts @@ -73,21 +73,19 @@ export default class Khoj extends Plugin { // Check if khoj backend is configured, note if cannot connect to backend let headers = { "Authorization": `Bearer ${this.settings.khojApiKey}` }; - if (this.settings.khojUrl === "https://app.khoj.dev") { - if (this.settings.khojApiKey === "") { - new Notice(`❗️Khoj API key is not configured. Please visit https://app.khoj.dev/config#clients to get an API key.`); - return; - } - - await request({ url: this.settings.khojUrl ,method: "GET", headers: headers }) - .then(response => { - this.settings.connectedToBackend = true; - }) - .catch(error => { - this.settings.connectedToBackend = false; - new Notice(`❗️Ensure Khoj backend is running and Khoj URL is pointing to it in the plugin settings.\n\n${error}`); - }); + if (this.settings.khojApiKey === "" && this.settings.khojUrl === "https://app.khoj.dev") { + new Notice(`❗️Khoj API key is not configured. Please visit https://app.khoj.dev/config#clients to get an API key.`); + return; } + + await request({ url: this.settings.khojUrl ,method: "GET", headers: headers }) + .then(response => { + this.settings.connectedToBackend = true; + }) + .catch(error => { + this.settings.connectedToBackend = false; + new Notice(`❗️Ensure Khoj backend is running and Khoj URL is pointing to it in the plugin settings.\n\n${error}`); + }); } async saveSettings() { diff --git a/src/interface/obsidian/src/search_modal.ts b/src/interface/obsidian/src/search_modal.ts index e841360e..7e97d1ea 100644 --- a/src/interface/obsidian/src/search_modal.ts +++ b/src/interface/obsidian/src/search_modal.ts @@ -87,27 +87,18 @@ export class KhojSearchModal extends SuggestModal { } async getSuggestions(query: string): Promise { - // Query Khoj backend for search results + // Setup Query Khoj backend for search results let encodedQuery = encodeURIComponent(query); let searchUrl = `${this.setting.khojUrl}/api/search?q=${encodedQuery}&n=${this.setting.resultsCount}&r=${this.rerank}&client=obsidian`; let headers = { 'Authorization': `Bearer ${this.setting.khojApiKey}` } - // Get search results for markdown and pdf files - let mdResponse = await request({ url: `${searchUrl}&t=markdown`, headers: headers }); - let pdfResponse = await request({ url: `${searchUrl}&t=pdf`, headers: headers }); + // Get search results from Khoj backend + let response = await request({ url: `${searchUrl}`, headers: headers }); // Parse search results - let mdData = JSON.parse(mdResponse) + let results = JSON.parse(response) .filter((result: any) => !this.find_similar_notes || !result.additional.file.endsWith(this.app.workspace.getActiveFile()?.path)) - .map((result: any) => { return { entry: result.entry, score: result.score, file: result.additional.file }; }); - let pdfData = JSON.parse(pdfResponse) - .filter((result: any) => !this.find_similar_notes || !result.additional.file.endsWith(this.app.workspace.getActiveFile()?.path)) - .map((result: any) => { return { entry: `## ${result.additional.compiled}`, score: result.score, file: result.additional.file } as SearchResult; }) - - // Combine markdown and PDF results and sort them by score - let results = mdData.concat(pdfData) - .sort((a: any, b: any) => b.score - a.score) - .map((result: any) => { return { entry: result.entry, file: result.file } as SearchResult; }) + .map((result: any) => { return { entry: result.entry, file: result.additional.file } as SearchResult; }); this.query = query; return results; diff --git a/src/app/README.md b/src/khoj/app/README.md similarity index 100% rename from src/app/README.md rename to src/khoj/app/README.md diff --git a/src/app/__init__.py b/src/khoj/app/__init__.py similarity index 100% rename from src/app/__init__.py rename to src/khoj/app/__init__.py diff --git a/src/app/settings.py b/src/khoj/app/settings.py similarity index 97% rename from src/app/settings.py rename to src/khoj/app/settings.py index 0803081d..721bcc87 100644 --- a/src/app/settings.py +++ b/src/khoj/app/settings.py @@ -14,7 +14,7 @@ from pathlib import Path import os # Build paths inside the project like this: BASE_DIR / 'subdir'. -BASE_DIR = Path(__file__).resolve().parent.parent.parent +BASE_DIR = Path(__file__).resolve().parent.parent.parent.parent # Quick-start development settings - unsuitable for production @@ -53,7 +53,7 @@ SESSION_COOKIE_SAMESITE = "None" INSTALLED_APPS = [ "django.contrib.auth", "django.contrib.contenttypes", - "database.apps.DatabaseConfig", + "khoj.database.apps.DatabaseConfig", "django.contrib.admin", "django.contrib.sessions", "django.contrib.messages", diff --git a/src/app/urls.py b/src/khoj/app/urls.py similarity index 100% rename from src/app/urls.py rename to src/khoj/app/urls.py diff --git a/src/app/wsgi.py b/src/khoj/app/wsgi.py similarity index 100% rename from src/app/wsgi.py rename to src/khoj/app/wsgi.py diff --git a/src/khoj/configure.py b/src/khoj/configure.py index 5ed92727..d34205d9 100644 --- a/src/khoj/configure.py +++ b/src/khoj/configure.py @@ -20,8 +20,8 @@ from starlette.authentication import ( ) # Internal Packages -from database.models import KhojUser, Subscription -from database.adapters import get_all_users, get_or_create_search_model +from khoj.database.models import KhojUser, Subscription +from khoj.database.adapters import get_all_users, get_or_create_search_model from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel from khoj.routers.indexer import configure_content, load_content, configure_search from khoj.utils import constants, state @@ -45,7 +45,7 @@ class UserAuthenticationBackend(AuthenticationBackend): def __init__( self, ): - from database.models import KhojUser, KhojApiUser + from khoj.database.models import KhojUser, KhojApiUser self.khojuser_manager = KhojUser.objects self.khojapiuser_manager = KhojApiUser.objects diff --git a/src/database/__init__.py b/src/khoj/database/__init__.py similarity index 100% rename from src/database/__init__.py rename to src/khoj/database/__init__.py diff --git a/src/database/adapters/__init__.py b/src/khoj/database/adapters/__init__.py similarity index 99% rename from src/database/adapters/__init__.py rename to src/khoj/database/adapters/__init__.py index 4cbf2d8b..0a263ed4 100644 --- a/src/database/adapters/__init__.py +++ b/src/khoj/database/adapters/__init__.py @@ -1,44 +1,40 @@ import math -from typing import Optional, Type, List -from datetime import date, datetime import secrets -from typing import Type, List -from datetime import date, timezone - -from django.db import models -from django.contrib.sessions.backends.db import SessionStore -from pgvector.django import CosineDistance -from django.db.models.manager import BaseManager -from django.db.models import Q -from torch import Tensor +from datetime import date, datetime, timezone +from typing import List, Optional, Type # Import sync_to_async from Django Channels from asgiref.sync import sync_to_async - +from django.contrib.sessions.backends.db import SessionStore +from django.db import models +from django.db.models import Q +from django.db.models.manager import BaseManager from fastapi import HTTPException +from pgvector.django import CosineDistance +from torch import Tensor -from database.models import ( - KhojUser, +from khoj.database.models import ( + ChatModelOptions, + Conversation, + Entry, + GithubConfig, + GithubRepoConfig, GoogleUser, KhojApiUser, + KhojUser, NotionConfig, - GithubConfig, - Entry, - GithubRepoConfig, - Conversation, - ChatModelOptions, + OfflineChatProcessorConversationConfig, + OpenAIProcessorConversationConfig, SearchModelConfig, Subscription, UserConversationConfig, - OpenAIProcessorConversationConfig, - OfflineChatProcessorConversationConfig, ) -from khoj.utils.helpers import generate_random_name +from khoj.search_filter.date_filter import DateFilter +from khoj.search_filter.file_filter import FileFilter +from khoj.search_filter.word_filter import WordFilter from khoj.utils import state from khoj.utils.config import GPT4AllProcessorModel -from khoj.search_filter.word_filter import WordFilter -from khoj.search_filter.file_filter import FileFilter -from khoj.search_filter.date_filter import DateFilter +from khoj.utils.helpers import generate_random_name async def set_notion_config(token: str, user: KhojUser): diff --git a/src/database/admin.py b/src/khoj/database/admin.py similarity index 93% rename from src/database/admin.py rename to src/khoj/database/admin.py index 8d2130ba..69f15b2f 100644 --- a/src/database/admin.py +++ b/src/khoj/database/admin.py @@ -3,7 +3,7 @@ from django.contrib.auth.admin import UserAdmin # Register your models here. -from database.models import ( +from khoj.database.models import ( KhojUser, ChatModelOptions, OpenAIProcessorConversationConfig, diff --git a/src/database/apps.py b/src/khoj/database/apps.py similarity index 82% rename from src/database/apps.py rename to src/khoj/database/apps.py index a3b71b13..eed64812 100644 --- a/src/database/apps.py +++ b/src/khoj/database/apps.py @@ -3,4 +3,4 @@ from django.apps import AppConfig class DatabaseConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" - name = "database" + name = "khoj.database" diff --git a/src/database/migrations/0001_khojuser.py b/src/khoj/database/migrations/0001_khojuser.py similarity index 100% rename from src/database/migrations/0001_khojuser.py rename to src/khoj/database/migrations/0001_khojuser.py diff --git a/src/database/migrations/0002_googleuser.py b/src/khoj/database/migrations/0002_googleuser.py similarity index 100% rename from src/database/migrations/0002_googleuser.py rename to src/khoj/database/migrations/0002_googleuser.py diff --git a/src/database/migrations/0003_vector_extension.py b/src/khoj/database/migrations/0003_vector_extension.py similarity index 100% rename from src/database/migrations/0003_vector_extension.py rename to src/khoj/database/migrations/0003_vector_extension.py diff --git a/src/database/migrations/0004_content_types_and_more.py b/src/khoj/database/migrations/0004_content_types_and_more.py similarity index 100% rename from src/database/migrations/0004_content_types_and_more.py rename to src/khoj/database/migrations/0004_content_types_and_more.py diff --git a/src/database/migrations/0005_embeddings_corpus_id.py b/src/khoj/database/migrations/0005_embeddings_corpus_id.py similarity index 100% rename from src/database/migrations/0005_embeddings_corpus_id.py rename to src/khoj/database/migrations/0005_embeddings_corpus_id.py diff --git a/src/database/migrations/0006_embeddingsdates.py b/src/khoj/database/migrations/0006_embeddingsdates.py similarity index 100% rename from src/database/migrations/0006_embeddingsdates.py rename to src/khoj/database/migrations/0006_embeddingsdates.py diff --git a/src/database/migrations/0007_add_conversation.py b/src/khoj/database/migrations/0007_add_conversation.py similarity index 100% rename from src/database/migrations/0007_add_conversation.py rename to src/khoj/database/migrations/0007_add_conversation.py diff --git a/src/database/migrations/0008_alter_conversation_conversation_log.py b/src/khoj/database/migrations/0008_alter_conversation_conversation_log.py similarity index 100% rename from src/database/migrations/0008_alter_conversation_conversation_log.py rename to src/khoj/database/migrations/0008_alter_conversation_conversation_log.py diff --git a/src/database/migrations/0009_khojapiuser.py b/src/khoj/database/migrations/0009_khojapiuser.py similarity index 100% rename from src/database/migrations/0009_khojapiuser.py rename to src/khoj/database/migrations/0009_khojapiuser.py diff --git a/src/database/migrations/0010_chatmodeloptions_and_more.py b/src/khoj/database/migrations/0010_chatmodeloptions_and_more.py similarity index 100% rename from src/database/migrations/0010_chatmodeloptions_and_more.py rename to src/khoj/database/migrations/0010_chatmodeloptions_and_more.py diff --git a/src/database/migrations/0010_rename_embeddings_entry_and_more.py b/src/khoj/database/migrations/0010_rename_embeddings_entry_and_more.py similarity index 100% rename from src/database/migrations/0010_rename_embeddings_entry_and_more.py rename to src/khoj/database/migrations/0010_rename_embeddings_entry_and_more.py diff --git a/src/database/migrations/0011_merge_20231102_0138.py b/src/khoj/database/migrations/0011_merge_20231102_0138.py similarity index 81% rename from src/database/migrations/0011_merge_20231102_0138.py rename to src/khoj/database/migrations/0011_merge_20231102_0138.py index 112c76a2..0206036e 100644 --- a/src/database/migrations/0011_merge_20231102_0138.py +++ b/src/khoj/database/migrations/0011_merge_20231102_0138.py @@ -2,6 +2,8 @@ from django.db import migrations +from typing import List, Any + class Migration(migrations.Migration): dependencies = [ @@ -9,4 +11,4 @@ class Migration(migrations.Migration): ("database", "0010_rename_embeddings_entry_and_more"), ] - operations = [] + operations: List[Any] = [] diff --git a/src/database/migrations/0012_entry_file_source.py b/src/khoj/database/migrations/0012_entry_file_source.py similarity index 100% rename from src/database/migrations/0012_entry_file_source.py rename to src/khoj/database/migrations/0012_entry_file_source.py diff --git a/src/database/migrations/0013_subscription.py b/src/khoj/database/migrations/0013_subscription.py similarity index 100% rename from src/database/migrations/0013_subscription.py rename to src/khoj/database/migrations/0013_subscription.py diff --git a/src/database/migrations/0014_alter_googleuser_picture.py b/src/khoj/database/migrations/0014_alter_googleuser_picture.py similarity index 100% rename from src/database/migrations/0014_alter_googleuser_picture.py rename to src/khoj/database/migrations/0014_alter_googleuser_picture.py diff --git a/src/database/migrations/0015_alter_subscription_user.py b/src/khoj/database/migrations/0015_alter_subscription_user.py similarity index 100% rename from src/database/migrations/0015_alter_subscription_user.py rename to src/khoj/database/migrations/0015_alter_subscription_user.py diff --git a/src/database/migrations/0016_alter_subscription_renewal_date.py b/src/khoj/database/migrations/0016_alter_subscription_renewal_date.py similarity index 100% rename from src/database/migrations/0016_alter_subscription_renewal_date.py rename to src/khoj/database/migrations/0016_alter_subscription_renewal_date.py diff --git a/src/database/migrations/0017_searchmodel.py b/src/khoj/database/migrations/0017_searchmodel.py similarity index 100% rename from src/database/migrations/0017_searchmodel.py rename to src/khoj/database/migrations/0017_searchmodel.py diff --git a/src/database/migrations/0018_searchmodelconfig_delete_searchmodel.py b/src/khoj/database/migrations/0018_searchmodelconfig_delete_searchmodel.py similarity index 100% rename from src/database/migrations/0018_searchmodelconfig_delete_searchmodel.py rename to src/khoj/database/migrations/0018_searchmodelconfig_delete_searchmodel.py diff --git a/src/database/migrations/0019_alter_googleuser_family_name_and_more.py b/src/khoj/database/migrations/0019_alter_googleuser_family_name_and_more.py similarity index 100% rename from src/database/migrations/0019_alter_googleuser_family_name_and_more.py rename to src/khoj/database/migrations/0019_alter_googleuser_family_name_and_more.py diff --git a/src/database/migrations/__init__.py b/src/khoj/database/migrations/__init__.py similarity index 100% rename from src/database/migrations/__init__.py rename to src/khoj/database/migrations/__init__.py diff --git a/src/database/models/__init__.py b/src/khoj/database/models/__init__.py similarity index 100% rename from src/database/models/__init__.py rename to src/khoj/database/models/__init__.py diff --git a/src/database/tests.py b/src/khoj/database/tests.py similarity index 100% rename from src/database/tests.py rename to src/khoj/database/tests.py diff --git a/src/khoj/main.py b/src/khoj/main.py index 9fa65fc3..c5e1f277 100644 --- a/src/khoj/main.py +++ b/src/khoj/main.py @@ -1,3 +1,7 @@ +""" Main module for Khoj Assistant + isort:skip_file +""" + # Standard Packages import os import sys @@ -25,7 +29,7 @@ from django.core.asgi import get_asgi_application from django.core.management import call_command # Initialize Django -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "app.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "khoj.app.settings") django.setup() # Initialize Django Database diff --git a/src/manage.py b/src/khoj/manage.py similarity index 89% rename from src/manage.py rename to src/khoj/manage.py index 1a64b14a..9b8f4b27 100755 --- a/src/manage.py +++ b/src/khoj/manage.py @@ -6,7 +6,7 @@ import sys def main(): """Run administrative tasks.""" - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "app.settings") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "khoj.app.settings") try: from django.core.management import execute_from_command_line except ImportError as exc: diff --git a/src/khoj/migrations/migrate_server_pg.py b/src/khoj/migrations/migrate_server_pg.py index 434e27d7..097ae0d6 100644 --- a/src/khoj/migrations/migrate_server_pg.py +++ b/src/khoj/migrations/migrate_server_pg.py @@ -60,7 +60,7 @@ import logging from packaging import version from khoj.utils.yaml import load_config_from_file, save_config_to_file -from database.models import ( +from khoj.database.models import ( OpenAIProcessorConversationConfig, OfflineChatProcessorConversationConfig, ChatModelOptions, diff --git a/src/khoj/processor/data_sources/github/github_to_entries.py b/src/khoj/processor/data_sources/github/github_to_entries.py index 592cfcf0..05d18d8a 100644 --- a/src/khoj/processor/data_sources/github/github_to_entries.py +++ b/src/khoj/processor/data_sources/github/github_to_entries.py @@ -2,19 +2,20 @@ import logging import time from datetime import datetime -from typing import Dict, List, Union, Tuple +from typing import Dict, List, Tuple, Union # External Packages import requests -# Internal Packages -from khoj.utils.helpers import timer -from khoj.utils.rawconfig import Entry, GithubContentConfig, GithubRepoConfig +from khoj.database.models import Entry as DbEntry +from khoj.database.models import GithubConfig, KhojUser from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries from khoj.processor.text_to_entries import TextToEntries -from database.models import Entry as DbEntry, GithubConfig, KhojUser +# Internal Packages +from khoj.utils.helpers import timer +from khoj.utils.rawconfig import Entry, GithubContentConfig, GithubRepoConfig logger = logging.getLogger(__name__) diff --git a/src/khoj/processor/data_sources/markdown/markdown_to_entries.py b/src/khoj/processor/data_sources/markdown/markdown_to_entries.py index 0dd71740..05531d70 100644 --- a/src/khoj/processor/data_sources/markdown/markdown_to_entries.py +++ b/src/khoj/processor/data_sources/markdown/markdown_to_entries.py @@ -1,17 +1,19 @@ # Standard Packages import logging import re -import urllib3 from pathlib import Path -from typing import Tuple, List +from typing import List, Tuple + +import urllib3 + +from khoj.database.models import Entry as DbEntry +from khoj.database.models import KhojUser # Internal Packages from khoj.processor.text_to_entries import TextToEntries -from khoj.utils.helpers import timer from khoj.utils.constants import empty_escape_sequences +from khoj.utils.helpers import timer from khoj.utils.rawconfig import Entry -from database.models import Entry as DbEntry, KhojUser - logger = logging.getLogger(__name__) diff --git a/src/khoj/processor/data_sources/notion/notion_to_entries.py b/src/khoj/processor/data_sources/notion/notion_to_entries.py index 7a88e2a1..4a2fd817 100644 --- a/src/khoj/processor/data_sources/notion/notion_to_entries.py +++ b/src/khoj/processor/data_sources/notion/notion_to_entries.py @@ -1,19 +1,18 @@ # Standard Packages import logging +from enum import Enum from typing import Tuple # External Packages import requests +from khoj.database.models import Entry as DbEntry +from khoj.database.models import KhojUser, NotionConfig +from khoj.processor.text_to_entries import TextToEntries + # Internal Packages from khoj.utils.helpers import timer from khoj.utils.rawconfig import Entry, NotionContentConfig -from khoj.processor.text_to_entries import TextToEntries -from khoj.utils.rawconfig import Entry -from database.models import Entry as DbEntry, KhojUser, NotionConfig - -from enum import Enum - logger = logging.getLogger(__name__) diff --git a/src/khoj/processor/data_sources/org_mode/org_to_entries.py b/src/khoj/processor/data_sources/org_mode/org_to_entries.py index b8d118a7..9e0d8e54 100644 --- a/src/khoj/processor/data_sources/org_mode/org_to_entries.py +++ b/src/khoj/processor/data_sources/org_mode/org_to_entries.py @@ -3,14 +3,15 @@ import logging from pathlib import Path from typing import Iterable, List, Tuple +from khoj.database.models import Entry as DbEntry +from khoj.database.models import KhojUser + # Internal Packages from khoj.processor.data_sources.org_mode import orgnode from khoj.processor.text_to_entries import TextToEntries +from khoj.utils import state from khoj.utils.helpers import timer from khoj.utils.rawconfig import Entry -from khoj.utils import state -from database.models import Entry as DbEntry, KhojUser - logger = logging.getLogger(__name__) diff --git a/src/khoj/processor/data_sources/pdf/pdf_to_entries.py b/src/khoj/processor/data_sources/pdf/pdf_to_entries.py index 3a47096a..f1769a49 100644 --- a/src/khoj/processor/data_sources/pdf/pdf_to_entries.py +++ b/src/khoj/processor/data_sources/pdf/pdf_to_entries.py @@ -1,18 +1,19 @@ # Standard Packages -import os -import logging -from typing import List, Tuple import base64 +import logging +import os +from typing import List, Tuple # External Packages from langchain.document_loaders import PyMuPDFLoader +from khoj.database.models import Entry as DbEntry +from khoj.database.models import KhojUser + # Internal Packages from khoj.processor.text_to_entries import TextToEntries from khoj.utils.helpers import timer from khoj.utils.rawconfig import Entry -from database.models import Entry as DbEntry, KhojUser - logger = logging.getLogger(__name__) diff --git a/src/khoj/processor/data_sources/plaintext/plaintext_to_entries.py b/src/khoj/processor/data_sources/plaintext/plaintext_to_entries.py index d42dae30..39966bef 100644 --- a/src/khoj/processor/data_sources/plaintext/plaintext_to_entries.py +++ b/src/khoj/processor/data_sources/plaintext/plaintext_to_entries.py @@ -2,15 +2,16 @@ import logging from pathlib import Path from typing import List, Tuple + from bs4 import BeautifulSoup +from khoj.database.models import Entry as DbEntry +from khoj.database.models import KhojUser # Internal Packages from khoj.processor.text_to_entries import TextToEntries from khoj.utils.helpers import timer from khoj.utils.rawconfig import Entry -from database.models import Entry as DbEntry, KhojUser - logger = logging.getLogger(__name__) diff --git a/src/khoj/processor/text_to_entries.py b/src/khoj/processor/text_to_entries.py index ac42105a..109c58e6 100644 --- a/src/khoj/processor/text_to_entries.py +++ b/src/khoj/processor/text_to_entries.py @@ -13,8 +13,8 @@ from khoj.utils.helpers import is_none_or_empty, timer, batcher # Internal Packages from khoj.utils.rawconfig import Entry from khoj.search_filter.date_filter import DateFilter -from database.models import KhojUser, Entry as DbEntry, EntryDates -from database.adapters import EntryAdapters +from khoj.database.models import KhojUser, Entry as DbEntry, EntryDates +from khoj.database.adapters import EntryAdapters logger = logging.getLogger(__name__) diff --git a/src/khoj/routers/api.py b/src/khoj/routers/api.py index 10db3a83..1b90127b 100644 --- a/src/khoj/routers/api.py +++ b/src/khoj/routers/api.py @@ -1,64 +1,63 @@ # Standard Packages import concurrent.futures +import json +import logging import math import time -import logging -import json -from typing import List, Optional, Union, Any, Dict +from typing import Any, Dict, List, Optional, Union + +from asgiref.sync import sync_to_async # External Packages -from fastapi import APIRouter, Depends, HTTPException, Header, Request +from fastapi import APIRouter, Depends, Header, HTTPException, Request +from fastapi.requests import Request +from fastapi.responses import Response, StreamingResponse from starlette.authentication import requires -from asgiref.sync import sync_to_async # Internal Packages from khoj.configure import configure_server -from khoj.search_type import image_search, text_search -from khoj.search_filter.date_filter import DateFilter -from khoj.search_filter.file_filter import FileFilter -from khoj.search_filter.word_filter import WordFilter -from khoj.utils.config import TextSearchModel, GPT4AllProcessorModel -from khoj.utils.helpers import ConversationCommand, is_none_or_empty, timer, command_descriptions -from khoj.utils.rawconfig import ( - FullConfig, - SearchConfig, - SearchResponse, - GithubContentConfig, - NotionContentConfig, -) -from khoj.utils.state import SearchType -from khoj.utils import state, constants -from khoj.utils.helpers import AsyncIteratorWrapper, get_device -from fastapi.responses import StreamingResponse, Response -from khoj.routers.helpers import ( - CommonQueryParams, - get_conversation_command, - validate_conversation_config, - agenerate_chat_response, - update_telemetry_state, - is_ready_to_chat, - ApiUserRateLimiter, -) -from khoj.processor.conversation.prompts import help_message, no_entries_found -from khoj.processor.conversation.openai.gpt import extract_questions -from khoj.processor.conversation.gpt4all.chat_model import extract_questions_offline -from khoj.processor.tools.online_search import search_with_google -from fastapi.requests import Request - -from database import adapters -from database.adapters import EntryAdapters, ConversationAdapters -from database.models import ( +from khoj.database import adapters +from khoj.database.adapters import ConversationAdapters, EntryAdapters +from khoj.database.models import ChatModelOptions +from khoj.database.models import Entry as DbEntry +from khoj.database.models import ( + GithubConfig, + KhojUser, LocalMarkdownConfig, LocalOrgConfig, LocalPdfConfig, LocalPlaintextConfig, - KhojUser, - Entry as DbEntry, - GithubConfig, NotionConfig, - ChatModelOptions, ) - +from khoj.processor.conversation.gpt4all.chat_model import extract_questions_offline +from khoj.processor.conversation.openai.gpt import extract_questions +from khoj.processor.conversation.prompts import help_message, no_entries_found +from khoj.processor.tools.online_search import search_with_google +from khoj.routers.helpers import ( + ApiUserRateLimiter, + CommonQueryParams, + agenerate_chat_response, + get_conversation_command, + is_ready_to_chat, + update_telemetry_state, + validate_conversation_config, +) +from khoj.search_filter.date_filter import DateFilter +from khoj.search_filter.file_filter import FileFilter +from khoj.search_filter.word_filter import WordFilter +from khoj.search_type import image_search, text_search +from khoj.utils import constants, state +from khoj.utils.config import GPT4AllProcessorModel, TextSearchModel +from khoj.utils.helpers import ( + AsyncIteratorWrapper, + ConversationCommand, + command_descriptions, + get_device, + is_none_or_empty, + timer, +) +from khoj.utils.rawconfig import FullConfig, GithubContentConfig, NotionContentConfig, SearchConfig, SearchResponse +from khoj.utils.state import SearchType # Initialize Router api = APIRouter() diff --git a/src/khoj/routers/auth.py b/src/khoj/routers/auth.py index a9a88325..df119548 100644 --- a/src/khoj/routers/auth.py +++ b/src/khoj/routers/auth.py @@ -15,8 +15,8 @@ from google.oauth2 import id_token from google.auth.transport import requests as google_requests # Internal Packages -from database.adapters import get_khoj_tokens, get_or_create_user, create_khoj_token, delete_khoj_token -from database.models import KhojApiUser +from khoj.database.adapters import get_khoj_tokens, get_or_create_user, create_khoj_token, delete_khoj_token +from khoj.database.models import KhojApiUser from khoj.routers.helpers import update_telemetry_state from khoj.utils import state diff --git a/src/khoj/routers/helpers.py b/src/khoj/routers/helpers.py index cdb91b6f..c6fcb436 100644 --- a/src/khoj/routers/helpers.py +++ b/src/khoj/routers/helpers.py @@ -1,31 +1,28 @@ # Standard Packages import asyncio +import json +import logging from collections import defaultdict from concurrent.futures import ThreadPoolExecutor from datetime import datetime from functools import partial -import logging from time import time -import json -from typing import Annotated, Iterator, List, Optional, Union, Tuple, Dict, Any - -from datetime import datetime - -from khoj.processor.conversation import prompts +from typing import Annotated, Any, Dict, Iterator, List, Optional, Tuple, Union # External Packages -from fastapi import HTTPException, Header, Request, Depends +from fastapi import Depends, Header, HTTPException, Request + +from khoj.database.adapters import ConversationAdapters +from khoj.database.models import KhojUser, Subscription +from khoj.processor.conversation import prompts +from khoj.processor.conversation.gpt4all.chat_model import converse_offline, send_message_to_model_offline +from khoj.processor.conversation.openai.gpt import converse, send_message_to_model +from khoj.processor.conversation.utils import ThreadedGenerator, message_to_log # Internal Packages from khoj.utils import state from khoj.utils.config import GPT4AllProcessorModel from khoj.utils.helpers import ConversationCommand, log_telemetry -from khoj.processor.conversation.openai.gpt import converse, send_message_to_model -from khoj.processor.conversation.gpt4all.chat_model import converse_offline, send_message_to_model_offline -from khoj.processor.conversation.utils import message_to_log, ThreadedGenerator -from database.models import KhojUser, Subscription, ChatModelOptions -from database.adapters import ConversationAdapters - logger = logging.getLogger(__name__) diff --git a/src/khoj/routers/indexer.py b/src/khoj/routers/indexer.py index 6c4cc9dd..00303f31 100644 --- a/src/khoj/routers/indexer.py +++ b/src/khoj/routers/indexer.py @@ -1,40 +1,26 @@ # Standard Packages -import logging -from typing import Optional, Union, Dict import asyncio +import logging +from typing import Dict, Optional, Union -# External Packages from fastapi import APIRouter, Header, Request, Response, UploadFile from pydantic import BaseModel from starlette.authentication import requires -# Internal Packages -from khoj.utils import state, constants +from khoj.database.models import GithubConfig, KhojUser, NotionConfig +from khoj.processor.data_sources.github.github_to_entries import GithubToEntries from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries +from khoj.processor.data_sources.notion.notion_to_entries import NotionToEntries from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries from khoj.processor.data_sources.pdf.pdf_to_entries import PdfToEntries -from khoj.processor.data_sources.github.github_to_entries import GithubToEntries -from khoj.processor.data_sources.notion.notion_to_entries import NotionToEntries from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries -from khoj.search_type import text_search, image_search from khoj.routers.helpers import update_telemetry_state -from khoj.utils.yaml import save_config_to_file_updated_state -from khoj.utils.config import SearchModels +from khoj.search_type import image_search, text_search +from khoj.utils import constants, state +from khoj.utils.config import ContentIndex, SearchModels from khoj.utils.helpers import LRU, get_file_type -from khoj.utils.rawconfig import ( - ContentConfig, - FullConfig, - SearchConfig, -) -from khoj.utils.config import ( - ContentIndex, - SearchModels, -) -from database.models import ( - KhojUser, - GithubConfig, - NotionConfig, -) +from khoj.utils.rawconfig import ContentConfig, FullConfig, SearchConfig +from khoj.utils.yaml import save_config_to_file_updated_state logger = logging.getLogger(__name__) diff --git a/src/khoj/routers/subscription.py b/src/khoj/routers/subscription.py index 62e50d72..edcbc135 100644 --- a/src/khoj/routers/subscription.py +++ b/src/khoj/routers/subscription.py @@ -10,7 +10,7 @@ from starlette.authentication import requires import stripe # Internal Packages -from database import adapters +from khoj.database import adapters # Stripe integration for Khoj Cloud Subscription diff --git a/src/khoj/routers/web_client.py b/src/khoj/routers/web_client.py index f30499d8..dab16fa8 100644 --- a/src/khoj/routers/web_client.py +++ b/src/khoj/routers/web_client.py @@ -8,8 +8,8 @@ from fastapi import Request from fastapi.responses import HTMLResponse, FileResponse, RedirectResponse from fastapi.templating import Jinja2Templates from starlette.authentication import requires -from database import adapters -from database.models import KhojUser +from khoj.database import adapters +from khoj.database.models import KhojUser from khoj.utils.rawconfig import ( GithubContentConfig, GithubRepoConfig, @@ -18,7 +18,7 @@ from khoj.utils.rawconfig import ( # Internal Packages from khoj.utils import constants, state -from database.adapters import ( +from khoj.database.adapters import ( EntryAdapters, get_user_github_config, get_user_notion_config, diff --git a/src/khoj/search_type/text_search.py b/src/khoj/search_type/text_search.py index 7e295903..a78ce522 100644 --- a/src/khoj/search_type/text_search.py +++ b/src/khoj/search_type/text_search.py @@ -19,8 +19,8 @@ from khoj.utils.state import SearchType from khoj.utils.rawconfig import SearchResponse, Entry from khoj.utils.jsonl import load_jsonl from khoj.processor.text_to_entries import TextToEntries -from database.adapters import EntryAdapters -from database.models import KhojUser, Entry as DbEntry +from khoj.database.adapters import EntryAdapters +from khoj.database.models import KhojUser, Entry as DbEntry logger = logging.getLogger(__name__) diff --git a/src/khoj/utils/constants.py b/src/khoj/utils/constants.py index 8a106153..ec23cddd 100644 --- a/src/khoj/utils/constants.py +++ b/src/khoj/utils/constants.py @@ -7,6 +7,7 @@ app_env_filepath = "~/.khoj/env" telemetry_server = "https://khoj.beta.haletic.com/v1/telemetry" content_directory = "~/.khoj/content/" default_offline_chat_model = "mistral-7b-instruct-v0.1.Q4_0.gguf" +default_online_chat_model = "gpt-4" empty_config = { "search-type": { diff --git a/src/khoj/utils/fs_syncer.py b/src/khoj/utils/fs_syncer.py index fc7e4a2d..57c79d9d 100644 --- a/src/khoj/utils/fs_syncer.py +++ b/src/khoj/utils/fs_syncer.py @@ -7,7 +7,7 @@ from bs4 import BeautifulSoup from khoj.utils.helpers import get_absolute_path, is_none_or_empty from khoj.utils.rawconfig import TextContentConfig from khoj.utils.config import SearchType -from database.models import LocalMarkdownConfig, LocalOrgConfig, LocalPdfConfig, LocalPlaintextConfig +from khoj.database.models import LocalMarkdownConfig, LocalOrgConfig, LocalPdfConfig, LocalPlaintextConfig logger = logging.getLogger(__name__) diff --git a/src/khoj/utils/initialization.py b/src/khoj/utils/initialization.py index c797f848..ffc4d47e 100644 --- a/src/khoj/utils/initialization.py +++ b/src/khoj/utils/initialization.py @@ -1,16 +1,17 @@ import logging import os -from database.models import ( +from khoj.database.models import ( KhojUser, OfflineChatProcessorConversationConfig, OpenAIProcessorConversationConfig, ChatModelOptions, ) -from khoj.utils.constants import default_offline_chat_model +from khoj.utils.constants import default_offline_chat_model, default_online_chat_model +from khoj.processor.conversation.utils import model_to_prompt_size, model_to_tokenizer -from database.adapters import ConversationAdapters +from khoj.database.adapters import ConversationAdapters logger = logging.getLogger(__name__) @@ -30,11 +31,6 @@ def initialization(): logger.info( "🗣️ Configure chat models available to your server. You can always update these at /server/admin using the credentials of your admin account" ) - try: - # Some environments don't support interactive input. We catch the exception and return if that's the case. The admin can still configure their settings from the admin page. - input() - except EOFError: - return try: # Note: gpt4all package is not available on all devices. @@ -47,15 +43,27 @@ def initialization(): OfflineChatProcessorConversationConfig.objects.create(enabled=True) offline_chat_model = input( - f"Enter the name of the offline chat model you want to use, based on the models in HuggingFace (press enter to use the default: {default_offline_chat_model}): " + f"Enter the offline chat model you want to use, See GPT4All for supported models (default: {default_offline_chat_model}): " ) if offline_chat_model == "": ChatModelOptions.objects.create( chat_model=default_offline_chat_model, model_type=ChatModelOptions.ModelType.OFFLINE ) else: - max_tokens = input("Enter the maximum number of tokens to use for the offline chat model:") - tokenizer = input("Enter the tokenizer to use for the offline chat model:") + default_max_tokens = model_to_prompt_size.get(offline_chat_model, 2000) + max_tokens = input( + f"Enter the maximum number of tokens to use for the offline chat model (default {default_max_tokens}):" + ) + max_tokens = max_tokens or default_max_tokens + + default_tokenizer = model_to_tokenizer.get( + offline_chat_model, "hf-internal-testing/llama-tokenizer" + ) + tokenizer = input( + f"Enter the tokenizer to use for the offline chat model (default: {default_tokenizer}):" + ) + tokenizer = tokenizer or default_tokenizer + ChatModelOptions.objects.create( chat_model=offline_chat_model, model_type=ChatModelOptions.ModelType.OFFLINE, @@ -71,10 +79,19 @@ def initialization(): logger.info("🗣️ Setting up OpenAI chat model") api_key = input("Enter your OpenAI API key: ") OpenAIProcessorConversationConfig.objects.create(api_key=api_key) - openai_chat_model = input("Enter the name of the OpenAI chat model you want to use: ") - max_tokens = input("Enter the maximum number of tokens to use for the OpenAI chat model:") + + openai_chat_model = input( + f"Enter the OpenAI chat model you want to use (default: {default_online_chat_model}): " + ) + openai_chat_model = openai_chat_model or default_online_chat_model + + default_max_tokens = model_to_prompt_size.get(openai_chat_model, 2000) + max_tokens = input( + f"Enter the maximum number of tokens to use for the OpenAI chat model (default: {default_max_tokens}): " + ) + max_tokens = max_tokens or default_max_tokens ChatModelOptions.objects.create( - chat_model=openai_chat_model, model_type=ChatModelOptions.ModelType.OPENAI, max_tokens=max_tokens + chat_model=openai_chat_model, model_type=ChatModelOptions.ModelType.OPENAI, max_prompt_size=max_tokens ) logger.info("🗣️ Chat model configuration complete") @@ -94,5 +111,8 @@ def initialization(): try: _create_chat_configuration() break + # Some environments don't support interactive input. We catch the exception and return if that's the case. The admin can still configure their settings from the admin page. + except EOFError: + return except Exception as e: logger.error(f"🚨 Failed to create chat configuration: {e}", exc_info=True) diff --git a/tests/conftest.py b/tests/conftest.py index 669759e5..c18c5f80 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,48 +1,40 @@ # External Packages import os -from fastapi.testclient import TestClient from pathlib import Path -import pytest -from fastapi.staticfiles import StaticFiles -from fastapi import FastAPI -import os -from fastapi import FastAPI +import pytest +from fastapi import FastAPI +from fastapi.staticfiles import StaticFiles +from fastapi.testclient import TestClient # Internal Packages -from khoj.configure import configure_routes, configure_search_types, configure_middleware -from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel +from khoj.configure import configure_middleware, configure_routes, configure_search_types +from khoj.database.models import ( + GithubConfig, + GithubRepoConfig, + KhojApiUser, + KhojUser, + LocalMarkdownConfig, + LocalOrgConfig, + LocalPlaintextConfig, +) +from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries +from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel +from khoj.routers.indexer import configure_content from khoj.search_type import image_search, text_search +from khoj.utils import fs_syncer, state from khoj.utils.config import SearchModels from khoj.utils.constants import web_directory from khoj.utils.helpers import resolve_absolute_path -from khoj.utils.rawconfig import ( - ContentConfig, - ImageContentConfig, - SearchConfig, - ImageSearchConfig, -) -from khoj.utils import state, fs_syncer -from khoj.routers.indexer import configure_content -from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries -from database.models import ( - KhojApiUser, - LocalOrgConfig, - LocalMarkdownConfig, - LocalPlaintextConfig, - GithubConfig, - KhojUser, - GithubRepoConfig, -) - +from khoj.utils.rawconfig import ContentConfig, ImageContentConfig, ImageSearchConfig, SearchConfig from tests.helpers import ( - UserFactory, ChatModelOptionsFactory, - OpenAIProcessorConversationConfigFactory, OfflineChatProcessorConversationConfigFactory, - UserConversationProcessorConfigFactory, + OpenAIProcessorConversationConfigFactory, SubscriptionFactory, + UserConversationProcessorConfigFactory, + UserFactory, ) diff --git a/tests/helpers.py b/tests/helpers.py index 079eb475..d0f9babc 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,7 +1,7 @@ import factory import os -from database.models import ( +from khoj.database.models import ( KhojUser, KhojApiUser, ChatModelOptions, diff --git a/tests/test_client.py b/tests/test_client.py index 52de0c95..aecd0498 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,23 +1,23 @@ # Standard Modules from io import BytesIO -from PIL import Image from urllib.parse import quote + import pytest +from fastapi import FastAPI # External Packages from fastapi.testclient import TestClient -from fastapi import FastAPI -import pytest +from PIL import Image # Internal Packages from khoj.configure import configure_routes, configure_search_types -from khoj.utils import state -from khoj.utils.state import search_models, content_index, config -from khoj.search_type import text_search, image_search -from khoj.utils.rawconfig import ContentConfig, SearchConfig +from khoj.database.adapters import EntryAdapters +from khoj.database.models import KhojApiUser, KhojUser from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries -from database.models import KhojUser, KhojApiUser -from database.adapters import EntryAdapters +from khoj.search_type import image_search, text_search +from khoj.utils import state +from khoj.utils.rawconfig import ContentConfig, SearchConfig +from khoj.utils.state import config, content_index, search_models # Test diff --git a/tests/test_multiple_users.py b/tests/test_multiple_users.py index 94d0560c..d2d7737b 100644 --- a/tests/test_multiple_users.py +++ b/tests/test_multiple_users.py @@ -1,24 +1,14 @@ # Standard Modules -from io import BytesIO -from PIL import Image from urllib.parse import quote -import pytest # External Packages -from fastapi.testclient import TestClient -from fastapi import FastAPI, UploadFile -from io import BytesIO import pytest -# Internal Packages -from khoj.configure import configure_routes, configure_search_types -from khoj.utils import state -from khoj.utils.state import search_models, content_index, config -from khoj.search_type import text_search, image_search -from khoj.utils.rawconfig import ContentConfig, SearchConfig +from khoj.database.models import KhojApiUser, KhojUser from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries -from database.models import KhojUser, KhojApiUser -from database.adapters import EntryAdapters + +# Internal Packages +from khoj.search_type import text_search # ---------------------------------------------------------------------------------------------------- diff --git a/tests/test_openai_chat_director.py b/tests/test_openai_chat_director.py index 07c4e0d8..b4e63364 100644 --- a/tests/test_openai_chat_director.py +++ b/tests/test_openai_chat_director.py @@ -10,7 +10,7 @@ from khoj.processor.conversation import prompts # Internal Packages from khoj.processor.conversation.utils import message_to_log from tests.helpers import ConversationFactory -from database.models import KhojUser +from khoj.database.models import KhojUser # Initialize variables for tests api_key = os.getenv("OPENAI_API_KEY") diff --git a/tests/test_plaintext_to_entries.py b/tests/test_plaintext_to_entries.py index 91a6569c..393fac17 100644 --- a/tests/test_plaintext_to_entries.py +++ b/tests/test_plaintext_to_entries.py @@ -3,11 +3,12 @@ import json import os from pathlib import Path +from khoj.database.models import KhojUser, LocalPlaintextConfig +from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries + # Internal Packages from khoj.utils.fs_syncer import get_plaintext_files from khoj.utils.rawconfig import TextContentConfig -from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries -from database.models import LocalPlaintextConfig, KhojUser def test_plaintext_file(tmp_path): diff --git a/tests/test_text_search.py b/tests/test_text_search.py index 4d08d5e8..f19903ae 100644 --- a/tests/test_text_search.py +++ b/tests/test_text_search.py @@ -1,19 +1,20 @@ # System Packages -import logging -from pathlib import Path -import os import asyncio +import logging +import os +from pathlib import Path # External Packages import pytest +from khoj.database.models import Entry, GithubConfig, KhojUser, LocalOrgConfig +from khoj.processor.data_sources.github.github_to_entries import GithubToEntries +from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries + # Internal Packages from khoj.search_type import text_search -from khoj.utils.rawconfig import ContentConfig, SearchConfig -from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries -from khoj.processor.data_sources.github.github_to_entries import GithubToEntries from khoj.utils.fs_syncer import collect_files, get_org_files -from database.models import LocalOrgConfig, KhojUser, Entry, GithubConfig +from khoj.utils.rawconfig import ContentConfig, SearchConfig logger = logging.getLogger(__name__)