From 4e98acbca754dc485eb257a567361da967871542 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Mon, 20 Nov 2023 14:52:37 -0800 Subject: [PATCH 01/14] Update minimum pydantic version to one with model_validate function --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a457aec6..17c11e77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ dependencies = [ "tiktoken >= 0.3.2", "tenacity >= 8.2.2", "pillow == 9.3.0", - "pydantic >= 1.10.10", + "pydantic >= 2.0.0", "pyyaml == 6.0", "rich >= 13.3.1", "schedule == 1.1.0", From d61b0dd55c0b78db6ea5cf080a03dc7f74ec9471 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Mon, 20 Nov 2023 14:55:00 -0800 Subject: [PATCH 02/14] Add Khoj Django app package to sys path to load Django module via pip install --- src/khoj/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/khoj/main.py b/src/khoj/main.py index 9fa65fc3..022efcc1 100644 --- a/src/khoj/main.py +++ b/src/khoj/main.py @@ -25,6 +25,7 @@ from django.core.asgi import get_asgi_application from django.core.management import call_command # Initialize Django +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "app.settings") django.setup() From c07401cf76ae56f298b0c312ef2347ff43245b64 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Mon, 20 Nov 2023 16:43:48 -0800 Subject: [PATCH 03/14] Fix, Improve chat config via CLI on first run by using defaults - Fix setting prompt size for online chat - generally improve chat config via cli by using default chat model, prompt size for online and offline chat --- src/khoj/utils/constants.py | 1 + src/khoj/utils/initialization.py | 44 +++++++++++++++++++++++--------- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/khoj/utils/constants.py b/src/khoj/utils/constants.py index 8a106153..ec23cddd 100644 --- a/src/khoj/utils/constants.py +++ b/src/khoj/utils/constants.py @@ -7,6 +7,7 @@ app_env_filepath = "~/.khoj/env" telemetry_server = "https://khoj.beta.haletic.com/v1/telemetry" content_directory = "~/.khoj/content/" default_offline_chat_model = "mistral-7b-instruct-v0.1.Q4_0.gguf" +default_online_chat_model = "gpt-4" empty_config = { "search-type": { diff --git a/src/khoj/utils/initialization.py b/src/khoj/utils/initialization.py index c797f848..b2d7aad5 100644 --- a/src/khoj/utils/initialization.py +++ b/src/khoj/utils/initialization.py @@ -8,7 +8,8 @@ from database.models import ( ChatModelOptions, ) -from khoj.utils.constants import default_offline_chat_model +from khoj.utils.constants import default_offline_chat_model, default_online_chat_model +from khoj.processor.conversation.utils import model_to_prompt_size, model_to_tokenizer from database.adapters import ConversationAdapters @@ -30,11 +31,6 @@ def initialization(): logger.info( "🗣️ Configure chat models available to your server. You can always update these at /server/admin using the credentials of your admin account" ) - try: - # Some environments don't support interactive input. We catch the exception and return if that's the case. The admin can still configure their settings from the admin page. - input() - except EOFError: - return try: # Note: gpt4all package is not available on all devices. @@ -47,15 +43,27 @@ def initialization(): OfflineChatProcessorConversationConfig.objects.create(enabled=True) offline_chat_model = input( - f"Enter the name of the offline chat model you want to use, based on the models in HuggingFace (press enter to use the default: {default_offline_chat_model}): " + f"Enter the offline chat model you want to use, See GPT4All for supported models (default: {default_offline_chat_model}): " ) if offline_chat_model == "": ChatModelOptions.objects.create( chat_model=default_offline_chat_model, model_type=ChatModelOptions.ModelType.OFFLINE ) else: - max_tokens = input("Enter the maximum number of tokens to use for the offline chat model:") - tokenizer = input("Enter the tokenizer to use for the offline chat model:") + default_max_tokens = model_to_prompt_size.get(offline_chat_model, 2000) + max_tokens = input( + f"Enter the maximum number of tokens to use for the offline chat model (default {default_max_tokens}):" + ) + max_tokens = max_tokens or default_max_tokens + + default_tokenizer = model_to_tokenizer.get( + offline_chat_model, "hf-internal-testing/llama-tokenizer" + ) + tokenizer = input( + f"Enter the tokenizer to use for the offline chat model (default: {default_tokenizer}):" + ) + tokenizer = tokenizer or default_tokenizer + ChatModelOptions.objects.create( chat_model=offline_chat_model, model_type=ChatModelOptions.ModelType.OFFLINE, @@ -71,10 +79,19 @@ def initialization(): logger.info("🗣️ Setting up OpenAI chat model") api_key = input("Enter your OpenAI API key: ") OpenAIProcessorConversationConfig.objects.create(api_key=api_key) - openai_chat_model = input("Enter the name of the OpenAI chat model you want to use: ") - max_tokens = input("Enter the maximum number of tokens to use for the OpenAI chat model:") + + openai_chat_model = input( + f"Enter the OpenAI chat model you want to use (default: {default_online_chat_model}): " + ) + openai_chat_model = openai_chat_model or default_online_chat_model + + default_max_tokens = model_to_prompt_size.get(openai_chat_model, 2000) + max_tokens = input( + f"Enter the maximum number of tokens to use for the OpenAI chat model (default: {default_max_tokens}): " + ) + max_tokens = max_tokens or default_max_tokens ChatModelOptions.objects.create( - chat_model=openai_chat_model, model_type=ChatModelOptions.ModelType.OPENAI, max_tokens=max_tokens + chat_model=openai_chat_model, model_type=ChatModelOptions.ModelType.OPENAI, max_prompt_size=max_tokens ) logger.info("🗣️ Chat model configuration complete") @@ -94,5 +111,8 @@ def initialization(): try: _create_chat_configuration() break + # Some environments don't support interactive input. We catch the exception and return if that's the case. The admin can still configure their settings from the admin page. + except EOFError: + return except Exception as e: logger.error(f"🚨 Failed to create chat configuration: {e}", exc_info=True) From f142999bce155ea9c2fac72788a364de69dce9f7 Mon Sep 17 00:00:00 2001 From: Daniel Grossmann-Kavanagh Date: Mon, 20 Nov 2023 17:07:30 -0800 Subject: [PATCH 04/14] fix khoj local server usage --- src/interface/obsidian/src/main.ts | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/interface/obsidian/src/main.ts b/src/interface/obsidian/src/main.ts index 26b0a5a1..b45a7fb5 100644 --- a/src/interface/obsidian/src/main.ts +++ b/src/interface/obsidian/src/main.ts @@ -73,21 +73,19 @@ export default class Khoj extends Plugin { // Check if khoj backend is configured, note if cannot connect to backend let headers = { "Authorization": `Bearer ${this.settings.khojApiKey}` }; - if (this.settings.khojUrl === "https://app.khoj.dev") { - if (this.settings.khojApiKey === "") { - new Notice(`❗️Khoj API key is not configured. Please visit https://app.khoj.dev/config#clients to get an API key.`); - return; - } - - await request({ url: this.settings.khojUrl ,method: "GET", headers: headers }) - .then(response => { - this.settings.connectedToBackend = true; - }) - .catch(error => { - this.settings.connectedToBackend = false; - new Notice(`❗️Ensure Khoj backend is running and Khoj URL is pointing to it in the plugin settings.\n\n${error}`); - }); + if (this.settings.khojApiKey === "" && this.settings.khojUrl === "https://app.khoj.dev") { + new Notice(`❗️Khoj API key is not configured. Please visit https://app.khoj.dev/config#clients to get an API key.`); + return; } + + await request({ url: this.settings.khojUrl ,method: "GET", headers: headers }) + .then(response => { + this.settings.connectedToBackend = true; + }) + .catch(error => { + this.settings.connectedToBackend = false; + new Notice(`❗️Ensure Khoj backend is running and Khoj URL is pointing to it in the plugin settings.\n\n${error}`); + }); } async saveSettings() { From c89bd49973c3c3afe73306339b83112d8b0e811a Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Tue, 21 Nov 2023 01:21:59 -0800 Subject: [PATCH 05/14] Fix ranking search results on Obsidian It's reversed since score of entries is now a distance metric on Khoj server. So lesser distance is better. Previously higher score was better --- src/interface/obsidian/src/search_modal.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/interface/obsidian/src/search_modal.ts b/src/interface/obsidian/src/search_modal.ts index e841360e..51870934 100644 --- a/src/interface/obsidian/src/search_modal.ts +++ b/src/interface/obsidian/src/search_modal.ts @@ -106,7 +106,7 @@ export class KhojSearchModal extends SuggestModal { // Combine markdown and PDF results and sort them by score let results = mdData.concat(pdfData) - .sort((a: any, b: any) => b.score - a.score) + .sort((a: any, b: any) => a.score - b.score) .map((result: any) => { return { entry: result.entry, file: result.file } as SearchResult; }) this.query = query; From a474c31e02e0400a8253e7ca2c77d9de11df0a85 Mon Sep 17 00:00:00 2001 From: sabaimran Date: Tue, 21 Nov 2023 10:56:04 -0800 Subject: [PATCH 06/14] Move the django app into the src/khoj folder for better organization and functionality - Our pypi package currently does not work because the django app and associated database is not included. To remedy this issue, move the app into the src/khoj folder. This has the added benefit of improved organization of the codebase, as all server related code is now in a single folder - Update associated file paths and system references --- Dockerfile | 2 +- prod.Dockerfile | 2 +- pytest.ini | 2 +- src/{ => khoj}/app/README.md | 0 src/{ => khoj}/app/__init__.py | 0 src/{ => khoj}/app/settings.py | 4 ++-- src/{ => khoj}/app/urls.py | 0 src/{ => khoj}/app/wsgi.py | 0 src/khoj/configure.py | 6 +++--- src/{ => khoj}/database/__init__.py | 0 src/{ => khoj}/database/adapters/__init__.py | 2 +- src/{ => khoj}/database/admin.py | 2 +- src/{ => khoj}/database/apps.py | 2 +- src/{ => khoj}/database/migrations/0001_khojuser.py | 0 src/{ => khoj}/database/migrations/0002_googleuser.py | 0 src/{ => khoj}/database/migrations/0003_vector_extension.py | 0 .../database/migrations/0004_content_types_and_more.py | 0 .../database/migrations/0005_embeddings_corpus_id.py | 0 src/{ => khoj}/database/migrations/0006_embeddingsdates.py | 0 src/{ => khoj}/database/migrations/0007_add_conversation.py | 0 .../migrations/0008_alter_conversation_conversation_log.py | 0 src/{ => khoj}/database/migrations/0009_khojapiuser.py | 0 .../database/migrations/0010_chatmodeloptions_and_more.py | 0 .../migrations/0010_rename_embeddings_entry_and_more.py | 0 .../database/migrations/0011_merge_20231102_0138.py | 0 .../database/migrations/0012_entry_file_source.py | 0 src/{ => khoj}/database/migrations/0013_subscription.py | 0 .../database/migrations/0014_alter_googleuser_picture.py | 0 .../database/migrations/0015_alter_subscription_user.py | 0 .../migrations/0016_alter_subscription_renewal_date.py | 0 src/{ => khoj}/database/migrations/0017_searchmodel.py | 0 .../migrations/0018_searchmodelconfig_delete_searchmodel.py | 0 .../0019_alter_googleuser_family_name_and_more.py | 0 src/{ => khoj}/database/migrations/__init__.py | 0 src/{ => khoj}/database/models/__init__.py | 0 src/{ => khoj}/database/tests.py | 0 src/khoj/main.py | 2 +- src/{ => khoj}/manage.py | 2 +- src/khoj/migrations/migrate_server_pg.py | 2 +- src/khoj/processor/github/github_to_entries.py | 2 +- src/khoj/processor/markdown/markdown_to_entries.py | 2 +- src/khoj/processor/notion/notion_to_entries.py | 2 +- src/khoj/processor/org_mode/org_to_entries.py | 2 +- src/khoj/processor/pdf/pdf_to_entries.py | 2 +- src/khoj/processor/plaintext/plaintext_to_entries.py | 2 +- src/khoj/processor/text_to_entries.py | 4 ++-- src/khoj/routers/api.py | 6 +++--- src/khoj/routers/auth.py | 4 ++-- src/khoj/routers/helpers.py | 4 ++-- src/khoj/routers/indexer.py | 2 +- src/khoj/routers/subscription.py | 2 +- src/khoj/routers/web_client.py | 6 +++--- src/khoj/search_type/text_search.py | 4 ++-- src/khoj/utils/fs_syncer.py | 2 +- src/khoj/utils/initialization.py | 4 ++-- tests/conftest.py | 2 +- tests/helpers.py | 2 +- tests/test_client.py | 4 ++-- tests/test_multiple_users.py | 4 ++-- tests/test_openai_chat_director.py | 2 +- tests/test_plaintext_to_entries.py | 2 +- tests/test_text_search.py | 2 +- 62 files changed, 48 insertions(+), 48 deletions(-) rename src/{ => khoj}/app/README.md (100%) rename src/{ => khoj}/app/__init__.py (100%) rename src/{ => khoj}/app/settings.py (97%) rename src/{ => khoj}/app/urls.py (100%) rename src/{ => khoj}/app/wsgi.py (100%) rename src/{ => khoj}/database/__init__.py (100%) rename src/{ => khoj}/database/adapters/__init__.py (99%) rename src/{ => khoj}/database/admin.py (93%) rename src/{ => khoj}/database/apps.py (82%) rename src/{ => khoj}/database/migrations/0001_khojuser.py (100%) rename src/{ => khoj}/database/migrations/0002_googleuser.py (100%) rename src/{ => khoj}/database/migrations/0003_vector_extension.py (100%) rename src/{ => khoj}/database/migrations/0004_content_types_and_more.py (100%) rename src/{ => khoj}/database/migrations/0005_embeddings_corpus_id.py (100%) rename src/{ => khoj}/database/migrations/0006_embeddingsdates.py (100%) rename src/{ => khoj}/database/migrations/0007_add_conversation.py (100%) rename src/{ => khoj}/database/migrations/0008_alter_conversation_conversation_log.py (100%) rename src/{ => khoj}/database/migrations/0009_khojapiuser.py (100%) rename src/{ => khoj}/database/migrations/0010_chatmodeloptions_and_more.py (100%) rename src/{ => khoj}/database/migrations/0010_rename_embeddings_entry_and_more.py (100%) rename src/{ => khoj}/database/migrations/0011_merge_20231102_0138.py (100%) rename src/{ => khoj}/database/migrations/0012_entry_file_source.py (100%) rename src/{ => khoj}/database/migrations/0013_subscription.py (100%) rename src/{ => khoj}/database/migrations/0014_alter_googleuser_picture.py (100%) rename src/{ => khoj}/database/migrations/0015_alter_subscription_user.py (100%) rename src/{ => khoj}/database/migrations/0016_alter_subscription_renewal_date.py (100%) rename src/{ => khoj}/database/migrations/0017_searchmodel.py (100%) rename src/{ => khoj}/database/migrations/0018_searchmodelconfig_delete_searchmodel.py (100%) rename src/{ => khoj}/database/migrations/0019_alter_googleuser_family_name_and_more.py (100%) rename src/{ => khoj}/database/migrations/__init__.py (100%) rename src/{ => khoj}/database/models/__init__.py (100%) rename src/{ => khoj}/database/tests.py (100%) rename src/{ => khoj}/manage.py (89%) diff --git a/Dockerfile b/Dockerfile index 9882a236..4512e884 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ RUN sed -i 's/dynamic = \["version"\]/version = "0.0.0"/' pyproject.toml && \ COPY . . # Set the PYTHONPATH environment variable in order for it to find the Django app. -ENV PYTHONPATH=/app/src:$PYTHONPATH +ENV PYTHONPATH=/app/src/khoj:$PYTHONPATH # Run the Application # There are more arguments required for the application to run, diff --git a/prod.Dockerfile b/prod.Dockerfile index 693a3a8b..a935f3c6 100644 --- a/prod.Dockerfile +++ b/prod.Dockerfile @@ -20,7 +20,7 @@ COPY . . RUN apt install vim -y # Set the PYTHONPATH environment variable in order for it to find the Django app. -ENV PYTHONPATH=/app/src:$PYTHONPATH +ENV PYTHONPATH=/app/src/khoj:$PYTHONPATH # Run the Application # There are more arguments required for the application to run, diff --git a/pytest.ini b/pytest.ini index b3e418d0..36d842ae 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -DJANGO_SETTINGS_MODULE = app.settings +DJANGO_SETTINGS_MODULE = khoj.app.settings pythonpath = . src testpaths = tests markers = diff --git a/src/app/README.md b/src/khoj/app/README.md similarity index 100% rename from src/app/README.md rename to src/khoj/app/README.md diff --git a/src/app/__init__.py b/src/khoj/app/__init__.py similarity index 100% rename from src/app/__init__.py rename to src/khoj/app/__init__.py diff --git a/src/app/settings.py b/src/khoj/app/settings.py similarity index 97% rename from src/app/settings.py rename to src/khoj/app/settings.py index 0803081d..721bcc87 100644 --- a/src/app/settings.py +++ b/src/khoj/app/settings.py @@ -14,7 +14,7 @@ from pathlib import Path import os # Build paths inside the project like this: BASE_DIR / 'subdir'. -BASE_DIR = Path(__file__).resolve().parent.parent.parent +BASE_DIR = Path(__file__).resolve().parent.parent.parent.parent # Quick-start development settings - unsuitable for production @@ -53,7 +53,7 @@ SESSION_COOKIE_SAMESITE = "None" INSTALLED_APPS = [ "django.contrib.auth", "django.contrib.contenttypes", - "database.apps.DatabaseConfig", + "khoj.database.apps.DatabaseConfig", "django.contrib.admin", "django.contrib.sessions", "django.contrib.messages", diff --git a/src/app/urls.py b/src/khoj/app/urls.py similarity index 100% rename from src/app/urls.py rename to src/khoj/app/urls.py diff --git a/src/app/wsgi.py b/src/khoj/app/wsgi.py similarity index 100% rename from src/app/wsgi.py rename to src/khoj/app/wsgi.py diff --git a/src/khoj/configure.py b/src/khoj/configure.py index 5ed92727..d34205d9 100644 --- a/src/khoj/configure.py +++ b/src/khoj/configure.py @@ -20,8 +20,8 @@ from starlette.authentication import ( ) # Internal Packages -from database.models import KhojUser, Subscription -from database.adapters import get_all_users, get_or_create_search_model +from khoj.database.models import KhojUser, Subscription +from khoj.database.adapters import get_all_users, get_or_create_search_model from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel from khoj.routers.indexer import configure_content, load_content, configure_search from khoj.utils import constants, state @@ -45,7 +45,7 @@ class UserAuthenticationBackend(AuthenticationBackend): def __init__( self, ): - from database.models import KhojUser, KhojApiUser + from khoj.database.models import KhojUser, KhojApiUser self.khojuser_manager = KhojUser.objects self.khojapiuser_manager = KhojApiUser.objects diff --git a/src/database/__init__.py b/src/khoj/database/__init__.py similarity index 100% rename from src/database/__init__.py rename to src/khoj/database/__init__.py diff --git a/src/database/adapters/__init__.py b/src/khoj/database/adapters/__init__.py similarity index 99% rename from src/database/adapters/__init__.py rename to src/khoj/database/adapters/__init__.py index 4141d3bb..ea0c0a85 100644 --- a/src/database/adapters/__init__.py +++ b/src/khoj/database/adapters/__init__.py @@ -17,7 +17,7 @@ from asgiref.sync import sync_to_async from fastapi import HTTPException -from database.models import ( +from khoj.database.models import ( KhojUser, GoogleUser, KhojApiUser, diff --git a/src/database/admin.py b/src/khoj/database/admin.py similarity index 93% rename from src/database/admin.py rename to src/khoj/database/admin.py index 8d2130ba..69f15b2f 100644 --- a/src/database/admin.py +++ b/src/khoj/database/admin.py @@ -3,7 +3,7 @@ from django.contrib.auth.admin import UserAdmin # Register your models here. -from database.models import ( +from khoj.database.models import ( KhojUser, ChatModelOptions, OpenAIProcessorConversationConfig, diff --git a/src/database/apps.py b/src/khoj/database/apps.py similarity index 82% rename from src/database/apps.py rename to src/khoj/database/apps.py index a3b71b13..eed64812 100644 --- a/src/database/apps.py +++ b/src/khoj/database/apps.py @@ -3,4 +3,4 @@ from django.apps import AppConfig class DatabaseConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" - name = "database" + name = "khoj.database" diff --git a/src/database/migrations/0001_khojuser.py b/src/khoj/database/migrations/0001_khojuser.py similarity index 100% rename from src/database/migrations/0001_khojuser.py rename to src/khoj/database/migrations/0001_khojuser.py diff --git a/src/database/migrations/0002_googleuser.py b/src/khoj/database/migrations/0002_googleuser.py similarity index 100% rename from src/database/migrations/0002_googleuser.py rename to src/khoj/database/migrations/0002_googleuser.py diff --git a/src/database/migrations/0003_vector_extension.py b/src/khoj/database/migrations/0003_vector_extension.py similarity index 100% rename from src/database/migrations/0003_vector_extension.py rename to src/khoj/database/migrations/0003_vector_extension.py diff --git a/src/database/migrations/0004_content_types_and_more.py b/src/khoj/database/migrations/0004_content_types_and_more.py similarity index 100% rename from src/database/migrations/0004_content_types_and_more.py rename to src/khoj/database/migrations/0004_content_types_and_more.py diff --git a/src/database/migrations/0005_embeddings_corpus_id.py b/src/khoj/database/migrations/0005_embeddings_corpus_id.py similarity index 100% rename from src/database/migrations/0005_embeddings_corpus_id.py rename to src/khoj/database/migrations/0005_embeddings_corpus_id.py diff --git a/src/database/migrations/0006_embeddingsdates.py b/src/khoj/database/migrations/0006_embeddingsdates.py similarity index 100% rename from src/database/migrations/0006_embeddingsdates.py rename to src/khoj/database/migrations/0006_embeddingsdates.py diff --git a/src/database/migrations/0007_add_conversation.py b/src/khoj/database/migrations/0007_add_conversation.py similarity index 100% rename from src/database/migrations/0007_add_conversation.py rename to src/khoj/database/migrations/0007_add_conversation.py diff --git a/src/database/migrations/0008_alter_conversation_conversation_log.py b/src/khoj/database/migrations/0008_alter_conversation_conversation_log.py similarity index 100% rename from src/database/migrations/0008_alter_conversation_conversation_log.py rename to src/khoj/database/migrations/0008_alter_conversation_conversation_log.py diff --git a/src/database/migrations/0009_khojapiuser.py b/src/khoj/database/migrations/0009_khojapiuser.py similarity index 100% rename from src/database/migrations/0009_khojapiuser.py rename to src/khoj/database/migrations/0009_khojapiuser.py diff --git a/src/database/migrations/0010_chatmodeloptions_and_more.py b/src/khoj/database/migrations/0010_chatmodeloptions_and_more.py similarity index 100% rename from src/database/migrations/0010_chatmodeloptions_and_more.py rename to src/khoj/database/migrations/0010_chatmodeloptions_and_more.py diff --git a/src/database/migrations/0010_rename_embeddings_entry_and_more.py b/src/khoj/database/migrations/0010_rename_embeddings_entry_and_more.py similarity index 100% rename from src/database/migrations/0010_rename_embeddings_entry_and_more.py rename to src/khoj/database/migrations/0010_rename_embeddings_entry_and_more.py diff --git a/src/database/migrations/0011_merge_20231102_0138.py b/src/khoj/database/migrations/0011_merge_20231102_0138.py similarity index 100% rename from src/database/migrations/0011_merge_20231102_0138.py rename to src/khoj/database/migrations/0011_merge_20231102_0138.py diff --git a/src/database/migrations/0012_entry_file_source.py b/src/khoj/database/migrations/0012_entry_file_source.py similarity index 100% rename from src/database/migrations/0012_entry_file_source.py rename to src/khoj/database/migrations/0012_entry_file_source.py diff --git a/src/database/migrations/0013_subscription.py b/src/khoj/database/migrations/0013_subscription.py similarity index 100% rename from src/database/migrations/0013_subscription.py rename to src/khoj/database/migrations/0013_subscription.py diff --git a/src/database/migrations/0014_alter_googleuser_picture.py b/src/khoj/database/migrations/0014_alter_googleuser_picture.py similarity index 100% rename from src/database/migrations/0014_alter_googleuser_picture.py rename to src/khoj/database/migrations/0014_alter_googleuser_picture.py diff --git a/src/database/migrations/0015_alter_subscription_user.py b/src/khoj/database/migrations/0015_alter_subscription_user.py similarity index 100% rename from src/database/migrations/0015_alter_subscription_user.py rename to src/khoj/database/migrations/0015_alter_subscription_user.py diff --git a/src/database/migrations/0016_alter_subscription_renewal_date.py b/src/khoj/database/migrations/0016_alter_subscription_renewal_date.py similarity index 100% rename from src/database/migrations/0016_alter_subscription_renewal_date.py rename to src/khoj/database/migrations/0016_alter_subscription_renewal_date.py diff --git a/src/database/migrations/0017_searchmodel.py b/src/khoj/database/migrations/0017_searchmodel.py similarity index 100% rename from src/database/migrations/0017_searchmodel.py rename to src/khoj/database/migrations/0017_searchmodel.py diff --git a/src/database/migrations/0018_searchmodelconfig_delete_searchmodel.py b/src/khoj/database/migrations/0018_searchmodelconfig_delete_searchmodel.py similarity index 100% rename from src/database/migrations/0018_searchmodelconfig_delete_searchmodel.py rename to src/khoj/database/migrations/0018_searchmodelconfig_delete_searchmodel.py diff --git a/src/database/migrations/0019_alter_googleuser_family_name_and_more.py b/src/khoj/database/migrations/0019_alter_googleuser_family_name_and_more.py similarity index 100% rename from src/database/migrations/0019_alter_googleuser_family_name_and_more.py rename to src/khoj/database/migrations/0019_alter_googleuser_family_name_and_more.py diff --git a/src/database/migrations/__init__.py b/src/khoj/database/migrations/__init__.py similarity index 100% rename from src/database/migrations/__init__.py rename to src/khoj/database/migrations/__init__.py diff --git a/src/database/models/__init__.py b/src/khoj/database/models/__init__.py similarity index 100% rename from src/database/models/__init__.py rename to src/khoj/database/models/__init__.py diff --git a/src/database/tests.py b/src/khoj/database/tests.py similarity index 100% rename from src/database/tests.py rename to src/khoj/database/tests.py diff --git a/src/khoj/main.py b/src/khoj/main.py index 022efcc1..cded9841 100644 --- a/src/khoj/main.py +++ b/src/khoj/main.py @@ -26,7 +26,7 @@ from django.core.management import call_command # Initialize Django sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "app.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "khoj.app.settings") django.setup() # Initialize Django Database diff --git a/src/manage.py b/src/khoj/manage.py similarity index 89% rename from src/manage.py rename to src/khoj/manage.py index 1a64b14a..9b8f4b27 100755 --- a/src/manage.py +++ b/src/khoj/manage.py @@ -6,7 +6,7 @@ import sys def main(): """Run administrative tasks.""" - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "app.settings") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "khoj.app.settings") try: from django.core.management import execute_from_command_line except ImportError as exc: diff --git a/src/khoj/migrations/migrate_server_pg.py b/src/khoj/migrations/migrate_server_pg.py index 434e27d7..097ae0d6 100644 --- a/src/khoj/migrations/migrate_server_pg.py +++ b/src/khoj/migrations/migrate_server_pg.py @@ -60,7 +60,7 @@ import logging from packaging import version from khoj.utils.yaml import load_config_from_file, save_config_to_file -from database.models import ( +from khoj.database.models import ( OpenAIProcessorConversationConfig, OfflineChatProcessorConversationConfig, ChatModelOptions, diff --git a/src/khoj/processor/github/github_to_entries.py b/src/khoj/processor/github/github_to_entries.py index 56279453..21344eca 100644 --- a/src/khoj/processor/github/github_to_entries.py +++ b/src/khoj/processor/github/github_to_entries.py @@ -13,7 +13,7 @@ from khoj.utils.rawconfig import Entry, GithubContentConfig, GithubRepoConfig from khoj.processor.markdown.markdown_to_entries import MarkdownToEntries from khoj.processor.org_mode.org_to_entries import OrgToEntries from khoj.processor.text_to_entries import TextToEntries -from database.models import Entry as DbEntry, GithubConfig, KhojUser +from khoj.database.models import Entry as DbEntry, GithubConfig, KhojUser logger = logging.getLogger(__name__) diff --git a/src/khoj/processor/markdown/markdown_to_entries.py b/src/khoj/processor/markdown/markdown_to_entries.py index 0dd71740..08ba4a77 100644 --- a/src/khoj/processor/markdown/markdown_to_entries.py +++ b/src/khoj/processor/markdown/markdown_to_entries.py @@ -10,7 +10,7 @@ from khoj.processor.text_to_entries import TextToEntries from khoj.utils.helpers import timer from khoj.utils.constants import empty_escape_sequences from khoj.utils.rawconfig import Entry -from database.models import Entry as DbEntry, KhojUser +from khoj.database.models import Entry as DbEntry, KhojUser logger = logging.getLogger(__name__) diff --git a/src/khoj/processor/notion/notion_to_entries.py b/src/khoj/processor/notion/notion_to_entries.py index 7a88e2a1..1e3c0553 100644 --- a/src/khoj/processor/notion/notion_to_entries.py +++ b/src/khoj/processor/notion/notion_to_entries.py @@ -10,7 +10,7 @@ from khoj.utils.helpers import timer from khoj.utils.rawconfig import Entry, NotionContentConfig from khoj.processor.text_to_entries import TextToEntries from khoj.utils.rawconfig import Entry -from database.models import Entry as DbEntry, KhojUser, NotionConfig +from khoj.database.models import Entry as DbEntry, KhojUser, NotionConfig from enum import Enum diff --git a/src/khoj/processor/org_mode/org_to_entries.py b/src/khoj/processor/org_mode/org_to_entries.py index e42b7498..80ccdb08 100644 --- a/src/khoj/processor/org_mode/org_to_entries.py +++ b/src/khoj/processor/org_mode/org_to_entries.py @@ -9,7 +9,7 @@ from khoj.processor.text_to_entries import TextToEntries from khoj.utils.helpers import timer from khoj.utils.rawconfig import Entry from khoj.utils import state -from database.models import Entry as DbEntry, KhojUser +from khoj.database.models import Entry as DbEntry, KhojUser logger = logging.getLogger(__name__) diff --git a/src/khoj/processor/pdf/pdf_to_entries.py b/src/khoj/processor/pdf/pdf_to_entries.py index 3a47096a..62f94d32 100644 --- a/src/khoj/processor/pdf/pdf_to_entries.py +++ b/src/khoj/processor/pdf/pdf_to_entries.py @@ -11,7 +11,7 @@ from langchain.document_loaders import PyMuPDFLoader from khoj.processor.text_to_entries import TextToEntries from khoj.utils.helpers import timer from khoj.utils.rawconfig import Entry -from database.models import Entry as DbEntry, KhojUser +from khoj.database.models import Entry as DbEntry, KhojUser logger = logging.getLogger(__name__) diff --git a/src/khoj/processor/plaintext/plaintext_to_entries.py b/src/khoj/processor/plaintext/plaintext_to_entries.py index d42dae30..081cc327 100644 --- a/src/khoj/processor/plaintext/plaintext_to_entries.py +++ b/src/khoj/processor/plaintext/plaintext_to_entries.py @@ -9,7 +9,7 @@ from bs4 import BeautifulSoup from khoj.processor.text_to_entries import TextToEntries from khoj.utils.helpers import timer from khoj.utils.rawconfig import Entry -from database.models import Entry as DbEntry, KhojUser +from khoj.database.models import Entry as DbEntry, KhojUser logger = logging.getLogger(__name__) diff --git a/src/khoj/processor/text_to_entries.py b/src/khoj/processor/text_to_entries.py index ac42105a..109c58e6 100644 --- a/src/khoj/processor/text_to_entries.py +++ b/src/khoj/processor/text_to_entries.py @@ -13,8 +13,8 @@ from khoj.utils.helpers import is_none_or_empty, timer, batcher # Internal Packages from khoj.utils.rawconfig import Entry from khoj.search_filter.date_filter import DateFilter -from database.models import KhojUser, Entry as DbEntry, EntryDates -from database.adapters import EntryAdapters +from khoj.database.models import KhojUser, Entry as DbEntry, EntryDates +from khoj.database.adapters import EntryAdapters logger = logging.getLogger(__name__) diff --git a/src/khoj/routers/api.py b/src/khoj/routers/api.py index 6d67fcbe..f2e5c966 100644 --- a/src/khoj/routers/api.py +++ b/src/khoj/routers/api.py @@ -44,9 +44,9 @@ from khoj.processor.conversation.openai.gpt import extract_questions from khoj.processor.conversation.gpt4all.chat_model import extract_questions_offline from fastapi.requests import Request -from database import adapters -from database.adapters import EntryAdapters, ConversationAdapters -from database.models import ( +from khoj.database import adapters +from khoj.database.adapters import EntryAdapters, ConversationAdapters +from khoj.database.models import ( LocalMarkdownConfig, LocalOrgConfig, LocalPdfConfig, diff --git a/src/khoj/routers/auth.py b/src/khoj/routers/auth.py index a9a88325..df119548 100644 --- a/src/khoj/routers/auth.py +++ b/src/khoj/routers/auth.py @@ -15,8 +15,8 @@ from google.oauth2 import id_token from google.auth.transport import requests as google_requests # Internal Packages -from database.adapters import get_khoj_tokens, get_or_create_user, create_khoj_token, delete_khoj_token -from database.models import KhojApiUser +from khoj.database.adapters import get_khoj_tokens, get_or_create_user, create_khoj_token, delete_khoj_token +from khoj.database.models import KhojApiUser from khoj.routers.helpers import update_telemetry_state from khoj.utils import state diff --git a/src/khoj/routers/helpers.py b/src/khoj/routers/helpers.py index 272f962d..63991263 100644 --- a/src/khoj/routers/helpers.py +++ b/src/khoj/routers/helpers.py @@ -18,8 +18,8 @@ from khoj.utils.helpers import ConversationCommand, log_telemetry from khoj.processor.conversation.openai.gpt import converse from khoj.processor.conversation.gpt4all.chat_model import converse_offline from khoj.processor.conversation.utils import message_to_log, ThreadedGenerator -from database.models import KhojUser, Subscription -from database.adapters import ConversationAdapters +from khoj.database.models import KhojUser, Subscription +from khoj.database.adapters import ConversationAdapters logger = logging.getLogger(__name__) diff --git a/src/khoj/routers/indexer.py b/src/khoj/routers/indexer.py index ccb65063..7a7be03e 100644 --- a/src/khoj/routers/indexer.py +++ b/src/khoj/routers/indexer.py @@ -30,7 +30,7 @@ from khoj.utils.config import ( ContentIndex, SearchModels, ) -from database.models import ( +from khoj.database.models import ( KhojUser, GithubConfig, NotionConfig, diff --git a/src/khoj/routers/subscription.py b/src/khoj/routers/subscription.py index 62e50d72..edcbc135 100644 --- a/src/khoj/routers/subscription.py +++ b/src/khoj/routers/subscription.py @@ -10,7 +10,7 @@ from starlette.authentication import requires import stripe # Internal Packages -from database import adapters +from khoj.database import adapters # Stripe integration for Khoj Cloud Subscription diff --git a/src/khoj/routers/web_client.py b/src/khoj/routers/web_client.py index f30499d8..dab16fa8 100644 --- a/src/khoj/routers/web_client.py +++ b/src/khoj/routers/web_client.py @@ -8,8 +8,8 @@ from fastapi import Request from fastapi.responses import HTMLResponse, FileResponse, RedirectResponse from fastapi.templating import Jinja2Templates from starlette.authentication import requires -from database import adapters -from database.models import KhojUser +from khoj.database import adapters +from khoj.database.models import KhojUser from khoj.utils.rawconfig import ( GithubContentConfig, GithubRepoConfig, @@ -18,7 +18,7 @@ from khoj.utils.rawconfig import ( # Internal Packages from khoj.utils import constants, state -from database.adapters import ( +from khoj.database.adapters import ( EntryAdapters, get_user_github_config, get_user_notion_config, diff --git a/src/khoj/search_type/text_search.py b/src/khoj/search_type/text_search.py index 7e295903..a78ce522 100644 --- a/src/khoj/search_type/text_search.py +++ b/src/khoj/search_type/text_search.py @@ -19,8 +19,8 @@ from khoj.utils.state import SearchType from khoj.utils.rawconfig import SearchResponse, Entry from khoj.utils.jsonl import load_jsonl from khoj.processor.text_to_entries import TextToEntries -from database.adapters import EntryAdapters -from database.models import KhojUser, Entry as DbEntry +from khoj.database.adapters import EntryAdapters +from khoj.database.models import KhojUser, Entry as DbEntry logger = logging.getLogger(__name__) diff --git a/src/khoj/utils/fs_syncer.py b/src/khoj/utils/fs_syncer.py index fc7e4a2d..57c79d9d 100644 --- a/src/khoj/utils/fs_syncer.py +++ b/src/khoj/utils/fs_syncer.py @@ -7,7 +7,7 @@ from bs4 import BeautifulSoup from khoj.utils.helpers import get_absolute_path, is_none_or_empty from khoj.utils.rawconfig import TextContentConfig from khoj.utils.config import SearchType -from database.models import LocalMarkdownConfig, LocalOrgConfig, LocalPdfConfig, LocalPlaintextConfig +from khoj.database.models import LocalMarkdownConfig, LocalOrgConfig, LocalPdfConfig, LocalPlaintextConfig logger = logging.getLogger(__name__) diff --git a/src/khoj/utils/initialization.py b/src/khoj/utils/initialization.py index b2d7aad5..ffc4d47e 100644 --- a/src/khoj/utils/initialization.py +++ b/src/khoj/utils/initialization.py @@ -1,7 +1,7 @@ import logging import os -from database.models import ( +from khoj.database.models import ( KhojUser, OfflineChatProcessorConversationConfig, OpenAIProcessorConversationConfig, @@ -11,7 +11,7 @@ from database.models import ( from khoj.utils.constants import default_offline_chat_model, default_online_chat_model from khoj.processor.conversation.utils import model_to_prompt_size, model_to_tokenizer -from database.adapters import ConversationAdapters +from khoj.database.adapters import ConversationAdapters logger = logging.getLogger(__name__) diff --git a/tests/conftest.py b/tests/conftest.py index 902994a0..d1df341a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,7 +26,7 @@ from khoj.utils.rawconfig import ( from khoj.utils import state, fs_syncer from khoj.routers.indexer import configure_content from khoj.processor.org_mode.org_to_entries import OrgToEntries -from database.models import ( +from khoj.database.models import ( KhojApiUser, LocalOrgConfig, LocalMarkdownConfig, diff --git a/tests/helpers.py b/tests/helpers.py index 079eb475..d0f9babc 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,7 +1,7 @@ import factory import os -from database.models import ( +from khoj.database.models import ( KhojUser, KhojApiUser, ChatModelOptions, diff --git a/tests/test_client.py b/tests/test_client.py index 5324e3c1..4e707e5f 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -16,8 +16,8 @@ from khoj.utils.state import search_models, content_index, config from khoj.search_type import text_search, image_search from khoj.utils.rawconfig import ContentConfig, SearchConfig from khoj.processor.org_mode.org_to_entries import OrgToEntries -from database.models import KhojUser, KhojApiUser -from database.adapters import EntryAdapters +from khoj.database.models import KhojUser, KhojApiUser +from khoj.database.adapters import EntryAdapters # Test diff --git a/tests/test_multiple_users.py b/tests/test_multiple_users.py index a94c173e..b5785350 100644 --- a/tests/test_multiple_users.py +++ b/tests/test_multiple_users.py @@ -17,8 +17,8 @@ from khoj.utils.state import search_models, content_index, config from khoj.search_type import text_search, image_search from khoj.utils.rawconfig import ContentConfig, SearchConfig from khoj.processor.org_mode.org_to_entries import OrgToEntries -from database.models import KhojUser, KhojApiUser -from database.adapters import EntryAdapters +from khoj.database.models import KhojUser, KhojApiUser +from khoj.database.adapters import EntryAdapters # ---------------------------------------------------------------------------------------------------- diff --git a/tests/test_openai_chat_director.py b/tests/test_openai_chat_director.py index 07c4e0d8..b4e63364 100644 --- a/tests/test_openai_chat_director.py +++ b/tests/test_openai_chat_director.py @@ -10,7 +10,7 @@ from khoj.processor.conversation import prompts # Internal Packages from khoj.processor.conversation.utils import message_to_log from tests.helpers import ConversationFactory -from database.models import KhojUser +from khoj.database.models import KhojUser # Initialize variables for tests api_key = os.getenv("OPENAI_API_KEY") diff --git a/tests/test_plaintext_to_entries.py b/tests/test_plaintext_to_entries.py index 23b0d652..0bbba6de 100644 --- a/tests/test_plaintext_to_entries.py +++ b/tests/test_plaintext_to_entries.py @@ -7,7 +7,7 @@ from pathlib import Path from khoj.utils.fs_syncer import get_plaintext_files from khoj.utils.rawconfig import TextContentConfig from khoj.processor.plaintext.plaintext_to_entries import PlaintextToEntries -from database.models import LocalPlaintextConfig, KhojUser +from khoj.database.models import LocalPlaintextConfig, KhojUser def test_plaintext_file(tmp_path): diff --git a/tests/test_text_search.py b/tests/test_text_search.py index b4507feb..025c1241 100644 --- a/tests/test_text_search.py +++ b/tests/test_text_search.py @@ -13,7 +13,7 @@ from khoj.utils.rawconfig import ContentConfig, SearchConfig from khoj.processor.org_mode.org_to_entries import OrgToEntries from khoj.processor.github.github_to_entries import GithubToEntries from khoj.utils.fs_syncer import collect_files, get_org_files -from database.models import LocalOrgConfig, KhojUser, Entry, GithubConfig +from khoj.database.models import LocalOrgConfig, KhojUser, Entry, GithubConfig logger = logging.getLogger(__name__) From 71e794c26f1bef2d527474e70c76dfc58295911c Mon Sep 17 00:00:00 2001 From: sabaimran Date: Tue, 21 Nov 2023 10:57:21 -0800 Subject: [PATCH 07/14] Remove the sys.append line in the main.py file, as it's not required --- src/khoj/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/khoj/main.py b/src/khoj/main.py index cded9841..23d36b3d 100644 --- a/src/khoj/main.py +++ b/src/khoj/main.py @@ -25,7 +25,6 @@ from django.core.asgi import get_asgi_application from django.core.management import call_command # Initialize Django -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "khoj.app.settings") django.setup() From 8932fc0c367abea6440616aaf0dee1dbfe8ff906 Mon Sep 17 00:00:00 2001 From: sabaimran Date: Tue, 21 Nov 2023 11:12:50 -0800 Subject: [PATCH 08/14] Ignore w004 check to bypass pypi warnings for check-wheel-contents - PyPi doesn't like to have files that start with numbers, however all of the generated django migration files start with numbers. To accommodate, skip this check. - Refer to https://pypi.org/project/check-wheel-contents/ for documentation and recommendation --- .github/workflows/pypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 21792f51..0f31102d 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -48,7 +48,7 @@ jobs: - name: 🌡️ Validate Python Package run: | # Validate PyPi Package - pipx run check-wheel-contents dist/*.whl + pipx run check-wheel-contents dist/*.whl --ignore w004 pipx run twine check dist/* - name: ⏫ Upload Python Package Artifacts From a1460a5bf9da6a5955a34cd31fbcdf6a865167c2 Mon Sep 17 00:00:00 2001 From: sabaimran Date: Tue, 21 Nov 2023 11:14:40 -0800 Subject: [PATCH 09/14] Set operations to typed empty list in migration file --- src/khoj/database/migrations/0011_merge_20231102_0138.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/khoj/database/migrations/0011_merge_20231102_0138.py b/src/khoj/database/migrations/0011_merge_20231102_0138.py index 112c76a2..89a581f0 100644 --- a/src/khoj/database/migrations/0011_merge_20231102_0138.py +++ b/src/khoj/database/migrations/0011_merge_20231102_0138.py @@ -2,6 +2,8 @@ from django.db import migrations +from typing import List, Any + class Migration(migrations.Migration): dependencies = [ @@ -9,4 +11,4 @@ class Migration(migrations.Migration): ("database", "0010_rename_embeddings_entry_and_more"), ] - operations = [] + operations = List[Any] = [] From 645fd966343e6f90e4858846b3ed33dc4fead7ab Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Tue, 21 Nov 2023 11:19:33 -0800 Subject: [PATCH 10/14] Search across all content types from Khoj Obsidian client Previously it was only searching for PDF and Markdown files. This was meant to show only content from current vault as results. But it has not scaled well as other clients also allow syncing PDF and markdown files now. So remove this content type filter for now. A proper solution would limit by using file/dir filters on server or client side. --- src/interface/obsidian/src/search_modal.ts | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/src/interface/obsidian/src/search_modal.ts b/src/interface/obsidian/src/search_modal.ts index 51870934..7e97d1ea 100644 --- a/src/interface/obsidian/src/search_modal.ts +++ b/src/interface/obsidian/src/search_modal.ts @@ -87,27 +87,18 @@ export class KhojSearchModal extends SuggestModal { } async getSuggestions(query: string): Promise { - // Query Khoj backend for search results + // Setup Query Khoj backend for search results let encodedQuery = encodeURIComponent(query); let searchUrl = `${this.setting.khojUrl}/api/search?q=${encodedQuery}&n=${this.setting.resultsCount}&r=${this.rerank}&client=obsidian`; let headers = { 'Authorization': `Bearer ${this.setting.khojApiKey}` } - // Get search results for markdown and pdf files - let mdResponse = await request({ url: `${searchUrl}&t=markdown`, headers: headers }); - let pdfResponse = await request({ url: `${searchUrl}&t=pdf`, headers: headers }); + // Get search results from Khoj backend + let response = await request({ url: `${searchUrl}`, headers: headers }); // Parse search results - let mdData = JSON.parse(mdResponse) + let results = JSON.parse(response) .filter((result: any) => !this.find_similar_notes || !result.additional.file.endsWith(this.app.workspace.getActiveFile()?.path)) - .map((result: any) => { return { entry: result.entry, score: result.score, file: result.additional.file }; }); - let pdfData = JSON.parse(pdfResponse) - .filter((result: any) => !this.find_similar_notes || !result.additional.file.endsWith(this.app.workspace.getActiveFile()?.path)) - .map((result: any) => { return { entry: `## ${result.additional.compiled}`, score: result.score, file: result.additional.file } as SearchResult; }) - - // Combine markdown and PDF results and sort them by score - let results = mdData.concat(pdfData) - .sort((a: any, b: any) => a.score - b.score) - .map((result: any) => { return { entry: result.entry, file: result.file } as SearchResult; }) + .map((result: any) => { return { entry: result.entry, file: result.additional.file } as SearchResult; }); this.query = query; return results; From 333cb3445c1cf051ad2cdf9087c6bb65c8a7c92c Mon Sep 17 00:00:00 2001 From: sabaimran Date: Tue, 21 Nov 2023 11:28:51 -0800 Subject: [PATCH 11/14] Use colon rather than equals to indicate typing --- src/khoj/database/migrations/0011_merge_20231102_0138.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/khoj/database/migrations/0011_merge_20231102_0138.py b/src/khoj/database/migrations/0011_merge_20231102_0138.py index 89a581f0..0206036e 100644 --- a/src/khoj/database/migrations/0011_merge_20231102_0138.py +++ b/src/khoj/database/migrations/0011_merge_20231102_0138.py @@ -11,4 +11,4 @@ class Migration(migrations.Migration): ("database", "0010_rename_embeddings_entry_and_more"), ] - operations = List[Any] = [] + operations: List[Any] = [] From 38144a7a69134821b8888d98aceb92f0b19fc32e Mon Sep 17 00:00:00 2001 From: sabaimran Date: Tue, 21 Nov 2023 11:33:07 -0800 Subject: [PATCH 12/14] pull_request path should be src/khoj rather than src/ --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 697579da..51301491 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,7 +3,7 @@ name: test on: pull_request: paths: - - src/** + - src/khoj/** - tests/** - config/** - pyproject.toml From 61f6b8c0d4334f74cb5d6b1db926044e2c7154bc Mon Sep 17 00:00:00 2001 From: sabaimran Date: Tue, 21 Nov 2023 11:33:43 -0800 Subject: [PATCH 13/14] Ignore-check step failed due to unrecognized code. Try using capital letters for indicator --- .github/workflows/pypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 0f31102d..1ac735aa 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -48,7 +48,7 @@ jobs: - name: 🌡️ Validate Python Package run: | # Validate PyPi Package - pipx run check-wheel-contents dist/*.whl --ignore w004 + pipx run check-wheel-contents dist/*.whl --ignore W004 pipx run twine check dist/* - name: ⏫ Upload Python Package Artifacts From 244b76ffedddb4dfbf172910727de4264a9d4e98 Mon Sep 17 00:00:00 2001 From: sabaimran Date: Tue, 21 Nov 2023 11:27:12 -0800 Subject: [PATCH 14/14] =?UTF-8?q?Add=20isort=20for=20automatic=20import=20?= =?UTF-8?q?sorting=20and=20skip=20main.py=20because=20it's=20a=20drama=20q?= =?UTF-8?q?ueen=20=F0=9F=91=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .pre-commit-config.yaml | 6 ++++++ src/khoj/main.py | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 77d4ec6c..b7253585 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,6 +4,12 @@ repos: hooks: - id: black +- repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort + name: isort (python) + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: diff --git a/src/khoj/main.py b/src/khoj/main.py index 23d36b3d..c5e1f277 100644 --- a/src/khoj/main.py +++ b/src/khoj/main.py @@ -1,3 +1,7 @@ +""" Main module for Khoj Assistant + isort:skip_file +""" + # Standard Packages import os import sys