Add isort to the pre-commit configuration and apply it to the whole project (#595)

* Apply isort to the entire repository
* Fix missing import issues in text_to_entries
* Fix imports in migration files

tests/conftest.py

@@ -1,4 +1,3 @@
-# External Packages
 import os
 from pathlib import Path
 
@@ -7,8 +6,11 @@ from fastapi import FastAPI
 from fastapi.staticfiles import StaticFiles
 from fastapi.testclient import TestClient
 
-# Internal Packages
-from khoj.configure import configure_middleware, configure_routes, configure_search_types
+from khoj.configure import (
+    configure_middleware,
+    configure_routes,
+    configure_search_types,
+)
 from khoj.database.models import (
     GithubConfig,
     GithubRepoConfig,
@@ -27,7 +29,12 @@ from khoj.utils import fs_syncer, state
 from khoj.utils.config import SearchModels
 from khoj.utils.constants import web_directory
 from khoj.utils.helpers import resolve_absolute_path
-from khoj.utils.rawconfig import ContentConfig, ImageContentConfig, ImageSearchConfig, SearchConfig
+from khoj.utils.rawconfig import (
+    ContentConfig,
+    ImageContentConfig,
+    ImageSearchConfig,
+    SearchConfig,
+)
 from tests.helpers import (
     ChatModelOptionsFactory,
     OfflineChatProcessorConversationConfigFactory,

tests/helpers.py

@@ -1,16 +1,17 @@
-import factory
 import os
 
+import factory
+
 from khoj.database.models import (
-    KhojUser,
-    KhojApiUser,
     ChatModelOptions,
+    Conversation,
+    KhojApiUser,
+    KhojUser,
     OfflineChatProcessorConversationConfig,
     OpenAIProcessorConversationConfig,
     SearchModelConfig,
-    UserConversationConfig,
-    Conversation,
     Subscription,
+    UserConversationConfig,
 )
 
 

tests/test_cli.py

@@ -2,7 +2,6 @@
 from pathlib import Path
 from random import random
 
-# Internal Packages
 from khoj.utils.cli import cli
 from khoj.utils.helpers import resolve_absolute_path
 

tests/test_client.py

@@ -4,12 +4,9 @@ from urllib.parse import quote
 
 import pytest
 from fastapi import FastAPI
-
-# External Packages
 from fastapi.testclient import TestClient
 from PIL import Image
 
-# Internal Packages
 from khoj.configure import configure_routes, configure_search_types
 from khoj.database.adapters import EntryAdapters
 from khoj.database.models import KhojApiUser, KhojUser

tests/test_conversation_utils.py

@@ -1,7 +1,8 @@
-from khoj.processor.conversation import utils
-from langchain.schema import ChatMessage
 import factory
 import tiktoken
+from langchain.schema import ChatMessage
+
+from khoj.processor.conversation import utils
 
 
 class ChatMessageFactory(factory.Factory):

tests/test_date_filter.py

@@ -1,11 +1,8 @@
-# Standard Packages
 import re
 from datetime import datetime
 
-# External Packages
 import pytest
 
-# Internal Packages
 from khoj.search_filter.date_filter import DateFilter
 
 

tests/test_gpt4all_chat_actors.py

@@ -1,7 +1,5 @@
-# Standard Packages
 from datetime import datetime
 
-# External Packages
 import pytest
 
 SKIP_TESTS = True
@@ -18,10 +16,13 @@ try:
 except ModuleNotFoundError as e:
     print("There was an error importing GPT4All. Please run pip install gpt4all in order to install it.")
 
-# Internal Packages
-from khoj.processor.conversation.offline.chat_model import converse_offline, extract_questions_offline, filter_questions
-from khoj.processor.conversation.offline.utils import download_model
 
+from khoj.processor.conversation.offline.chat_model import (
+    converse_offline,
+    extract_questions_offline,
+    filter_questions,
+)
+from khoj.processor.conversation.offline.utils import download_model
 from khoj.processor.conversation.utils import message_to_log
 
 MODEL_NAME = "mistral-7b-instruct-v0.1.Q4_0.gguf"

tests/test_gpt4all_chat_director.py

@@ -1,13 +1,10 @@
-# Standard Packages
 import urllib.parse
 from urllib.parse import quote
 
-# External Packages
 import pytest
-from freezegun import freeze_time
 from faker import Faker
+from freezegun import freeze_time
 
-# Internal Packages
 from khoj.processor.conversation import prompts
 from khoj.processor.conversation.utils import message_to_log
 from tests.helpers import ConversationFactory

tests/test_helpers.py

@@ -1,13 +1,10 @@
-# Standard Packages
-import numpy as np
-import psutil
-from scipy.stats import linregress
 import secrets
 
-# External Packages
+import numpy as np
+import psutil
 import pytest
+from scipy.stats import linregress
 
-# Internal Packages
 from khoj.processor.embeddings import EmbeddingsModel
 from khoj.utils import helpers
 

tests/test_image_search.py

@@ -1,18 +1,16 @@
 # Standard Modules
 import logging
 from pathlib import Path
+
+import pytest
 from PIL import Image
 
-# External Packages
-import pytest
-from khoj.utils.config import SearchModels
-
-# Internal Packages
-from khoj.utils.state import content_index, search_models
-from khoj.utils.constants import web_directory
 from khoj.search_type import image_search
+from khoj.utils.config import SearchModels
+from khoj.utils.constants import web_directory
 from khoj.utils.helpers import resolve_absolute_path
 from khoj.utils.rawconfig import ContentConfig, SearchConfig
+from khoj.utils.state import content_index, search_models
 
 
 # Test

tests/test_markdown_to_entries.py

@@ -1,9 +1,7 @@
-# Standard Packages
 import json
-from pathlib import Path
 import os
+from pathlib import Path
 
-# Internal Packages
 from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntries
 from khoj.utils.fs_syncer import get_markdown_files
 from khoj.utils.rawconfig import TextContentConfig

tests/test_multiple_users.py

@@ -1,13 +1,10 @@
 # Standard Modules
 from urllib.parse import quote
 
-# External Packages
 import pytest
 
 from khoj.database.models import KhojApiUser, KhojUser
 from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
-
-# Internal Packages
 from khoj.search_type import text_search
 
 

tests/test_openai_chat_actors.py

@@ -1,17 +1,13 @@
-# Standard Packages
 import os
 from datetime import datetime
 
-# External Packages
-import pytest
 import freezegun
+import pytest
 from freezegun import freeze_time
 
-# Internal Packages
 from khoj.processor.conversation.openai.gpt import converse, extract_questions
 from khoj.processor.conversation.utils import message_to_log
 
-
 # Initialize variables for tests
 api_key = os.getenv("OPENAI_API_KEY")
 if api_key is None:

tests/test_openai_chat_director.py

@@ -1,17 +1,14 @@
-# Standard Packages
 import os
 import urllib.parse
 from urllib.parse import quote
 
-# External Packages
 import pytest
 from freezegun import freeze_time
-from khoj.processor.conversation import prompts
 
-# Internal Packages
+from khoj.database.models import KhojUser
+from khoj.processor.conversation import prompts
 from khoj.processor.conversation.utils import message_to_log
 from tests.helpers import ConversationFactory
-from khoj.database.models import KhojUser
 
 # Initialize variables for tests
 api_key = os.getenv("OPENAI_API_KEY")

tests/test_org_to_entries.py

@@ -1,14 +1,11 @@
-# Standard Packages
 import json
 import os
 
-# Internal Packages
 from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
 from khoj.processor.content.text_to_entries import TextToEntries
-from khoj.utils.helpers import is_none_or_empty
-from khoj.utils.rawconfig import Entry
 from khoj.utils.fs_syncer import get_org_files
-from khoj.utils.rawconfig import TextContentConfig
+from khoj.utils.helpers import is_none_or_empty
+from khoj.utils.rawconfig import Entry, TextContentConfig
 
 
 def test_configure_heading_entry_to_jsonl(tmp_path):

tests/test_orgnode.py

@@ -1,7 +1,5 @@
-# Standard Packages
 import datetime
 
-# Internal Packages
 from khoj.processor.content.org_mode import orgnode
 
 

tests/test_pdf_to_entries.py

@@ -1,10 +1,7 @@
-# Standard Packages
 import json
 import os
 
-# Internal Packages
 from khoj.processor.content.pdf.pdf_to_entries import PdfToEntries
-
 from khoj.utils.fs_syncer import get_pdf_files
 from khoj.utils.rawconfig import TextContentConfig
 

tests/test_plaintext_to_entries.py

@@ -1,12 +1,9 @@
-# Standard Packages
 import json
 import os
 from pathlib import Path
 
 from khoj.database.models import KhojUser, LocalPlaintextConfig
 from khoj.processor.content.plaintext.plaintext_to_entries import PlaintextToEntries
-
-# Internal Packages
 from khoj.utils.fs_syncer import get_plaintext_files
 from khoj.utils.rawconfig import TextContentConfig
 

tests/test_rawconfig.py

@@ -1,8 +1,6 @@
-# External Packages
 import pytest
 
-# Internal Packages
-from khoj.utils.rawconfig import TextContentConfig, ImageContentConfig
+from khoj.utils.rawconfig import ImageContentConfig, TextContentConfig
 
 
 # Test

tests/test_text_search.py

@@ -4,14 +4,11 @@ import logging
 import os
 from pathlib import Path
 
-# External Packages
 import pytest
 
 from khoj.database.models import Entry, GithubConfig, KhojUser, LocalOrgConfig
 from khoj.processor.content.github.github_to_entries import GithubToEntries
 from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
-
-# Internal Packages
 from khoj.search_type import text_search
 from khoj.utils.fs_syncer import collect_files, get_org_files
 from khoj.utils.rawconfig import ContentConfig, SearchConfig