diff --git a/src/khoj/processor/data_sources/__init__.py b/src/khoj/processor/content/__init__.py similarity index 100% rename from src/khoj/processor/data_sources/__init__.py rename to src/khoj/processor/content/__init__.py diff --git a/src/khoj/processor/data_sources/github/__init__.py b/src/khoj/processor/content/github/__init__.py similarity index 100% rename from src/khoj/processor/data_sources/github/__init__.py rename to src/khoj/processor/content/github/__init__.py diff --git a/src/khoj/processor/data_sources/github/github_to_entries.py b/src/khoj/processor/content/github/github_to_entries.py similarity index 98% rename from src/khoj/processor/data_sources/github/github_to_entries.py rename to src/khoj/processor/content/github/github_to_entries.py index 05d18d8a..b0854850 100644 --- a/src/khoj/processor/data_sources/github/github_to_entries.py +++ b/src/khoj/processor/content/github/github_to_entries.py @@ -9,8 +9,8 @@ import requests from khoj.database.models import Entry as DbEntry from khoj.database.models import GithubConfig, KhojUser -from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries -from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries +from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntries +from khoj.processor.content.org_mode.org_to_entries import OrgToEntries from khoj.processor.text_to_entries import TextToEntries # Internal Packages diff --git a/src/khoj/processor/data_sources/markdown/__init__.py b/src/khoj/processor/content/markdown/__init__.py similarity index 100% rename from src/khoj/processor/data_sources/markdown/__init__.py rename to src/khoj/processor/content/markdown/__init__.py diff --git a/src/khoj/processor/data_sources/markdown/markdown_to_entries.py b/src/khoj/processor/content/markdown/markdown_to_entries.py similarity index 100% rename from src/khoj/processor/data_sources/markdown/markdown_to_entries.py rename to src/khoj/processor/content/markdown/markdown_to_entries.py diff --git a/src/khoj/processor/data_sources/notion/notion_to_entries.py b/src/khoj/processor/content/notion/notion_to_entries.py similarity index 100% rename from src/khoj/processor/data_sources/notion/notion_to_entries.py rename to src/khoj/processor/content/notion/notion_to_entries.py diff --git a/src/khoj/processor/data_sources/org_mode/__init__.py b/src/khoj/processor/content/org_mode/__init__.py similarity index 100% rename from src/khoj/processor/data_sources/org_mode/__init__.py rename to src/khoj/processor/content/org_mode/__init__.py diff --git a/src/khoj/processor/data_sources/org_mode/org_to_entries.py b/src/khoj/processor/content/org_mode/org_to_entries.py similarity index 99% rename from src/khoj/processor/data_sources/org_mode/org_to_entries.py rename to src/khoj/processor/content/org_mode/org_to_entries.py index 9e0d8e54..37c569e9 100644 --- a/src/khoj/processor/data_sources/org_mode/org_to_entries.py +++ b/src/khoj/processor/content/org_mode/org_to_entries.py @@ -7,7 +7,7 @@ from khoj.database.models import Entry as DbEntry from khoj.database.models import KhojUser # Internal Packages -from khoj.processor.data_sources.org_mode import orgnode +from khoj.processor.content.org_mode import orgnode from khoj.processor.text_to_entries import TextToEntries from khoj.utils import state from khoj.utils.helpers import timer diff --git a/src/khoj/processor/data_sources/org_mode/orgnode.py b/src/khoj/processor/content/org_mode/orgnode.py similarity index 100% rename from src/khoj/processor/data_sources/org_mode/orgnode.py rename to src/khoj/processor/content/org_mode/orgnode.py diff --git a/src/khoj/processor/data_sources/pdf/__init__.py b/src/khoj/processor/content/pdf/__init__.py similarity index 100% rename from src/khoj/processor/data_sources/pdf/__init__.py rename to src/khoj/processor/content/pdf/__init__.py diff --git a/src/khoj/processor/data_sources/pdf/pdf_to_entries.py b/src/khoj/processor/content/pdf/pdf_to_entries.py similarity index 100% rename from src/khoj/processor/data_sources/pdf/pdf_to_entries.py rename to src/khoj/processor/content/pdf/pdf_to_entries.py diff --git a/src/khoj/processor/data_sources/plaintext/__init__.py b/src/khoj/processor/content/plaintext/__init__.py similarity index 100% rename from src/khoj/processor/data_sources/plaintext/__init__.py rename to src/khoj/processor/content/plaintext/__init__.py diff --git a/src/khoj/processor/data_sources/plaintext/plaintext_to_entries.py b/src/khoj/processor/content/plaintext/plaintext_to_entries.py similarity index 100% rename from src/khoj/processor/data_sources/plaintext/plaintext_to_entries.py rename to src/khoj/processor/content/plaintext/plaintext_to_entries.py diff --git a/src/khoj/routers/indexer.py b/src/khoj/routers/indexer.py index 6fcf3348..0432eed0 100644 --- a/src/khoj/routers/indexer.py +++ b/src/khoj/routers/indexer.py @@ -7,12 +7,12 @@ from pydantic import BaseModel from starlette.authentication import requires from khoj.database.models import GithubConfig, KhojUser, NotionConfig -from khoj.processor.data_sources.github.github_to_entries import GithubToEntries -from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries -from khoj.processor.data_sources.notion.notion_to_entries import NotionToEntries -from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries -from khoj.processor.data_sources.pdf.pdf_to_entries import PdfToEntries -from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries +from khoj.processor.content.github.github_to_entries import GithubToEntries +from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntries +from khoj.processor.content.notion.notion_to_entries import NotionToEntries +from khoj.processor.content.org_mode.org_to_entries import OrgToEntries +from khoj.processor.content.pdf.pdf_to_entries import PdfToEntries +from khoj.processor.content.plaintext.plaintext_to_entries import PlaintextToEntries from khoj.routers.helpers import update_telemetry_state from khoj.search_type import image_search, text_search from khoj.utils import constants, state diff --git a/tests/conftest.py b/tests/conftest.py index c18c5f80..54c664d5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,8 +18,8 @@ from khoj.database.models import ( LocalOrgConfig, LocalPlaintextConfig, ) -from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries -from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries +from khoj.processor.content.org_mode.org_to_entries import OrgToEntries +from khoj.processor.content.plaintext.plaintext_to_entries import PlaintextToEntries from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel from khoj.routers.indexer import configure_content from khoj.search_type import image_search, text_search diff --git a/tests/test_client.py b/tests/test_client.py index aecd0498..19aba03b 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -13,7 +13,7 @@ from PIL import Image from khoj.configure import configure_routes, configure_search_types from khoj.database.adapters import EntryAdapters from khoj.database.models import KhojApiUser, KhojUser -from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries +from khoj.processor.content.org_mode.org_to_entries import OrgToEntries from khoj.search_type import image_search, text_search from khoj.utils import state from khoj.utils.rawconfig import ContentConfig, SearchConfig diff --git a/tests/test_markdown_to_entries.py b/tests/test_markdown_to_entries.py index 9ec88382..b20040f9 100644 --- a/tests/test_markdown_to_entries.py +++ b/tests/test_markdown_to_entries.py @@ -4,7 +4,7 @@ from pathlib import Path import os # Internal Packages -from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries +from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntries from khoj.utils.fs_syncer import get_markdown_files from khoj.utils.rawconfig import TextContentConfig diff --git a/tests/test_multiple_users.py b/tests/test_multiple_users.py index d2d7737b..dae881bf 100644 --- a/tests/test_multiple_users.py +++ b/tests/test_multiple_users.py @@ -5,7 +5,7 @@ from urllib.parse import quote import pytest from khoj.database.models import KhojApiUser, KhojUser -from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries +from khoj.processor.content.org_mode.org_to_entries import OrgToEntries # Internal Packages from khoj.search_type import text_search diff --git a/tests/test_org_to_entries.py b/tests/test_org_to_entries.py index fb37426a..742c9f8e 100644 --- a/tests/test_org_to_entries.py +++ b/tests/test_org_to_entries.py @@ -3,7 +3,7 @@ import json import os # Internal Packages -from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries +from khoj.processor.content.org_mode.org_to_entries import OrgToEntries from khoj.processor.text_to_entries import TextToEntries from khoj.utils.helpers import is_none_or_empty from khoj.utils.rawconfig import Entry diff --git a/tests/test_orgnode.py b/tests/test_orgnode.py index aa6a3cb9..a543ceb5 100644 --- a/tests/test_orgnode.py +++ b/tests/test_orgnode.py @@ -2,7 +2,7 @@ import datetime # Internal Packages -from khoj.processor.data_sources.org_mode import orgnode +from khoj.processor.content.org_mode import orgnode # Test diff --git a/tests/test_pdf_to_entries.py b/tests/test_pdf_to_entries.py index ebf53025..9aeb438f 100644 --- a/tests/test_pdf_to_entries.py +++ b/tests/test_pdf_to_entries.py @@ -3,7 +3,7 @@ import json import os # Internal Packages -from khoj.processor.data_sources.pdf.pdf_to_entries import PdfToEntries +from khoj.processor.content.pdf.pdf_to_entries import PdfToEntries from khoj.utils.fs_syncer import get_pdf_files from khoj.utils.rawconfig import TextContentConfig diff --git a/tests/test_plaintext_to_entries.py b/tests/test_plaintext_to_entries.py index 393fac17..d3d50c94 100644 --- a/tests/test_plaintext_to_entries.py +++ b/tests/test_plaintext_to_entries.py @@ -4,7 +4,7 @@ import os from pathlib import Path from khoj.database.models import KhojUser, LocalPlaintextConfig -from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries +from khoj.processor.content.plaintext.plaintext_to_entries import PlaintextToEntries # Internal Packages from khoj.utils.fs_syncer import get_plaintext_files diff --git a/tests/test_text_search.py b/tests/test_text_search.py index f19903ae..186ef57c 100644 --- a/tests/test_text_search.py +++ b/tests/test_text_search.py @@ -8,8 +8,8 @@ from pathlib import Path import pytest from khoj.database.models import Entry, GithubConfig, KhojUser, LocalOrgConfig -from khoj.processor.data_sources.github.github_to_entries import GithubToEntries -from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries +from khoj.processor.content.github.github_to_entries import GithubToEntries +from khoj.processor.content.org_mode.org_to_entries import OrgToEntries # Internal Packages from khoj.search_type import text_search