From 1e2af083f0efe88aba29801464b6b0d727ae922f Mon Sep 17 00:00:00 2001 From: sabaimran Date: Tue, 21 Nov 2023 22:11:32 -0800 Subject: [PATCH] Rename the data_sources module to content --- .../processor/{data_sources => content}/__init__.py | 0 .../{data_sources => content}/github/__init__.py | 0 .../github/github_to_entries.py | 4 ++-- .../{data_sources => content}/markdown/__init__.py | 0 .../markdown/markdown_to_entries.py | 0 .../notion/notion_to_entries.py | 0 .../{data_sources => content}/org_mode/__init__.py | 0 .../org_mode/org_to_entries.py | 2 +- .../{data_sources => content}/org_mode/orgnode.py | 0 .../{data_sources => content}/pdf/__init__.py | 0 .../{data_sources => content}/pdf/pdf_to_entries.py | 0 .../{data_sources => content}/plaintext/__init__.py | 0 .../plaintext/plaintext_to_entries.py | 0 src/khoj/routers/indexer.py | 12 ++++++------ tests/conftest.py | 4 ++-- tests/test_client.py | 2 +- tests/test_markdown_to_entries.py | 2 +- tests/test_multiple_users.py | 2 +- tests/test_org_to_entries.py | 2 +- tests/test_orgnode.py | 2 +- tests/test_pdf_to_entries.py | 2 +- tests/test_plaintext_to_entries.py | 2 +- tests/test_text_search.py | 4 ++-- 23 files changed, 20 insertions(+), 20 deletions(-) rename src/khoj/processor/{data_sources => content}/__init__.py (100%) rename src/khoj/processor/{data_sources => content}/github/__init__.py (100%) rename src/khoj/processor/{data_sources => content}/github/github_to_entries.py (98%) rename src/khoj/processor/{data_sources => content}/markdown/__init__.py (100%) rename src/khoj/processor/{data_sources => content}/markdown/markdown_to_entries.py (100%) rename src/khoj/processor/{data_sources => content}/notion/notion_to_entries.py (100%) rename src/khoj/processor/{data_sources => content}/org_mode/__init__.py (100%) rename src/khoj/processor/{data_sources => content}/org_mode/org_to_entries.py (99%) rename src/khoj/processor/{data_sources => content}/org_mode/orgnode.py (100%) rename src/khoj/processor/{data_sources => content}/pdf/__init__.py (100%) rename src/khoj/processor/{data_sources => content}/pdf/pdf_to_entries.py (100%) rename src/khoj/processor/{data_sources => content}/plaintext/__init__.py (100%) rename src/khoj/processor/{data_sources => content}/plaintext/plaintext_to_entries.py (100%) diff --git a/src/khoj/processor/data_sources/__init__.py b/src/khoj/processor/content/__init__.py similarity index 100% rename from src/khoj/processor/data_sources/__init__.py rename to src/khoj/processor/content/__init__.py diff --git a/src/khoj/processor/data_sources/github/__init__.py b/src/khoj/processor/content/github/__init__.py similarity index 100% rename from src/khoj/processor/data_sources/github/__init__.py rename to src/khoj/processor/content/github/__init__.py diff --git a/src/khoj/processor/data_sources/github/github_to_entries.py b/src/khoj/processor/content/github/github_to_entries.py similarity index 98% rename from src/khoj/processor/data_sources/github/github_to_entries.py rename to src/khoj/processor/content/github/github_to_entries.py index 05d18d8a..b0854850 100644 --- a/src/khoj/processor/data_sources/github/github_to_entries.py +++ b/src/khoj/processor/content/github/github_to_entries.py @@ -9,8 +9,8 @@ import requests from khoj.database.models import Entry as DbEntry from khoj.database.models import GithubConfig, KhojUser -from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries -from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries +from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntries +from khoj.processor.content.org_mode.org_to_entries import OrgToEntries from khoj.processor.text_to_entries import TextToEntries # Internal Packages diff --git a/src/khoj/processor/data_sources/markdown/__init__.py b/src/khoj/processor/content/markdown/__init__.py similarity index 100% rename from src/khoj/processor/data_sources/markdown/__init__.py rename to src/khoj/processor/content/markdown/__init__.py diff --git a/src/khoj/processor/data_sources/markdown/markdown_to_entries.py b/src/khoj/processor/content/markdown/markdown_to_entries.py similarity index 100% rename from src/khoj/processor/data_sources/markdown/markdown_to_entries.py rename to src/khoj/processor/content/markdown/markdown_to_entries.py diff --git a/src/khoj/processor/data_sources/notion/notion_to_entries.py b/src/khoj/processor/content/notion/notion_to_entries.py similarity index 100% rename from src/khoj/processor/data_sources/notion/notion_to_entries.py rename to src/khoj/processor/content/notion/notion_to_entries.py diff --git a/src/khoj/processor/data_sources/org_mode/__init__.py b/src/khoj/processor/content/org_mode/__init__.py similarity index 100% rename from src/khoj/processor/data_sources/org_mode/__init__.py rename to src/khoj/processor/content/org_mode/__init__.py diff --git a/src/khoj/processor/data_sources/org_mode/org_to_entries.py b/src/khoj/processor/content/org_mode/org_to_entries.py similarity index 99% rename from src/khoj/processor/data_sources/org_mode/org_to_entries.py rename to src/khoj/processor/content/org_mode/org_to_entries.py index 9e0d8e54..37c569e9 100644 --- a/src/khoj/processor/data_sources/org_mode/org_to_entries.py +++ b/src/khoj/processor/content/org_mode/org_to_entries.py @@ -7,7 +7,7 @@ from khoj.database.models import Entry as DbEntry from khoj.database.models import KhojUser # Internal Packages -from khoj.processor.data_sources.org_mode import orgnode +from khoj.processor.content.org_mode import orgnode from khoj.processor.text_to_entries import TextToEntries from khoj.utils import state from khoj.utils.helpers import timer diff --git a/src/khoj/processor/data_sources/org_mode/orgnode.py b/src/khoj/processor/content/org_mode/orgnode.py similarity index 100% rename from src/khoj/processor/data_sources/org_mode/orgnode.py rename to src/khoj/processor/content/org_mode/orgnode.py diff --git a/src/khoj/processor/data_sources/pdf/__init__.py b/src/khoj/processor/content/pdf/__init__.py similarity index 100% rename from src/khoj/processor/data_sources/pdf/__init__.py rename to src/khoj/processor/content/pdf/__init__.py diff --git a/src/khoj/processor/data_sources/pdf/pdf_to_entries.py b/src/khoj/processor/content/pdf/pdf_to_entries.py similarity index 100% rename from src/khoj/processor/data_sources/pdf/pdf_to_entries.py rename to src/khoj/processor/content/pdf/pdf_to_entries.py diff --git a/src/khoj/processor/data_sources/plaintext/__init__.py b/src/khoj/processor/content/plaintext/__init__.py similarity index 100% rename from src/khoj/processor/data_sources/plaintext/__init__.py rename to src/khoj/processor/content/plaintext/__init__.py diff --git a/src/khoj/processor/data_sources/plaintext/plaintext_to_entries.py b/src/khoj/processor/content/plaintext/plaintext_to_entries.py similarity index 100% rename from src/khoj/processor/data_sources/plaintext/plaintext_to_entries.py rename to src/khoj/processor/content/plaintext/plaintext_to_entries.py diff --git a/src/khoj/routers/indexer.py b/src/khoj/routers/indexer.py index 6fcf3348..0432eed0 100644 --- a/src/khoj/routers/indexer.py +++ b/src/khoj/routers/indexer.py @@ -7,12 +7,12 @@ from pydantic import BaseModel from starlette.authentication import requires from khoj.database.models import GithubConfig, KhojUser, NotionConfig -from khoj.processor.data_sources.github.github_to_entries import GithubToEntries -from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries -from khoj.processor.data_sources.notion.notion_to_entries import NotionToEntries -from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries -from khoj.processor.data_sources.pdf.pdf_to_entries import PdfToEntries -from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries +from khoj.processor.content.github.github_to_entries import GithubToEntries +from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntries +from khoj.processor.content.notion.notion_to_entries import NotionToEntries +from khoj.processor.content.org_mode.org_to_entries import OrgToEntries +from khoj.processor.content.pdf.pdf_to_entries import PdfToEntries +from khoj.processor.content.plaintext.plaintext_to_entries import PlaintextToEntries from khoj.routers.helpers import update_telemetry_state from khoj.search_type import image_search, text_search from khoj.utils import constants, state diff --git a/tests/conftest.py b/tests/conftest.py index c18c5f80..54c664d5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,8 +18,8 @@ from khoj.database.models import ( LocalOrgConfig, LocalPlaintextConfig, ) -from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries -from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries +from khoj.processor.content.org_mode.org_to_entries import OrgToEntries +from khoj.processor.content.plaintext.plaintext_to_entries import PlaintextToEntries from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel from khoj.routers.indexer import configure_content from khoj.search_type import image_search, text_search diff --git a/tests/test_client.py b/tests/test_client.py index aecd0498..19aba03b 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -13,7 +13,7 @@ from PIL import Image from khoj.configure import configure_routes, configure_search_types from khoj.database.adapters import EntryAdapters from khoj.database.models import KhojApiUser, KhojUser -from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries +from khoj.processor.content.org_mode.org_to_entries import OrgToEntries from khoj.search_type import image_search, text_search from khoj.utils import state from khoj.utils.rawconfig import ContentConfig, SearchConfig diff --git a/tests/test_markdown_to_entries.py b/tests/test_markdown_to_entries.py index 9ec88382..b20040f9 100644 --- a/tests/test_markdown_to_entries.py +++ b/tests/test_markdown_to_entries.py @@ -4,7 +4,7 @@ from pathlib import Path import os # Internal Packages -from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries +from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntries from khoj.utils.fs_syncer import get_markdown_files from khoj.utils.rawconfig import TextContentConfig diff --git a/tests/test_multiple_users.py b/tests/test_multiple_users.py index d2d7737b..dae881bf 100644 --- a/tests/test_multiple_users.py +++ b/tests/test_multiple_users.py @@ -5,7 +5,7 @@ from urllib.parse import quote import pytest from khoj.database.models import KhojApiUser, KhojUser -from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries +from khoj.processor.content.org_mode.org_to_entries import OrgToEntries # Internal Packages from khoj.search_type import text_search diff --git a/tests/test_org_to_entries.py b/tests/test_org_to_entries.py index fb37426a..742c9f8e 100644 --- a/tests/test_org_to_entries.py +++ b/tests/test_org_to_entries.py @@ -3,7 +3,7 @@ import json import os # Internal Packages -from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries +from khoj.processor.content.org_mode.org_to_entries import OrgToEntries from khoj.processor.text_to_entries import TextToEntries from khoj.utils.helpers import is_none_or_empty from khoj.utils.rawconfig import Entry diff --git a/tests/test_orgnode.py b/tests/test_orgnode.py index aa6a3cb9..a543ceb5 100644 --- a/tests/test_orgnode.py +++ b/tests/test_orgnode.py @@ -2,7 +2,7 @@ import datetime # Internal Packages -from khoj.processor.data_sources.org_mode import orgnode +from khoj.processor.content.org_mode import orgnode # Test diff --git a/tests/test_pdf_to_entries.py b/tests/test_pdf_to_entries.py index ebf53025..9aeb438f 100644 --- a/tests/test_pdf_to_entries.py +++ b/tests/test_pdf_to_entries.py @@ -3,7 +3,7 @@ import json import os # Internal Packages -from khoj.processor.data_sources.pdf.pdf_to_entries import PdfToEntries +from khoj.processor.content.pdf.pdf_to_entries import PdfToEntries from khoj.utils.fs_syncer import get_pdf_files from khoj.utils.rawconfig import TextContentConfig diff --git a/tests/test_plaintext_to_entries.py b/tests/test_plaintext_to_entries.py index 393fac17..d3d50c94 100644 --- a/tests/test_plaintext_to_entries.py +++ b/tests/test_plaintext_to_entries.py @@ -4,7 +4,7 @@ import os from pathlib import Path from khoj.database.models import KhojUser, LocalPlaintextConfig -from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries +from khoj.processor.content.plaintext.plaintext_to_entries import PlaintextToEntries # Internal Packages from khoj.utils.fs_syncer import get_plaintext_files diff --git a/tests/test_text_search.py b/tests/test_text_search.py index f19903ae..186ef57c 100644 --- a/tests/test_text_search.py +++ b/tests/test_text_search.py @@ -8,8 +8,8 @@ from pathlib import Path import pytest from khoj.database.models import Entry, GithubConfig, KhojUser, LocalOrgConfig -from khoj.processor.data_sources.github.github_to_entries import GithubToEntries -from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries +from khoj.processor.content.github.github_to_entries import GithubToEntries +from khoj.processor.content.org_mode.org_to_entries import OrgToEntries # Internal Packages from khoj.search_type import text_search