diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 84fbb1aa..697579da 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,7 +61,7 @@ jobs: env: DEBIAN_FRONTEND: noninteractive run: | - apt update && apt install -y libegl1 sqlite3 libsqlite3-dev libsqlite3-0 + apt update && apt install -y libegl1 sqlite3 libsqlite3-dev libsqlite3-0 ffmpeg libsm6 libxext6 - name: ⬇️ Install Postgres env: diff --git a/prod.Dockerfile b/prod.Dockerfile index 3cf6a600..693a3a8b 100644 --- a/prod.Dockerfile +++ b/prod.Dockerfile @@ -4,7 +4,7 @@ FROM nvidia/cuda:12.2.0-devel-ubuntu22.04 LABEL org.opencontainers.image.source https://github.com/khoj-ai/khoj # Install System Dependencies -RUN apt update -y && apt -y install python3-pip git +RUN apt update -y && apt -y install python3-pip git libsqlite3-0 ffmpeg libsm6 libxext6 WORKDIR /app diff --git a/pyproject.toml b/pyproject.toml index c816f4d2..25a78ab9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,7 @@ dependencies = [ "gunicorn == 21.2.0", "lxml == 4.9.3", "tzdata == 2023.3", + "rapidocr-onnxruntime == 1.3.8" ] dynamic = ["version"] diff --git a/src/database/adapters/__init__.py b/src/database/adapters/__init__.py index 7fbc5287..a54a640a 100644 --- a/src/database/adapters/__init__.py +++ b/src/database/adapters/__init__.py @@ -291,8 +291,11 @@ class EntryAdapters: return deleted_count @staticmethod - def delete_all_entries(user: KhojUser, file_type: str): - deleted_count, _ = Entry.objects.filter(user=user, file_type=file_type).delete() + def delete_all_entries(user: KhojUser, file_type: str = None): + if file_type is None: + deleted_count, _ = Entry.objects.filter(user=user).delete() + else: + deleted_count, _ = Entry.objects.filter(user=user, file_type=file_type).delete() return deleted_count @staticmethod @@ -314,6 +317,18 @@ class EntryAdapters: async def user_has_entries(user: KhojUser): return await Entry.objects.filter(user=user).aexists() + @staticmethod + async def adelete_entry_by_file(user: KhojUser, file_path: str): + return await Entry.objects.filter(user=user, file_path=file_path).adelete() + + @staticmethod + def aget_all_filenames(user: KhojUser): + return Entry.objects.filter(user=user).distinct("file_path").values_list("file_path", flat=True) + + @staticmethod + async def adelete_all_entries(user: KhojUser): + return await Entry.objects.filter(user=user).adelete() + @staticmethod def apply_filters(user: KhojUser, query: str, file_type_filter: str = None): q_filter_terms = Q() diff --git a/src/interface/desktop/assets/khoj.css b/src/interface/desktop/assets/khoj.css index 649f79bb..b2e048c5 100644 --- a/src/interface/desktop/assets/khoj.css +++ b/src/interface/desktop/assets/khoj.css @@ -103,21 +103,6 @@ img.khoj-logo { justify-self: center; } -a.khoj-banner { - color: black; - text-decoration: none; -} - -p.khoj-banner { - font-size: small; - margin: 0; - padding: 10px; -} - -p#khoj-banner { - display: inline; -} - @media only screen and (max-width: 600px) { div.khoj-header { display: grid; diff --git a/src/interface/desktop/chat.html b/src/interface/desktop/chat.html index b27bbc90..b908b747 100644 --- a/src/interface/desktop/chat.html +++ b/src/interface/desktop/chat.html @@ -274,8 +274,9 @@ } -
+
+
-

- Enroll in Khoj cloud to get your own assistant -

- - - - {% endif %} +
@@ -480,12 +470,6 @@ margin: 4px; grid-template-columns: auto; } - a.khoj-banner { - display: block; - } - p.khoj-banner { - padding: 0; - } } @media only screen and (min-width: 700px) { body { @@ -497,14 +481,6 @@ } } - div.khoj-banner-container { - background: linear-gradient(-45deg, #FFC107, #FF9800, #FF5722, #FF9800, #FFC107); - background-size: 400% 400%; - animation: gradient 15s ease infinite; - text-align: center; - padding: 10px; - } - div#chat-tooltip { text-align: left; font-size: medium; @@ -526,19 +502,7 @@ text-align: center; } - button#khoj-banner-submit, - input#khoj-banner-email { - padding: 10px; - border-radius: 5px; - border: 1px solid var(--main-text-color); - background: #f9fafc; - } - - button#khoj-banner-submit:hover, - input#khoj-banner-email:hover { - box-shadow: 0 0 11px #aaa; - } - div.khoj-banner-container-hidden { + div.khoj-empty-container { margin: 0px; padding: 0px; } @@ -558,39 +522,4 @@ white-space: pre-wrap; } - diff --git a/src/khoj/interface/web/config.html b/src/khoj/interface/web/config.html index 851a18d0..b19bbff6 100644 --- a/src/khoj/interface/web/config.html +++ b/src/khoj/interface/web/config.html @@ -67,130 +67,6 @@
{% endif %}
-
-
- markdown -

- Markdown - {% if current_model_state.markdown == True%} - Configured - {% endif %} -

-
-
-

Set markdown files to index

-
-
- - {% if current_model_state.markdown %} - Update - {% else %} - Setup - {% endif %} - - -
- {% if current_model_state.markdown %} -
- -
- {% endif %} -
-
-
- org -

- Org - {% if current_model_state.org == True %} - Configured - {% endif %} -

-
-
-

Set org files to index

-
-
- - {% if current_model_state.org %} - Update - {% else %} - Setup - {% endif %} - - -
- {% if current_model_state.org %} -
- -
- {% endif %} -
-
-
- PDF -

- PDF - {% if current_model_state.pdf == True %} - Configured - {% endif %} -

-
-
-

Set PDF files to index

-
-
- - {% if current_model_state.pdf %} - Update - {% else %} - Setup - {% endif %} - - -
- {% if current_model_state.pdf %} -
- -
- {% endif %} -
-
-
- Plaintext -

- Plaintext - {% if current_model_state.plaintext == True %} - Configured - {% endif %} -

-
-
-

Set Plaintext files to index

-
-
- - {% if current_model_state.plaintext %} - Update - {% else %} - Setup - {% endif %} - - -
- {% if current_model_state.plaintext %} -
- -
- {% endif %} -
@@ -246,6 +122,16 @@
+
+

Manage Data

+
+
+ +
+
+
+
+
@@ -291,8 +177,8 @@ }; function clearContentType(content_type) { - fetch('/api/delete/config/data/content_type/' + content_type, { - method: 'POST', + fetch('/api/config/data/content_type/' + content_type, { + method: 'DELETE', headers: { 'Content-Type': 'application/json', } @@ -462,5 +348,84 @@ // List user's API keys on page load listApiKeys(); + function removeFile(path) { + fetch('/api/config/data/file?filename=' + path, { + method: 'DELETE', + headers: { + 'Content-Type': 'application/json', + } + }) + .then(response => response.json()) + .then(data => { + if (data.status == "ok") { + getAllFilenames(); + } + }) + } + + // Get all currently indexed files + function getAllFilenames() { + fetch('/api/config/data/all') + .then(response => response.json()) + .then(data => { + var indexedFiles = document.getElementsByClassName("indexed-files")[0]; + indexedFiles.innerHTML = ""; + + if (data.length == 0) { + document.getElementById("delete-all-files").style.display = "none"; + indexedFiles.innerHTML = "
Use the Khoj Desktop client to index files.
"; + } else { + document.getElementById("delete-all-files").style.display = "block"; + } + + for (var filename of data) { + let fileElement = document.createElement("div"); + fileElement.classList.add("file-element"); + + let fileNameElement = document.createElement("div"); + fileNameElement.classList.add("content-name"); + fileNameElement.innerHTML = filename; + fileElement.appendChild(fileNameElement); + + let buttonContainer = document.createElement("div"); + buttonContainer.classList.add("remove-button-container"); + let removeFileButton = document.createElement("button"); + removeFileButton.classList.add("remove-file-button"); + removeFileButton.innerHTML = "🗑️"; + removeFileButton.addEventListener("click", ((filename) => { + return () => { + removeFile(filename); + }; + })(filename)); + buttonContainer.appendChild(removeFileButton); + fileElement.appendChild(buttonContainer); + indexedFiles.appendChild(fileElement); + } + }) + .catch((error) => { + console.error('Error:', error); + }); + } + + // Get all currently indexed files on page load + getAllFilenames(); + + let deleteAllFilesButton = document.getElementById("delete-all-files"); + deleteAllFilesButton.addEventListener("click", function(event) { + event.preventDefault(); + fetch('/api/config/data/all', { + method: 'DELETE', + headers: { + 'Content-Type': 'application/json', + } + }) + .then(response => response.json()) + .then(data => { + if (data.status == "ok") { + getAllFilenames(); + } + }) + }); + {% endblock %} diff --git a/src/khoj/interface/web/login.html b/src/khoj/interface/web/login.html index 2aace820..a191de7b 100644 --- a/src/khoj/interface/web/login.html +++ b/src/khoj/interface/web/login.html @@ -10,18 +10,6 @@ - {% if demo %} - -
- -

- Enroll in Khoj cloud to get your own assistant -

- - - -
- {% endif %}
@@ -106,19 +94,6 @@ justify-self: center; } - button#khoj-banner-submit, - input#khoj-banner-email { - padding: 10px; - border-radius: 5px; - border: 1px solid #475569; - background: #f9fafc; - } - - button#khoj-banner-submit:hover, - input#khoj-banner-email:hover { - box-shadow: 0 0 11px #aaa; - } - div#login-modal { display: grid; grid-template-columns: 1fr; @@ -143,12 +118,6 @@ } @media only screen and (max-width: 700px) { - a.khoj-banner { - display: block; - } - p.khoj-banner { - padding: 0; - } div#login-modal { margin-left: 10%; margin-right: 10%; @@ -156,34 +125,5 @@ } - diff --git a/src/khoj/interface/web/search.html b/src/khoj/interface/web/search.html index 1a014a93..8f98d0f3 100644 --- a/src/khoj/interface/web/search.html +++ b/src/khoj/interface/web/search.html @@ -270,19 +270,6 @@ - {% if demo %} - -
- -

- Enroll in Khoj cloud to get your own assistant -

- - - -
- {% endif %} - {% import 'utils.html' as utils %} {{ utils.heading_pane(user_photo, username) }} @@ -458,14 +445,6 @@ max-width: 90%; } - div.khoj-banner-container { - background: linear-gradient(-45deg, #FFC107, #FF9800, #FF5722, #FF9800, #FFC107); - background-size: 400% 400%; - animation: gradient 15s ease infinite; - text-align: center; - padding: 10px; - } - @keyframes gradient { 0% { background-position: 0% 50%; @@ -482,57 +461,6 @@ text-align: center; } - button#khoj-banner-submit, - input#khoj-banner-email { - padding: 10px; - border-radius: 5px; - border: 1px solid var(--main-text-color); - background: #f9fafc; - } - - button#khoj-banner-submit:hover, - input#khoj-banner-email:hover { - box-shadow: 0 0 11px #aaa; - } - - @media only screen and (max-width: 700px) { - a.khoj-banner { - display: block; - } - p.khoj-banner { - padding: 0; - } - } - - diff --git a/src/khoj/main.py b/src/khoj/main.py index 33029b94..f92e8cbe 100644 --- a/src/khoj/main.py +++ b/src/khoj/main.py @@ -119,7 +119,6 @@ def set_state(args): state.verbose = args.verbose state.host = args.host state.port = args.port - state.demo = args.demo state.anonymous_mode = args.anonymous_mode state.khoj_version = version("khoj-assistant") state.chat_on_gpu = args.chat_on_gpu diff --git a/src/khoj/processor/pdf/pdf_to_entries.py b/src/khoj/processor/pdf/pdf_to_entries.py index 24dcdc5a..81c2250f 100644 --- a/src/khoj/processor/pdf/pdf_to_entries.py +++ b/src/khoj/processor/pdf/pdf_to_entries.py @@ -68,13 +68,17 @@ class PdfToEntries(TextToEntries): with open(f"{tmp_file}", "wb") as f: bytes = pdf_files[pdf_file] f.write(bytes) - loader = PyMuPDFLoader(f"{tmp_file}") - pdf_entries_per_file = [page.page_content for page in loader.load()] + try: + loader = PyMuPDFLoader(f"{tmp_file}", extract_images=True) + pdf_entries_per_file = [page.page_content for page in loader.load()] + except ImportError: + loader = PyMuPDFLoader(f"{tmp_file}") + pdf_entries_per_file = [page.page_content for page in loader.load()] entry_to_location_map += zip(pdf_entries_per_file, [pdf_file] * len(pdf_entries_per_file)) entries.extend(pdf_entries_per_file) except Exception as e: logger.warning(f"Unable to process file: {pdf_file}. This file will not be indexed.") - logger.warning(e) + logger.warning(e, exc_info=True) finally: if os.path.exists(f"{tmp_file}"): os.remove(f"{tmp_file}") diff --git a/src/khoj/routers/api.py b/src/khoj/routers/api.py index b8a5350b..84e63b09 100644 --- a/src/khoj/routers/api.py +++ b/src/khoj/routers/api.py @@ -45,7 +45,15 @@ from fastapi.requests import Request from database import adapters from database.adapters import EntryAdapters, ConversationAdapters -from database.models import LocalMarkdownConfig, LocalOrgConfig, LocalPdfConfig, LocalPlaintextConfig, KhojUser +from database.models import ( + LocalMarkdownConfig, + LocalOrgConfig, + LocalPdfConfig, + LocalPlaintextConfig, + KhojUser, + GithubConfig, + NotionConfig, +) # Initialize Router @@ -54,14 +62,10 @@ logger = logging.getLogger(__name__) def map_config_to_object(content_type: str): - if content_type == "org": - return LocalOrgConfig - if content_type == "markdown": - return LocalMarkdownConfig - if content_type == "pdf": - return LocalPdfConfig - if content_type == "plaintext": - return LocalPlaintextConfig + if content_type == "github": + return GithubConfig + if content_type == "notion": + return NotionConfig async def map_config_to_db(config: FullConfig, user: KhojUser): @@ -111,183 +115,220 @@ async def map_config_to_db(config: FullConfig, user: KhojUser): ) -# If it's a demo instance, prevent updating any of the configuration. -if not state.demo: +def _initialize_config(): + if state.config is None: + state.config = FullConfig() + state.config.search_type = SearchConfig.parse_obj(constants.default_config["search-type"]) - def _initialize_config(): - if state.config is None: - state.config = FullConfig() - state.config.search_type = SearchConfig.parse_obj(constants.default_config["search-type"]) - @api.get("/config/data", response_model=FullConfig) - @requires(["authenticated"]) - def get_config_data(request: Request): - user = request.user.object - EntryAdapters.get_unique_file_types(user) +@api.get("/config/data", response_model=FullConfig) +@requires(["authenticated"]) +def get_config_data(request: Request): + user = request.user.object + EntryAdapters.get_unique_file_types(user) - return state.config + return state.config - @api.post("/config/data") - @requires(["authenticated"]) - async def set_config_data( - request: Request, - updated_config: FullConfig, - client: Optional[str] = None, - ): - user = request.user.object - await map_config_to_db(updated_config, user) - configuration_update_metadata = {} +@api.post("/config/data") +@requires(["authenticated"]) +async def set_config_data( + request: Request, + updated_config: FullConfig, + client: Optional[str] = None, +): + user = request.user.object + await map_config_to_db(updated_config, user) - enabled_content = await sync_to_async(EntryAdapters.get_unique_file_types)(user) + configuration_update_metadata = {} - if state.config.content_type is not None: - configuration_update_metadata["github"] = "github" in enabled_content - configuration_update_metadata["notion"] = "notion" in enabled_content - configuration_update_metadata["org"] = "org" in enabled_content - configuration_update_metadata["pdf"] = "pdf" in enabled_content - configuration_update_metadata["markdown"] = "markdown" in enabled_content + enabled_content = await sync_to_async(EntryAdapters.get_unique_file_types)(user) - if state.config.processor is not None: - configuration_update_metadata["conversation_processor"] = state.config.processor.conversation is not None + if state.config.content_type is not None: + configuration_update_metadata["github"] = "github" in enabled_content + configuration_update_metadata["notion"] = "notion" in enabled_content + configuration_update_metadata["org"] = "org" in enabled_content + configuration_update_metadata["pdf"] = "pdf" in enabled_content + configuration_update_metadata["markdown"] = "markdown" in enabled_content - update_telemetry_state( - request=request, - telemetry_type="api", - api="set_config", - client=client, - metadata=configuration_update_metadata, - ) - return state.config + if state.config.processor is not None: + configuration_update_metadata["conversation_processor"] = state.config.processor.conversation is not None - @api.post("/config/data/content_type/github", status_code=200) - @requires(["authenticated"]) - async def set_content_config_github_data( - request: Request, - updated_config: Union[GithubContentConfig, None], - client: Optional[str] = None, - ): - _initialize_config() + update_telemetry_state( + request=request, + telemetry_type="api", + api="set_config", + client=client, + metadata=configuration_update_metadata, + ) + return state.config - user = request.user.object - await adapters.set_user_github_config( - user=user, - pat_token=updated_config.pat_token, - repos=updated_config.repos, - ) +@api.post("/config/data/content_type/github", status_code=200) +@requires(["authenticated"]) +async def set_content_config_github_data( + request: Request, + updated_config: Union[GithubContentConfig, None], + client: Optional[str] = None, +): + _initialize_config() - update_telemetry_state( - request=request, - telemetry_type="api", - api="set_content_config", - client=client, - metadata={"content_type": "github"}, - ) + user = request.user.object - return {"status": "ok"} + await adapters.set_user_github_config( + user=user, + pat_token=updated_config.pat_token, + repos=updated_config.repos, + ) - @api.post("/config/data/content_type/notion", status_code=200) - @requires(["authenticated"]) - async def set_content_config_notion_data( - request: Request, - updated_config: Union[NotionContentConfig, None], - client: Optional[str] = None, - ): - _initialize_config() + update_telemetry_state( + request=request, + telemetry_type="api", + api="set_content_config", + client=client, + metadata={"content_type": "github"}, + ) - user = request.user.object + return {"status": "ok"} - await adapters.set_notion_config( - user=user, - token=updated_config.token, - ) - update_telemetry_state( - request=request, - telemetry_type="api", - api="set_content_config", - client=client, - metadata={"content_type": "notion"}, - ) +@api.post("/config/data/content_type/notion", status_code=200) +@requires(["authenticated"]) +async def set_content_config_notion_data( + request: Request, + updated_config: Union[NotionContentConfig, None], + client: Optional[str] = None, +): + _initialize_config() - return {"status": "ok"} + user = request.user.object - @api.post("/delete/config/data/content_type/{content_type}", status_code=200) - @requires(["authenticated"]) - async def remove_content_config_data( - request: Request, - content_type: str, - client: Optional[str] = None, - ): - user = request.user.object + await adapters.set_notion_config( + user=user, + token=updated_config.token, + ) - update_telemetry_state( - request=request, - telemetry_type="api", - api="delete_content_config", - client=client, - metadata={"content_type": content_type}, - ) + update_telemetry_state( + request=request, + telemetry_type="api", + api="set_content_config", + client=client, + metadata={"content_type": "notion"}, + ) - content_object = map_config_to_object(content_type) - if content_object is None: - raise ValueError(f"Invalid content type: {content_type}") + return {"status": "ok"} - await content_object.objects.filter(user=user).adelete() - await sync_to_async(EntryAdapters.delete_all_entries)(user, content_type) - enabled_content = await sync_to_async(EntryAdapters.get_unique_file_types)(user) - return {"status": "ok"} +@api.delete("/config/data/content_type/{content_type}", status_code=200) +@requires(["authenticated"]) +async def remove_content_config_data( + request: Request, + content_type: str, + client: Optional[str] = None, +): + user = request.user.object - @api.post("/config/data/content_type/{content_type}", status_code=200) - @requires(["authenticated"]) - async def set_content_config_data( - request: Request, - content_type: str, - updated_config: Union[TextContentConfig, None], - client: Optional[str] = None, - ): - _initialize_config() + update_telemetry_state( + request=request, + telemetry_type="api", + api="delete_content_config", + client=client, + metadata={"content_type": content_type}, + ) - user = request.user.object + content_object = map_config_to_object(content_type) + if content_object is None: + raise ValueError(f"Invalid content type: {content_type}") - content_object = map_config_to_object(content_type) - await adapters.set_text_content_config(user, content_object, updated_config) + await content_object.objects.filter(user=user).adelete() + await sync_to_async(EntryAdapters.delete_all_entries)(user, content_type) - update_telemetry_state( - request=request, - telemetry_type="api", - api="set_content_config", - client=client, - metadata={"content_type": content_type}, - ) + enabled_content = await sync_to_async(EntryAdapters.get_unique_file_types)(user) + return {"status": "ok"} - return {"status": "ok"} - @api.post("/config/data/conversation/model", status_code=200) - @requires(["authenticated"]) - async def update_chat_model( - request: Request, - id: str, - client: Optional[str] = None, - ): - user = request.user.object +@api.delete("/config/data/file", status_code=200) +@requires(["authenticated"]) +async def remove_file_data( + request: Request, + filename: str, + client: Optional[str] = None, +): + user = request.user.object - new_config = await ConversationAdapters.aset_user_conversation_processor(user, int(id)) + update_telemetry_state( + request=request, + telemetry_type="api", + api="delete_file", + client=client, + ) - update_telemetry_state( - request=request, - telemetry_type="api", - api="set_conversation_chat_model", - client=client, - metadata={"processor_conversation_type": "conversation"}, - ) + await EntryAdapters.adelete_entry_by_file(user, filename) - if new_config is None: - return {"status": "error", "message": "Model not found"} + return {"status": "ok"} - return {"status": "ok"} + +@api.get("/config/data/all", response_model=List[str]) +@requires(["authenticated"]) +async def get_all_filenames( + request: Request, + client: Optional[str] = None, +): + user = request.user.object + + update_telemetry_state( + request=request, + telemetry_type="api", + api="get_all_filenames", + client=client, + ) + + return await sync_to_async(list)(EntryAdapters.aget_all_filenames(user)) + + +@api.delete("/config/data/all", status_code=200) +@requires(["authenticated"]) +async def remove_all_config_data( + request: Request, + client: Optional[str] = None, +): + user = request.user.object + + update_telemetry_state( + request=request, + telemetry_type="api", + api="delete_all_config", + client=client, + ) + + await EntryAdapters.adelete_all_entries(user) + + return {"status": "ok"} + + +@api.post("/config/data/conversation/model", status_code=200) +@requires(["authenticated"]) +async def update_chat_model( + request: Request, + id: str, + client: Optional[str] = None, +): + user = request.user.object + + new_config = await ConversationAdapters.aset_user_conversation_processor(user, int(id)) + + update_telemetry_state( + request=request, + telemetry_type="api", + api="set_conversation_chat_model", + client=client, + metadata={"processor_conversation_type": "conversation"}, + ) + + if new_config is None: + return {"status": "error", "message": "Model not found"} + + return {"status": "ok"} # Create Routes @@ -377,6 +418,7 @@ async def search( SearchType.Github, SearchType.Notion, SearchType.Plaintext, + SearchType.Pdf, ]: # query markdown notes search_futures += [ diff --git a/src/khoj/routers/web_client.py b/src/khoj/routers/web_client.py index 1ab6beb7..35603e18 100644 --- a/src/khoj/routers/web_client.py +++ b/src/khoj/routers/web_client.py @@ -38,7 +38,6 @@ def index(request: Request): "chat.html", context={ "request": request, - "demo": state.demo, "username": user.username, "user_photo": user_picture, }, @@ -55,7 +54,6 @@ def index_post(request: Request): "chat.html", context={ "request": request, - "demo": state.demo, "username": user.username, "user_photo": user_picture, }, @@ -72,7 +70,6 @@ def search_page(request: Request): "search.html", context={ "request": request, - "demo": state.demo, "username": user.username, "user_photo": user_picture, }, @@ -89,7 +86,6 @@ def chat_page(request: Request): "chat.html", context={ "request": request, - "demo": state.demo, "username": user.username, "user_photo": user_picture, }, @@ -107,7 +103,6 @@ def login_page(request: Request): "login.html", context={ "request": request, - "demo": state.demo, "google_client_id": google_client_id, "redirect_uri": redirect_uri, }, @@ -125,142 +120,139 @@ def map_config_to_object(content_type: str): return LocalPlaintextConfig -if not state.demo: +@web_client.get("/config", response_class=HTMLResponse) +@requires(["authenticated"], redirect="login_page") +def config_page(request: Request): + user = request.user.object + user_picture = request.session.get("user", {}).get("picture") + enabled_content = set(EntryAdapters.get_unique_file_types(user).all()) - @web_client.get("/config", response_class=HTMLResponse) - @requires(["authenticated"], redirect="login_page") - def config_page(request: Request): - user = request.user.object - user_picture = request.session.get("user", {}).get("picture") - enabled_content = set(EntryAdapters.get_unique_file_types(user).all()) + successfully_configured = { + "pdf": ("pdf" in enabled_content), + "markdown": ("markdown" in enabled_content), + "org": ("org" in enabled_content), + "image": False, + "github": ("github" in enabled_content), + "notion": ("notion" in enabled_content), + "plaintext": ("plaintext" in enabled_content), + } - successfully_configured = { - "pdf": ("pdf" in enabled_content), - "markdown": ("markdown" in enabled_content), - "org": ("org" in enabled_content), - "image": False, - "github": ("github" in enabled_content), - "notion": ("notion" in enabled_content), - "plaintext": ("plaintext" in enabled_content), - } - - if state.content_index: - successfully_configured.update( - { - "image": state.content_index.image is not None, - } - ) - - conversation_options = ConversationAdapters.get_conversation_processor_options().all() - all_conversation_options = list() - for conversation_option in conversation_options: - all_conversation_options.append( - {"chat_model": conversation_option.chat_model, "id": conversation_option.id} - ) - - selected_conversation_config = ConversationAdapters.get_conversation_config(user) - - return templates.TemplateResponse( - "config.html", - context={ - "request": request, - "current_model_state": successfully_configured, - "anonymous_mode": state.anonymous_mode, - "username": user.username if user else None, - "conversation_options": all_conversation_options, - "selected_conversation_config": selected_conversation_config.id - if selected_conversation_config - else None, - "user_photo": user_picture, - }, + if state.content_index: + successfully_configured.update( + { + "image": state.content_index.image is not None, + } ) - @web_client.get("/config/content_type/github", response_class=HTMLResponse) - @requires(["authenticated"], redirect="login_page") - def github_config_page(request: Request): - user = request.user.object - user_picture = request.session.get("user", {}).get("picture") - current_github_config = get_user_github_config(user) + conversation_options = ConversationAdapters.get_conversation_processor_options().all() + all_conversation_options = list() + for conversation_option in conversation_options: + all_conversation_options.append({"chat_model": conversation_option.chat_model, "id": conversation_option.id}) - if current_github_config: - raw_repos = current_github_config.githubrepoconfig.all() - repos = [] - for repo in raw_repos: - repos.append( - GithubRepoConfig( - name=repo.name, - owner=repo.owner, - branch=repo.branch, - ) + selected_conversation_config = ConversationAdapters.get_conversation_config(user) + + return templates.TemplateResponse( + "config.html", + context={ + "request": request, + "current_model_state": successfully_configured, + "anonymous_mode": state.anonymous_mode, + "username": user.username if user else None, + "conversation_options": all_conversation_options, + "selected_conversation_config": selected_conversation_config.id if selected_conversation_config else None, + "user_photo": user_picture, + }, + ) + + +@web_client.get("/config/content_type/github", response_class=HTMLResponse) +@requires(["authenticated"], redirect="login_page") +def github_config_page(request: Request): + user = request.user.object + user_picture = request.session.get("user", {}).get("picture") + current_github_config = get_user_github_config(user) + + if current_github_config: + raw_repos = current_github_config.githubrepoconfig.all() + repos = [] + for repo in raw_repos: + repos.append( + GithubRepoConfig( + name=repo.name, + owner=repo.owner, + branch=repo.branch, ) - current_config = GithubContentConfig( - pat_token=current_github_config.pat_token, - repos=repos, ) - current_config = json.loads(current_config.json()) - else: - current_config = {} # type: ignore - - return templates.TemplateResponse( - "content_type_github_input.html", - context={ - "request": request, - "current_config": current_config, - "username": user.username, - "user_photo": user_picture, - }, - ) - - @web_client.get("/config/content_type/notion", response_class=HTMLResponse) - @requires(["authenticated"], redirect="login_page") - def notion_config_page(request: Request): - user = request.user.object - user_picture = request.session.get("user", {}).get("picture") - current_notion_config = get_user_notion_config(user) - - current_config = NotionContentConfig( - token=current_notion_config.token if current_notion_config else "", - ) - - current_config = json.loads(current_config.json()) - - return templates.TemplateResponse( - "content_type_notion_input.html", - context={ - "request": request, - "current_config": current_config, - "username": user.username, - "user_photo": user_picture, - }, - ) - - @web_client.get("/config/content_type/{content_type}", response_class=HTMLResponse) - @requires(["authenticated"], redirect="login_page") - def content_config_page(request: Request, content_type: str): - if content_type not in VALID_TEXT_CONTENT_TYPES: - return templates.TemplateResponse("config.html", context={"request": request}) - - object = map_config_to_object(content_type) - user = request.user.object - user_picture = request.session.get("user", {}).get("picture") - config = object.objects.filter(user=user).first() - if config == None: - config = object.objects.create(user=user) - - current_config = TextContentConfig( - input_files=config.input_files, - input_filter=config.input_filter, - index_heading_entries=config.index_heading_entries, + current_config = GithubContentConfig( + pat_token=current_github_config.pat_token, + repos=repos, ) current_config = json.loads(current_config.json()) + else: + current_config = {} # type: ignore - return templates.TemplateResponse( - "content_type_input.html", - context={ - "request": request, - "current_config": current_config, - "content_type": content_type, - "username": user.username, - "user_photo": user_picture, - }, - ) + return templates.TemplateResponse( + "content_type_github_input.html", + context={ + "request": request, + "current_config": current_config, + "username": user.username, + "user_photo": user_picture, + }, + ) + + +@web_client.get("/config/content_type/notion", response_class=HTMLResponse) +@requires(["authenticated"], redirect="login_page") +def notion_config_page(request: Request): + user = request.user.object + user_picture = request.session.get("user", {}).get("picture") + current_notion_config = get_user_notion_config(user) + + current_config = NotionContentConfig( + token=current_notion_config.token if current_notion_config else "", + ) + + current_config = json.loads(current_config.json()) + + return templates.TemplateResponse( + "content_type_notion_input.html", + context={ + "request": request, + "current_config": current_config, + "username": user.username, + "user_photo": user_picture, + }, + ) + + +@web_client.get("/config/content_type/{content_type}", response_class=HTMLResponse) +@requires(["authenticated"], redirect="login_page") +def content_config_page(request: Request, content_type: str): + if content_type not in VALID_TEXT_CONTENT_TYPES: + return templates.TemplateResponse("config.html", context={"request": request}) + + object = map_config_to_object(content_type) + user = request.user.object + user_picture = request.session.get("user", {}).get("picture") + config = object.objects.filter(user=user).first() + if config == None: + config = object.objects.create(user=user) + + current_config = TextContentConfig( + input_files=config.input_files, + input_filter=config.input_filter, + index_heading_entries=config.index_heading_entries, + ) + current_config = json.loads(current_config.json()) + + return templates.TemplateResponse( + "content_type_input.html", + context={ + "request": request, + "current_config": current_config, + "content_type": content_type, + "username": user.username, + "user_photo": user_picture, + }, + ) diff --git a/src/khoj/utils/cli.py b/src/khoj/utils/cli.py index c0928d5e..ddf45658 100644 --- a/src/khoj/utils/cli.py +++ b/src/khoj/utils/cli.py @@ -42,7 +42,6 @@ def cli(args=None): parser.add_argument( "--disable-chat-on-gpu", action="store_true", default=False, help="Disable using GPU for the offline chat model" ) - parser.add_argument("--demo", action="store_true", default=False, help="Run Khoj in demo mode") parser.add_argument( "--anonymous-mode", action="store_true", diff --git a/src/khoj/utils/state.py b/src/khoj/utils/state.py index 1b830245..748ca15a 100644 --- a/src/khoj/utils/state.py +++ b/src/khoj/utils/state.py @@ -31,7 +31,6 @@ config_lock = threading.Lock() chat_lock = threading.Lock() SearchType = utils_config.SearchType telemetry: List[Dict[str, str]] = [] -demo: bool = False khoj_version: str = None device = get_device() chat_on_gpu: bool = True diff --git a/tests/data/pdf/ocr_samples.pdf b/tests/data/pdf/ocr_samples.pdf new file mode 100644 index 00000000..100f60e0 Binary files /dev/null and b/tests/data/pdf/ocr_samples.pdf differ diff --git a/tests/test_markdown_to_jsonl.py b/tests/test_markdown_to_entries.py similarity index 100% rename from tests/test_markdown_to_jsonl.py rename to tests/test_markdown_to_entries.py diff --git a/tests/test_org_to_jsonl.py b/tests/test_org_to_entries.py similarity index 100% rename from tests/test_org_to_jsonl.py rename to tests/test_org_to_entries.py diff --git a/tests/test_pdf_to_jsonl.py b/tests/test_pdf_to_entries.py similarity index 84% rename from tests/test_pdf_to_jsonl.py rename to tests/test_pdf_to_entries.py index 81ea18c8..3ab44639 100644 --- a/tests/test_pdf_to_jsonl.py +++ b/tests/test_pdf_to_entries.py @@ -50,6 +50,23 @@ def test_multi_page_pdf_to_jsonl(): assert len(jsonl_data) == 6 +def test_ocr_page_pdf_to_jsonl(): + "Convert multiple pages from single PDF file to jsonl." + # Act + # Extract Entries from specified Pdf files + with open("tests/data/pdf/ocr_samples.pdf", "rb") as f: + pdf_bytes = f.read() + + data = {"tests/data/pdf/ocr_samples.pdf": pdf_bytes} + entries, entry_to_file_map = PdfToEntries.extract_pdf_entries(pdf_files=data) + + # Process Each Entry from All Pdf Files + entries = PdfToEntries.convert_pdf_entries_to_maps(entries, entry_to_file_map) + + assert len(entries) == 1 + assert "playing on a strip of marsh" in entries[0].raw + + def test_get_pdf_files(tmp_path): "Ensure Pdf files specified via input-filter, input-files extracted" # Arrange diff --git a/tests/test_plaintext_to_jsonl.py b/tests/test_plaintext_to_entries.py similarity index 100% rename from tests/test_plaintext_to_jsonl.py rename to tests/test_plaintext_to_entries.py