From a353d883a08dd7ed7d6eec6c8306ec0872598294 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Fri, 5 Jul 2024 16:07:42 +0530 Subject: [PATCH 1/5] Make it optional to set the encoder, cross-encoder configs via admin UI Upgrade sentence-transformer, add einops dependency for some sentence transformer models like nomic --- pyproject.toml | 3 ++- src/khoj/database/models/__init__.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b3b4af35..de82586f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,8 @@ dependencies = [ "pyyaml ~= 6.0", "rich >= 13.3.1", "schedule == 1.1.0", - "sentence-transformers == 2.5.1", + "sentence-transformers == 3.0.1", + "einops == 0.8.0", "transformers >= 4.28.0", "torch == 2.2.2", "uvicorn == 0.17.6", diff --git a/src/khoj/database/models/__init__.py b/src/khoj/database/models/__init__.py index 62afdd2b..096d14bc 100644 --- a/src/khoj/database/models/__init__.py +++ b/src/khoj/database/models/__init__.py @@ -215,11 +215,11 @@ class SearchModelConfig(BaseModel): # Bi-encoder model of sentence-transformer type to load from HuggingFace bi_encoder = models.CharField(max_length=200, default="thenlper/gte-small") # Config passed to the sentence-transformer model constructor. E.g. device="cuda:0", trust_remote_server=True etc. - bi_encoder_model_config = models.JSONField(default=dict) + bi_encoder_model_config = models.JSONField(default=dict, blank=True) # Query encode configs like prompt, precision, normalize_embeddings, etc. for sentence-transformer models - bi_encoder_query_encode_config = models.JSONField(default=dict) + bi_encoder_query_encode_config = models.JSONField(default=dict, blank=True) # Docs encode configs like prompt, precision, normalize_embeddings, etc. for sentence-transformer models - bi_encoder_docs_encode_config = models.JSONField(default=dict) + bi_encoder_docs_encode_config = models.JSONField(default=dict, blank=True) # Cross-encoder model of sentence-transformer type to load from HuggingFace cross_encoder = models.CharField(max_length=200, default="mixedbread-ai/mxbai-rerank-xsmall-v1") # Inference server API endpoint to use for embeddings inference. Bi-encoder model should be hosted on this server From 69c9e8cc0859d6d66350e099d5477db1fffc4ae1 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Sun, 23 Jun 2024 16:16:53 +0530 Subject: [PATCH 2/5] Disable CSP in Khoj Obsidian as it interferes with Obsidian functionality The Khoj CSP interferes with other Obsidian features and plugins as CSP is applied page wide. For now chat message sanitization via Dompurify should suffice. Enable CSP when can scope it to only the Khoj Obsidian plugin. --- src/interface/obsidian/src/chat_view.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/interface/obsidian/src/chat_view.ts b/src/interface/obsidian/src/chat_view.ts index 0e91d4b9..d6b7f76f 100644 --- a/src/interface/obsidian/src/chat_view.ts +++ b/src/interface/obsidian/src/chat_view.ts @@ -96,8 +96,9 @@ export class KhojChatView extends KhojPaneView { const objectSrc = `object-src 'none';`; const csp = `${defaultSrc} ${scriptSrc} ${connectSrc} ${styleSrc} ${imgSrc} ${childSrc} ${objectSrc}`; - // Add CSP meta tag to the Khoj Chat modal - document.head.createEl("meta", { attr: { "http-equiv": "Content-Security-Policy", "content": `${csp}` } }); + // WARNING: CSP DISABLED for now as it breaks other Obsidian plugins. Enable when can scope CSP to only Khoj plugin. + // CSP meta tag for the Khoj Chat modal + // document.head.createEl("meta", { attr: { "http-equiv": "Content-Security-Policy", "content": `${csp}` } }); // Create area for chat logs let chatBodyEl = contentEl.createDiv({ attr: { id: "khoj-chat-body", class: "khoj-chat-body" } }); From 2f034f807a7e78f9b5d219b5f960a0fe88b6d283 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Sun, 23 Jun 2024 22:57:56 +0530 Subject: [PATCH 3/5] Construct config page elements on Web via DOM scripting. Minimize isage of innerHTML to prevent DOM clobbering and unintended escape by user Input --- src/khoj/interface/web/agents.html | 31 ++-- src/khoj/interface/web/chat.html | 44 +++-- src/khoj/interface/web/config.html | 160 ++++++++++++------ .../web/content_source_computer_input.html | 9 +- .../web/content_source_github_input.html | 78 ++++++--- .../web/content_source_notion_input.html | 20 +-- .../interface/web/public_conversation.html | 13 +- 7 files changed, 229 insertions(+), 126 deletions(-) diff --git a/src/khoj/interface/web/agents.html b/src/khoj/interface/web/agents.html index 9b2793c8..b8ff8dae 100644 --- a/src/khoj/interface/web/agents.html +++ b/src/khoj/interface/web/agents.html @@ -242,18 +242,25 @@ - + src="https://assets.khoj.dev/intl-tel-input/intlTelInput.min.js"> + From 9bdb48807be68bb4f5cbc87205faea3aa3fabd67 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Sat, 6 Jul 2024 12:53:19 +0530 Subject: [PATCH 5/5] Ratelimit text to speech model. Validate share chat url domain - Do not log auth error message on server when Resend setup as Magic links for sign-in are now supported --- src/khoj/routers/api_chat.py | 25 +++++++++++++++++++------ src/khoj/routers/auth.py | 8 ++++++-- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/khoj/routers/api_chat.py b/src/khoj/routers/api_chat.py index a42838d9..be28622b 100644 --- a/src/khoj/routers/api_chat.py +++ b/src/khoj/routers/api_chat.py @@ -13,6 +13,7 @@ from starlette.authentication import requires from starlette.websockets import WebSocketDisconnect from websockets import ConnectionClosedOK +from khoj.app.settings import ALLOWED_HOSTS from khoj.database.adapters import ( ConversationAdapters, DataStoreAdapters, @@ -189,7 +190,17 @@ async def sendfeedback(request: Request, data: FeedbackData): @api_chat.post("/speech") @requires(["authenticated", "premium"]) -async def text_to_speech(request: Request, common: CommonQueryParams, text: str): +async def text_to_speech( + request: Request, + common: CommonQueryParams, + text: str, + rate_limiter_per_minute=Depends( + ApiUserRateLimiter(requests=5, subscribed_requests=20, window=60, slug="chat_minute") + ), + rate_limiter_per_day=Depends( + ApiUserRateLimiter(requests=5, subscribed_requests=300, window=60 * 60 * 24, slug="chat_day") + ), +) -> Response: voice_model = await ConversationAdapters.aget_voice_model_config(request.user.object) params = {"text_to_speak": text} @@ -386,17 +397,19 @@ def duplicate_chat_history_public_conversation( conversation_id: int, ): user = request.user.object + domain = request.headers.get("host") + scheme = request.url.scheme + + # Throw unauthorized exception if domain not in ALLOWED_HOSTS + host_domain = domain.split(":")[0] + if host_domain not in ALLOWED_HOSTS: + raise HTTPException(status_code=401, detail="Unauthorized domain") # Duplicate Conversation History to Public Conversation conversation = ConversationAdapters.get_conversation_by_user(user, request.user.client_app, conversation_id) - public_conversation = ConversationAdapters.make_public_conversation_copy(conversation) - public_conversation_url = PublicConversationAdapters.get_public_conversation_url(public_conversation) - domain = request.headers.get("host") - scheme = request.url.scheme - update_telemetry_state( request=request, telemetry_type="api", diff --git a/src/khoj/routers/auth.py b/src/khoj/routers/auth.py index 8249d66e..e7d28301 100644 --- a/src/khoj/routers/auth.py +++ b/src/khoj/routers/auth.py @@ -42,8 +42,12 @@ if not state.anonymous_mode: from google.oauth2 import id_token except ImportError: missing_requirements += ["Install the Khoj production package with `pip install khoj-assistant[prod]`"] - if not os.environ.get("GOOGLE_CLIENT_ID") or not os.environ.get("GOOGLE_CLIENT_SECRET"): - missing_requirements += ["Set your GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET as environment variables"] + if not os.environ.get("RESEND_API_KEY") and ( + not os.environ.get("GOOGLE_CLIENT_ID") or not os.environ.get("GOOGLE_CLIENT_SECRET") + ): + missing_requirements += [ + "Set your RESEND_API_KEY or GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET as environment variables" + ] if missing_requirements: requirements_string = "\n - " + "\n - ".join(missing_requirements) error_msg = f"🚨 Start Khoj with --anonymous-mode flag or to enable authentication:{requirements_string}"