Make Production Dependencies for Khoj Cloud Optional to Install (#647)

- Remove unused git dependency from Docker images
- Move python packages used for test into dev dependency group
- Only enable API token, Whatsapp cards on Web UI when Stripe, Twilio setup
- Move production dependencies to prod python packages group
- Fix docs links in Khoj welcome chat message

Dockerfile

@@ -3,7 +3,7 @@ FROM ubuntu:jammy
 LABEL org.opencontainers.image.source https://github.com/khoj-ai/khoj
 
 # Install System Dependencies
-RUN apt update -y && apt -y install python3-pip git swig
+RUN apt update -y && apt -y install python3-pip swig
 
 WORKDIR /app
 

prod.Dockerfile

@@ -4,7 +4,9 @@ FROM nvidia/cuda:12.2.0-devel-ubuntu22.04
 LABEL org.opencontainers.image.source https://github.com/khoj-ai/khoj
 
 # Install System Dependencies
-RUN apt update -y && apt -y install python3-pip git libsqlite3-0 ffmpeg libsm6 libxext6
+RUN apt update -y && apt -y install python3-pip libsqlite3-0 ffmpeg libsm6 libxext6
+# Install Optional Dependencies
+RUN apt install vim -y
 
 WORKDIR /app
 
@@ -13,13 +15,11 @@ COPY pyproject.toml .
 COPY README.md .
 ARG VERSION=0.0.0
 RUN sed -i "s/dynamic = \\[\"version\"\\]/version = \"$VERSION\"/" pyproject.toml && \
-    TMPDIR=/home/cache/ pip install --cache-dir=/home/cache/ -e .
+    TMPDIR=/home/cache/ pip install --cache-dir=/home/cache/ -e .[prod]
 
 # Copy Source Code
 COPY . .
 
-RUN apt install vim -y
-
 # Set the PYTHONPATH environment variable in order for it to find the Django app.
 ENV PYTHONPATH=/app/src:$PYTHONPATH
 

pyproject.toml

@@ -40,7 +40,7 @@ dependencies = [
     "dateparser >= 1.1.1",
     "defusedxml == 0.7.1",
     "fastapi >= 0.104.1",
-    "python-multipart >= 0.0.5",
+    "python-multipart >= 0.0.7",
     "jinja2 == 3.1.3",
     "openai >= 1.0.0",
     "tiktoken >= 0.3.2",
@@ -50,7 +50,7 @@ dependencies = [
     "pyyaml == 6.0",
     "rich >= 13.3.1",
     "schedule == 1.1.0",
-    "sentence-transformers == 2.2.2",
+    "sentence-transformers == 2.3.1",
     "transformers >= 4.28.0",
     "torch == 2.0.1",
     "uvicorn == 0.17.6",
@@ -61,7 +61,7 @@ dependencies = [
     "bs4 >= 0.0.1",
     "anyio == 3.7.1",
     "pymupdf >= 1.23.5",
-    "django == 4.2.7",
+    "django == 4.2.10",
     "authlib == 1.2.1",
     "gpt4all == 2.1.0; platform_system == 'Linux' and platform_machine == 'x86_64'",
     "gpt4all == 2.1.0; platform_system == 'Windows' or platform_system == 'Darwin'",
@@ -69,17 +69,13 @@ dependencies = [
     "httpx == 0.25.0",
     "pgvector == 0.2.4",
     "psycopg2-binary == 2.9.9",
-    "google-auth == 2.23.3",
-    "python-multipart == 0.0.6",
     "gunicorn == 21.2.0",
     "lxml == 4.9.3",
     "tzdata == 2023.3",
-    "rapidocr-onnxruntime == 1.3.8",
-    "stripe == 7.3.0",
+    "rapidocr-onnxruntime == 1.3.11",
     "openai-whisper >= 20231117",
     "django-phonenumber-field == 7.3.0",
     "phonenumbers == 8.13.27",
-    "twilio == 8.11"
 ]
 dynamic = ["version"]
 
@@ -93,21 +89,23 @@ Releases = "https://github.com/khoj-ai/khoj/releases"
 khoj = "khoj.main:run"
 
 [project.optional-dependencies]
-test = [
-    "pytest >= 7.1.2",
-    "freezegun >= 1.2.0",
-    "factory-boy >= 3.2.1",
-    "trio >= 0.22.0",
-    "pytest-xdist",
-    "psutil >= 5.8.0",
+prod = [
+    "google-auth == 2.23.3",
+    "stripe == 7.3.0",
+    "twilio == 8.11",
 ]
 dev = [
-    "khoj-assistant[test]",
+    "khoj-assistant[prod]",
+    "pytest >= 7.1.2",
+    "pytest-xdist[psutil]",
+    "pytest-django == 4.5.2",
+    "pytest-asyncio == 0.21.1",
+    "freezegun >= 1.2.0",
+    "factory-boy >= 3.2.1",
+    "psutil >= 5.8.0",
     "mypy >= 1.0.1",
     "black >= 23.1.0",
     "pre-commit >= 3.0.4",
-    "pytest-django == 4.5.2",
-    "pytest-asyncio == 0.21.1",
 ]
 
 [tool.hatch.version]

src/khoj/configure.py

@@ -73,6 +73,7 @@ class UserAuthenticationBackend(AuthenticationBackend):
             Subscription.objects.create(user=default_user, type="standard", renewal_date=renewal_date)
 
     async def authenticate(self, request: HTTPConnection):
+        # Request from Web client
         current_user = request.session.get("user")
         if current_user and current_user.get("email"):
             user = (
@@ -93,6 +94,8 @@ class UserAuthenticationBackend(AuthenticationBackend):
                 if subscribed:
                     return AuthCredentials(["authenticated", "premium"]), AuthenticatedKhojUser(user)
                 return AuthCredentials(["authenticated"]), AuthenticatedKhojUser(user)
+
+        # Request from Desktop, Emacs, Obsidian clients
         if len(request.headers.get("Authorization", "").split("Bearer ")) == 2:
             # Get bearer token from header
             bearer_token = request.headers["Authorization"].split("Bearer ")[1]
@@ -116,7 +119,8 @@ class UserAuthenticationBackend(AuthenticationBackend):
                 if subscribed:
                     return AuthCredentials(["authenticated", "premium"]), AuthenticatedKhojUser(user_with_token.user)
                 return AuthCredentials(["authenticated"]), AuthenticatedKhojUser(user_with_token.user)
-        # Get query params for client_id and client_secret
+
+        # Request from Whatsapp client
         client_id = request.query_params.get("client_id")
         if client_id:
             # Get the client secret, which is passed in the Authorization header
@@ -163,6 +167,8 @@ class UserAuthenticationBackend(AuthenticationBackend):
                     AuthenticatedKhojUser(user, client_application),
                 )
             return AuthCredentials(["authenticated"]), AuthenticatedKhojUser(user, client_application)
+
+        # No auth required if server in anonymous mode
         if state.anonymous_mode:
             user = await self.khojuser_manager.filter(username="default").prefetch_related("subscription").afirst()
             if user:
@@ -258,28 +264,32 @@ def configure_routes(app):
     from khoj.routers.api import api
     from khoj.routers.api_chat import api_chat
     from khoj.routers.api_config import api_config
-    from khoj.routers.auth import auth_router
     from khoj.routers.indexer import indexer
     from khoj.routers.web_client import web_client
 
     app.include_router(api, prefix="/api")
+    app.include_router(api_chat, prefix="/api/chat")
     app.include_router(api_config, prefix="/api/config")
     app.include_router(indexer, prefix="/api/v1/index")
     app.include_router(web_client)
-    app.include_router(auth_router, prefix="/auth")
-    app.include_router(api_chat, prefix="/api/chat")
+
+    if not state.anonymous_mode:
+        from khoj.routers.auth import auth_router
+
+        app.include_router(auth_router, prefix="/auth")
+        logger.info("🔑 Enabled Authentication")
 
     if state.billing_enabled:
         from khoj.routers.subscription import subscription_router
 
-        logger.info("💳 Enabled Billing")
         app.include_router(subscription_router, prefix="/api/subscription")
+        logger.info("💳 Enabled Billing")
 
     if is_twilio_enabled():
-        logger.info("📞 Enabled Twilio")
         from khoj.routers.api_phone import api_phone
 
         app.include_router(api_phone, prefix="/api/config/phone")
+        logger.info("📞 Enabled Twilio")
 
 
 def configure_middleware(app):

src/khoj/interface/web/chat.html

@@ -16,11 +16,11 @@ Hi, I am Khoj, your open, personal AI 👋🏽. I can help:
 - 🧠 Answer general knowledge questions
 - 💡 Be a sounding board for your ideas
 - 📜 Chat with your notes & documents
-- 🌄 Generate images based on your messages (start your prompt with "/image")
-- 🔎 Search the web for answers to your questions (start your prompt with "/online")
+- 🌄 Generate images based on your messages
+- 🔎 Search the web for answers to your questions
 - 🎙️ Listen to your audio messages (use the mic by the input box to speak your message)
 
-Get the Khoj [Desktop](https://khoj.dev/downloads), [Obsidian](https://docs.khoj.dev/#/obsidian?id=setup) or [Emacs](https://docs.khoj.dev/#/emacs?id=setup) app to search, chat with your 🖥️ computer docs.
+Get the Khoj [Desktop](https://khoj.dev/downloads), [Obsidian](https://docs.khoj.dev/clients/obsidian#setup), [Emacs](https://docs.khoj.dev/clients/emacs#setup) apps to search, chat with your 🖥️ computer docs.
 
 To get started, just start typing below. You can also type / to see a list of commands.
 `.trim()

src/khoj/interface/web/config.html

@@ -187,8 +187,10 @@
             </div>
         </div>
     </div>
+    {% if not anonymous_mode or is_twilio_enabled %}
     <div id="clients" class="section">
         <h2 class="section-title">Clients</h2>
+        {% if not anonymous_mode %}
         <div id="clients-api" class="api-settings">
             <div class="card-title-row">
                 <img class="card-icon" src="/static/assets/icons/key.svg" alt="API Key">
@@ -213,6 +215,8 @@
                 </button>
             </div>
         </div>
+        {% endif %}
+        {% if is_twilio_enabled %}
         <div id="phone-number-input-card" class="api-settings">
             <div class="card-title-row">
                 <img class="card-icon" src="/static/assets/icons/whatsapp.svg" alt="WhatsApp icon">
@@ -244,7 +248,9 @@
                 </button>
             </div>
         </div>
+        {% endif %}
     </div>
+    {% endif %}
     {% if billing_enabled %}
     <div id="billing" class="section">
         <h2 class="section-title">Billing</h2>

src/khoj/routers/auth.py

@@ -2,10 +2,7 @@ import logging
 import os
 from typing import Optional
 
-from authlib.integrations.starlette_client import OAuth, OAuthError
 from fastapi import APIRouter
-from google.auth.transport import requests as google_requests
-from google.oauth2 import id_token
 from starlette.authentication import requires
 from starlette.config import Config
 from starlette.requests import Request
@@ -17,7 +14,6 @@ from khoj.database.adapters import (
     get_khoj_tokens,
     get_or_create_user,
 )
-from khoj.database.models import KhojApiUser
 from khoj.routers.helpers import update_telemetry_state
 from khoj.utils import state
 
@@ -25,11 +21,23 @@ logger = logging.getLogger(__name__)
 
 auth_router = APIRouter()
 
-if not state.anonymous_mode and not (os.environ.get("GOOGLE_CLIENT_ID") and os.environ.get("GOOGLE_CLIENT_SECRET")):
-    logger.warning(
-        "🚨 Use --anonymous-mode flag to disable Google OAuth or set GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET environment variables to enable it"
-    )
-else:
+
+if not state.anonymous_mode:
+    missing_requirements = []
+    from authlib.integrations.starlette_client import OAuth, OAuthError
+
+    try:
+        from google.auth.transport import requests as google_requests
+        from google.oauth2 import id_token
+    except ImportError:
+        missing_requirements += ["Install the Khoj production package with `pip install khoj-assistant[prod]`"]
+    if not os.environ.get("GOOGLE_CLIENT_ID") or not os.environ.get("GOOGLE_CLIENT_SECRET"):
+        missing_requirements += ["Set your GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET as environment variables"]
+    if missing_requirements:
+        requirements_string = "\n   - " + "\n   - ".join(missing_requirements)
+        error_msg = f"🚨 Start Khoj with --anonymous-mode flag or to enable authentication:{requirements_string}"
+        logger.error(error_msg)
+
     config = Config(environ=os.environ)
 
     oauth = OAuth(config)

src/khoj/routers/subscription.py

@@ -2,16 +2,18 @@ import logging
 import os
 from datetime import datetime, timezone
 
-import stripe
 from asgiref.sync import sync_to_async
 from fastapi import APIRouter, Request
-from fastapi.responses import Response
 from starlette.authentication import requires
 
 from khoj.database import adapters
+from khoj.utils import state
 
 # Stripe integration for Khoj Cloud Subscription
-stripe.api_key = os.getenv("STRIPE_API_KEY")
+if state.billing_enabled:
+    import stripe
+
+    stripe.api_key = os.getenv("STRIPE_API_KEY")
 endpoint_secret = os.getenv("STRIPE_SIGNING_SECRET")
 logger = logging.getLogger(__name__)
 subscription_router = APIRouter()

src/khoj/routers/twilio.py

@@ -1,8 +1,6 @@
 import logging
 import os
 
-from twilio.rest import Client
-
 from khoj.database.models import KhojUser
 
 logger = logging.getLogger(__name__)
@@ -13,6 +11,8 @@ verification_service_sid = os.getenv("TWILIO_VERIFICATION_SID")
 
 twilio_enabled = account_sid is not None and auth_token is not None and verification_service_sid is not None
 if twilio_enabled:
+    from twilio.rest import Client
+
     client = Client(account_sid, auth_token)
 
 

src/khoj/routers/web_client.py

@@ -180,8 +180,8 @@ def config_page(request: Request):
             "khoj_cloud_subscription_url": os.getenv("KHOJ_CLOUD_SUBSCRIPTION_URL"),
             "is_active": has_required_scope(request, ["premium"]),
             "has_documents": has_documents,
-            "phone_number": user.phone_number,
             "is_twilio_enabled": is_twilio_enabled(),
+            "phone_number": user.phone_number,
             "is_phone_number_verified": user.verified_phone_number,
         },
     )