From 50f37d541a72a5e60af48f4da43caadb7c20bcf6 Mon Sep 17 00:00:00 2001 From: Debanjum Date: Tue, 3 Jun 2025 01:23:19 -0700 Subject: [PATCH] Pre-install server deps for fast devcontainer start. Fix dev launch.json There seems to be a more standard mechanism of specifying launch.json params for devcontainers. Previous mechanism to write launch.json to .vscode/launch.json in post creation step does not work. Improve default launch.json to include khoj admin username, password with placeholder values to get started with local development faster. Define dockerfile for devcontainer to pre-built server, web app dependencies during dev container image creation stage. So install on dev container startup is sped up as no need to install dependencies. --- .devcontainer/Dockerfile | 36 +++++++++++++++ .devcontainer/devcontainer.json | 40 +++++++++++++++-- .vscode/launch.json | 4 +- scripts/dev_setup.sh | 77 +-------------------------------- 4 files changed, 78 insertions(+), 79 deletions(-) create mode 100644 .devcontainer/Dockerfile diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000..d43ae132 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,36 @@ +ARG PYTHON_VERSION=3.10 +FROM mcr.microsoft.com/devcontainers/python:${PYTHON_VERSION} + +WORKDIR /workspace + +# --- Python Server App Dependencies --- +# Create Python virtual environment +RUN python3 -m venv .venv +# Add venv to PATH for subsequent RUN commands and for the container environment +ENV PATH="/workspace/.venv/bin:${PATH}" + +# Copy files required for Python dependency installation. +COPY pyproject.toml README.md ./ + +# Setup python environment + # Use the pre-built llama-cpp-python, torch cpu wheel +ENV PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu https://abetlen.github.io/llama-cpp-python/whl/cpu" \ + # Avoid downloading unused cuda specific python packages + CUDA_VISIBLE_DEVICES="" \ + # Use static version to build app without git dependency + VERSION=0.0.0 +# Install Python dependencies from pyproject.toml in editable mode +RUN sed -i "s/dynamic = \\[\"version\"\\]/version = \"$VERSION\"/" pyproject.toml && \ + pip install --no-cache-dir ".[dev]" + +# --- Web App Dependencies --- +# Copy web app manifest files +COPY src/interface/web/package.json src/interface/web/yarn.lock ./src/interface/web/ + +# Install web app dependencies +# note: node and yarn will be available from the "features" in devcontainer.json +RUN yarn install --cwd ./src/interface/web && yarn export --cwd ./src/interface/web + +# The .venv and node_modules are now populated in the image. +# The rest of the source code will be mounted by VS Code from your local checkout, +# overlaying any files copied here if they are part of the workspace mount. diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index acc2ca7d..da775470 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,12 @@ { "name": "Khoj Development Environment", - "image": "mcr.microsoft.com/devcontainers/python:3.10", + "build": { + "dockerfile": "Dockerfile", + "context": "..", // Build context is the project root + "args": { + "PYTHON_VERSION": "3.10" + } + }, "forwardPorts": [ 42110 ], @@ -40,11 +46,39 @@ "tests" ], "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true + "python.testing.pytestEnabled": true, + "launch": { + "configurations": [ + { + "name": "Launch Khoj", + "type": "debugpy", + "request": "launch", + "module": "src.khoj.main", + "console": "integratedTerminal", + "justMyCode": false, + "sudo": true, + "args": [ + "-v", + "--anonymous-mode", + "--non-interactive", + "--port=42110" + ], + "envFile": "${workspaceFolder}/.env", + "env": { + "KHOJ_ADMIN_EMAIL": "admin", + "KHOJ_ADMIN_PASSWORD": "admin", + "USE_EMBEDDED_DB": "true", + "KHOJ_DEBUG": "true", + "KHOJ_TELEMETRY_DISABLED": "true", + "PROMPTRACE_DIR": "${workspaceFolder}/promptrace" + } + } + ] + } } } }, - "postCreateCommand": "scripts/dev_setup.sh --devcontainer", + "postCreateCommand": "scripts/dev_setup.sh", "features": { "ghcr.io/devcontainers-contrib/features/black:2": {}, "ghcr.io/devcontainers-contrib/features/mypy:2": {}, diff --git a/.vscode/launch.json b/.vscode/launch.json index b8ae078c..53f6c5fe 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -20,9 +20,11 @@ ], "envFile": "${workspaceFolder}/.env", "env": { + "KHOJ_ADMIN_EMAIL": "admin", + "KHOJ_ADMIN_PASSWORD": "admin", + "USE_EMBEDDED_DB": "true", "KHOJ_DEBUG": "true", "KHOJ_TELEMETRY_DISABLED": "true", - "USE_EMBEDDED_DB": "true", // Configure Code Sandbox // "KHOJ_TERRARIUM_URL": "http://localhost:8080", // Enable Promptracer to debug prompt flows diff --git a/scripts/dev_setup.sh b/scripts/dev_setup.sh index 3de55d1d..71a2db6e 100755 --- a/scripts/dev_setup.sh +++ b/scripts/dev_setup.sh @@ -2,19 +2,14 @@ # --- PROJECT_ROOT=$(git rev-parse --show-toplevel) -# Default is a minimal, local installation unless config flags are set. +# Default to minimal installation unless --full flag passed INSTALL_FULL=false -DEVCONTAINER=false for arg in "$@" do if [ "$arg" == "--full" ] then INSTALL_FULL=true fi - if [ "$arg" == "--devcontainer" ] - then - DEVCONTAINER=true - fi done # Install Server App @@ -28,7 +23,7 @@ python3 -m venv .venv && . .venv/bin/activate && python3 -m pip install -e '.[de # --- echo "Installing Web App..." cd $PROJECT_ROOT/src/interface/web -yarn install +yarn install && yarn export # Install Obsidian App # --- @@ -46,74 +41,6 @@ if [ "$INSTALL_FULL" = true ] ; then yarn install fi -# Create .vscode/launch.json if it doesn't exist -# --- -VSCODE_DIR="$PROJECT_ROOT/.vscode" -LAUNCH_JSON_PATH="$VSCODE_DIR/launch.json" - -# Overwrite project vscode launch.json in devcontainer -# --- -# Differs from default launch.json with sudo true. -# Required by pgserver in github codespaces, not in local dev. -mkdir -p "$VSCODE_DIR" -if [ "$DEVCONTAINER" = true ]; then - echo "Creating $LAUNCH_JSON_PATH..." - cat << EOF > "$LAUNCH_JSON_PATH" -{ - "version": "0.2.0", - "configurations": [ - { - "name": "Launch Khoj", - "type": "debugpy", - "request": "launch", - "module": "src.khoj.main", - "justMyCode": false, - "console": "integratedTerminal", - "sudo": true, - "args": [ - "-v", - "--anonymous-mode", - "--non-interactive", - "--port=42110", - ], - // You can load environment variables via a .env file, the env field below or Github Codespace secrets. - "envFile": "\${workspaceFolder}/.env", - "env": { - "KHOJ_ADMIN_EMAIL": "admin", - "KHOJ_ADMIN_PASSWORD": "admin", - "USE_EMBEDDED_DB": "True", - "KHOJ_DEBUG": "True", - "KHOJ_TELEMETRY_DISABLE": "True", - // Set LLM Provider API keys - // --- - // "GEMINI_API_KEY": "", - // "ANTHROPIC_API_KEY": "", - // "OPENAI_API_KEY": "", - // "OPENAI_BASE_URL": "http://localhost:11434/v1/", - // "KHOJ_DEFAULT_CHAT_MODEL": "claude-sonnet-4-0", - // Set Search Provider API keys - // --- - // "SERPER_DEV_API_KEY": "", - // "OLOSTEP_API_KEY": "", - // "FIRECRAWL_API_KEY": "", - // "JINA_API_KEY": "", - // Enable Khoj Operator - // --- - // "KHOJ_OPERATOR_ENABLED": "True", - // Configure Code Sandbox - // --- - // "KHOJ_TERRARIUM_URL": "http://localhost:8080", - // "E2B_API_KEY": "", - // Enable Promptracer to debug prompt flows - // --- - // "PROMPTRACE_DIR": "\${workspaceFolder}/promptrace", - } - }, - ] -} -EOF -fi - # Install pre-commit hooks # ---- echo "Installing pre-commit hooks..."