Use UV to install server for speed, package locks in dev setup, workflows

It's much faster than pip, includes dependency locks via uv.lock and
comes with standard convenience utilities (e.g pipx, venv replacement)

.github/workflows/pre-commit.yml

@@ -35,18 +35,20 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Set up Python 3.11
+      - name: Install uv
-        uses: actions/setup-python@v4
+        uses: astral-sh/setup-uv@v4
         with:
-          python-version: 3.11
+          version: "latest"
+
+      - name: Set up Python 3.11
+        run: uv python install 3.11
 
       - name: ⏬️ Install Dependencies
         run: |
           sudo apt update && sudo apt install -y libegl1
-          python -m pip install --upgrade pip
 
       - name: ⬇️ Install Application
-        run: pip install --no-cache-dir --upgrade .[dev]
+        run: uv sync --all-extras
 
       - name: 🌡️ Validate Application
         run: pre-commit run --hook-stage manual --all

.github/workflows/pypi.yml

@@ -34,13 +34,16 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Set up Python 3.11
+      - name: Install uv
-        uses: actions/setup-python@v4
+        uses: astral-sh/setup-uv@v4
         with:
-          python-version: '3.11.12'
+          version: "latest"
+
+      - name: Set up Python 3.11
+        run: uv python install 3.11
 
       - name: ⬇️ Install Server
-        run: python -m pip install --upgrade pip && pip install --upgrade .
+        run: uv sync --all-extras
 
       - name: ⬇️ Install Web Client
         run: |
@@ -51,7 +54,7 @@ jobs:
       - name: 📂 Copy Generated Files
         run: |
           mkdir -p src/khoj/interface/compiled
-          cp -r /opt/hostedtoolcache/Python/3.11.12/x64/lib/python3.11/site-packages/khoj/interface/compiled/* src/khoj/interface/compiled/
+          cp -r $(uv python find-python 3.11)/lib/python3.11/site-packages/khoj/interface/compiled/* src/khoj/interface/compiled/
 
       - name: ⚙️ Build Python Package
         run: |
@@ -61,13 +64,13 @@ jobs:
           rm -rf dist
 
           # Build PyPI Package
-          pipx run build
+          uv build
 
       - name: 🌡️ Validate Python Package
         run: |
           # Validate PyPi Package
-          pipx run check-wheel-contents dist/*.whl --ignore W004
+          uv tool run check-wheel-contents dist/*.whl --ignore W004
-          pipx run twine check dist/*
+          uv tool run twine check dist/*
 
       - name: ⏫ Upload Python Package Artifacts
         uses: actions/upload-artifact@v4

.github/workflows/run_evals.yml

@@ -106,10 +106,13 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Set up Python
+      - name: Install uv
-        uses: actions/setup-python@v4
+        uses: astral-sh/setup-uv@v4
         with:
-          python-version: '3.10'
+          version: "latest"
+
+      - name: Set up Python
+        run: uv python install 3.10
 
       - name: Get App Version
         id: hatch
@@ -127,16 +130,14 @@ jobs:
           DEBIAN_FRONTEND: noninteractive
         run: |
           # install dependencies
-          sudo apt update && sudo apt install -y git python3-pip libegl1 sqlite3 libsqlite3-dev libsqlite3-0 ffmpeg libsm6 libxext6
+          sudo apt update && sudo apt install -y git libegl1 sqlite3 libsqlite3-dev libsqlite3-0 ffmpeg libsm6 libxext6
-          # upgrade pip
-          python -m ensurepip --upgrade && python -m pip install --upgrade pip
           # install terrarium for code sandbox
           git clone https://github.com/khoj-ai/terrarium.git && cd terrarium && npm install --legacy-peer-deps && mkdir pyodide_cache
 
       - name: ⬇️ Install Application
         run: |
           sed -i 's/dynamic = \["version"\]/version = "${{ steps.hatch.outputs.version }}"/' pyproject.toml
-          pip install --upgrade .[dev]
+          uv sync --all-extras
 
       - name: 📝 Run Eval
         env:

.github/workflows/test.yml

@@ -54,10 +54,13 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Set up Python
+      - name: Install uv
-        uses: actions/setup-python@v4
+        uses: astral-sh/setup-uv@v4
         with:
-          python-version: ${{ matrix.python_version }}
+          version: "latest"
+
+      - name: Set up Python ${{ matrix.python_version }}
+        run: uv python install ${{ matrix.python_version }}
 
       - name: ⏬️ Install Dependencies
         env:
@@ -71,17 +74,11 @@ jobs:
         run : |
           apt install -y postgresql postgresql-client && apt install -y postgresql-server-dev-16
 
-      - name: ⬇️ Install pip
-        run: |
-          apt install -y python3-pip
-          python3 -m ensurepip --upgrade
-          python3 -m pip install --upgrade pip
-
       - name: ⬇️ Install Application
         env:
           PIP_EXTRA_INDEX_URL: "https://download.pytorch.org/whl/cpu https://abetlen.github.io/llama-cpp-python/whl/cpu"
           CUDA_VISIBLE_DEVICES: ""
-        run: sed -i 's/dynamic = \["version"\]/version = "0.0.0"/' pyproject.toml && pip install --break-system-packages --upgrade .[dev]
+        run: sed -i 's/dynamic = \["version"\]/version = "0.0.0"/' pyproject.toml && uv sync --all-extras
 
       - name: 🧪 Test Application
         env:

documentation/docs/contributing/development.mdx

@@ -30,7 +30,7 @@ git clone https://github.com/khoj-ai/khoj && cd khoj
 python3 -m venv .venv && source .venv/bin/activate
 
 # For MacOS or zsh users run this
-pip install -e '.[dev]'
+uv sync --all-extras
     ```
     </TabItem>
     <TabItem value="windows" label="Windows">
@@ -42,7 +42,7 @@ git clone https://github.com/khoj-ai/khoj && cd khoj
 python3 -m venv .venv && .venv\Scripts\activate
 
 # Install Khoj for Development
-pip install -e '.[dev]'
+uv sync --all-extras
       ```
     </TabItem>
     <TabItem value="linux" label="Linux">
@@ -54,7 +54,7 @@ git clone https://github.com/khoj-ai/khoj && cd khoj
 python3 -m venv .venv && source .venv/bin/activate
 
 # Install Khoj for Development
-pip install -e '.[dev]'
+uv sync --all-extras
       ```
     </TabItem>
   </Tabs>
@@ -129,7 +129,7 @@ Always run `yarn export` to test your front-end changes on http://localhost:4211
     - Try reactivating the virtual environment and rerunning the `khoj` command.
     - If it still doesn't work repeat the installation process.
 2. Python Package Missing
-    - Use `pip install xxx` and try running the `khoj` command.
+    - Use `uv add xxx` and try running the `khoj` command.
 3. Command `createdb` Not Recognized
     - make sure path to postgres binaries is included in environment variables. It usually looks something like
     ```

scripts/dev_setup.sh

@@ -29,7 +29,13 @@ if [ "$DEVCONTAINER" = true ]; then
     # PATH should already include /opt/venv/bin from Dockerfile
 
     # Install khoj in editable mode (dependencies already installed)
-    python3 -m pip install -e '.[dev]'
+    # Use uv if available, else fall back to pipx
+    if command -v uv &> /dev/null
+    then
+        uv sync --all-extras
+    else
+        python3 -m pip install -e '.[dev]'
+    fi
 
     # Install Web App using cached dependencies
     echo "Installing Web App using cached dependencies..."
@@ -39,7 +45,14 @@ else
     # Standard setup
     echo "Installing Server App..."
     cd "$PROJECT_ROOT"
-    python3 -m venv .venv && . .venv/bin/activate && python3 -m pip install -e '.[dev]'
+    if command -v uv &> /dev/null
+    then
+        uv venv
+        uv sync --all-extras
+    else
+        python3 -m venv .venv && . .venv/bin/activate
+        python3 -m pip install -e '.[dev]'
+    fi
 
     echo "Installing Web App..."
     cd "$PROJECT_ROOT/src/interface/web"

src/khoj/app/README.md

@@ -65,7 +65,7 @@ sudo -u postgres createdb khoj
 ### Install Khoj
 
 ```bash
-pip install -e '.[dev]'
+uv sync --all-extras
 ```
 
 ### Make Khoj DB migrations