diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
index 641bf68b..58d20853 100644
--- a/.devcontainer/Dockerfile
+++ b/.devcontainer/Dockerfile
@@ -1,40 +1,49 @@
-ARG PYTHON_VERSION=3.10
+ARG PYTHON_VERSION=3.12
FROM mcr.microsoft.com/devcontainers/python:${PYTHON_VERSION}
-# Install Node.js and Yarn
-RUN curl -fsSL https://deb.nodesource.com/setup_lts.x | bash - && \
- apt-get install -y nodejs
+# Install UV and Bun
+RUN curl -fsSL https://bun.sh/install | bash && mv /root/.bun/bin/bun /usr/local/bin/bun
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+RUN uv python pin $PYTHON_VERSION
+# create python virtual environment
+RUN uv venv /opt/venv --python $PYTHON_VERSION --seed
+# Add venv to PATH for subsequent RUN commands and for the container environment
+ENV PATH="/opt/venv/bin:$PATH"
+# Tell pip, uv to use this virtual environment
+ENV VIRTUAL_ENV="/opt/venv"
+ENV UV_PROJECT_ENVIRONMENT="/opt/venv"
# Setup working directory
-WORKDIR /workspace
+WORKDIR /workspaces/khoj
# --- Python Server App Dependencies ---
-# Create Python virtual environment
-RUN python3 -m venv /opt/venv
-# Add venv to PATH for subsequent RUN commands and for the container environment
-ENV PATH="/opt/venv/bin:${PATH}"
-
# Copy files required for Python dependency installation.
COPY pyproject.toml README.md ./
# Setup python environment
# Use the pre-built torch cpu wheel
-ENV PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu" \
+ENV UV_INDEX="https://download.pytorch.org/whl/cpu" \
+ UV_INDEX_STRATEGY="unsafe-best-match" \
# Avoid downloading unused cuda specific python packages
CUDA_VISIBLE_DEVICES="" \
# Use static version to build app without git dependency
- VERSION=0.0.0
+ VERSION=0.0.0 \
+ # Use embedded db
+ USE_EMBEDDED_DB="True" \
+ PGSERVER_DATA_DIR="/opt/khoj_db"
# Install Python dependencies from pyproject.toml in editable mode
RUN sed -i "s/dynamic = \\[\"version\"\\]/version = \"$VERSION\"/" pyproject.toml && \
- pip install --no-cache-dir ".[dev]"
+ uv sync --all-extras && \
+ # Save the lock file generated with correct Linux platform wheels
+ cp uv.lock /opt/uv.lock.linux && \
+ chown -R vscode:vscode /opt/venv
# --- Web App Dependencies ---
# Copy web app manifest files
-COPY src/interface/web/package.json src/interface/web/yarn.lock /tmp/web/
+COPY src/interface/web/package.json src/interface/web/bun.lock /opt/khoj_web/
# Install web app dependencies
-# note: yarn will be available from the "features" in devcontainer.json
-RUN yarn install --cwd /tmp/web --cache-folder /opt/yarn-cache
+RUN cd /opt/khoj_web && bun install && chown -R vscode:vscode .
# The .venv and node_modules are now populated in the image.
# The rest of the source code will be mounted by VS Code from your local checkout,
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index fa1cd1e3..eb060c8c 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -4,7 +4,7 @@
"dockerfile": "Dockerfile",
"context": "..", // Build context is the project root
"args": {
- "PYTHON_VERSION": "3.10"
+ "PYTHON_VERSION": "3.12"
}
},
"forwardPorts": [
@@ -53,11 +53,6 @@
"postCreateCommand": "scripts/dev_setup.sh --devcontainer",
"features": {
"ghcr.io/devcontainers/features/github-cli:1": {},
- "ghcr.io/devcontainers/features/node:1": {
- "version": "lts",
- "installYarnUsingApt": false,
- "nodeGypDependencies": true
- }
},
"remoteUser": "vscode"
}
diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
index c5ce4c44..6352e688 100644
--- a/.github/workflows/pre-commit.yml
+++ b/.github/workflows/pre-commit.yml
@@ -35,18 +35,24 @@ jobs:
with:
fetch-depth: 0
- - name: Set up Python 3.11
- uses: actions/setup-python@v4
+ - name: Install uv
+ uses: astral-sh/setup-uv@v4
with:
- python-version: 3.11
+ version: "latest"
+
+ - name: Set up Python 3.11
+ run: uv python install 3.11
- name: ⏬️ Install Dependencies
run: |
sudo apt update && sudo apt install -y libegl1
- python -m pip install --upgrade pip
- name: ⬇️ Install Application
- run: pip install --no-cache-dir --upgrade .[dev]
+ env:
+ UV_INDEX: "https://download.pytorch.org/whl/cpu"
+ UV_INDEX_STRATEGY: "unsafe-best-match"
+ CUDA_VISIBLE_DEVICES: ""
+ run: uv sync --all-extras
- name: 🌡️ Validate Application
- run: pre-commit run --hook-stage manual --all
+ run: uv run pre-commit run --hook-stage manual --all
diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml
index e933e9a5..b419269b 100644
--- a/.github/workflows/pypi.yml
+++ b/.github/workflows/pypi.yml
@@ -34,25 +34,28 @@ jobs:
with:
fetch-depth: 0
- - name: Set up Python 3.11
- uses: actions/setup-python@v4
+ - name: Install uv
+ uses: astral-sh/setup-uv@v4
with:
- python-version: '3.11.12'
+ version: "latest"
+
+ - name: Set up Python 3.11
+ run: uv python install 3.11.12
- name: ⬇️ Install Server
- run: python -m pip install --upgrade pip && pip install --upgrade .
+ run: uv sync --all-extras
+
+ - name: Install bun
+ uses: oven-sh/setup-bun@v2
+ with:
+ bun-version: latest
- name: ⬇️ Install Web Client
run: |
- yarn install
- yarn pypiciexport
+ bun install
+ bun pypiciexport
working-directory: src/interface/web
- - name: 📂 Copy Generated Files
- run: |
- mkdir -p src/khoj/interface/compiled
- cp -r /opt/hostedtoolcache/Python/3.11.12/x64/lib/python3.11/site-packages/khoj/interface/compiled/* src/khoj/interface/compiled/
-
- name: ⚙️ Build Python Package
run: |
# Setup Environment for Reproducible Builds
@@ -61,13 +64,13 @@ jobs:
rm -rf dist
# Build PyPI Package
- pipx run build
+ uv build
- name: 🌡️ Validate Python Package
run: |
# Validate PyPi Package
- pipx run check-wheel-contents dist/*.whl --ignore W004
- pipx run twine check dist/*
+ uv tool run check-wheel-contents dist/*.whl --ignore W004
+ uv tool run twine check dist/*
- name: ⏫ Upload Python Package Artifacts
uses: actions/upload-artifact@v4
diff --git a/.github/workflows/run_evals.yml b/.github/workflows/run_evals.yml
index 0d6490d5..ccab4034 100644
--- a/.github/workflows/run_evals.yml
+++ b/.github/workflows/run_evals.yml
@@ -106,10 +106,13 @@ jobs:
with:
fetch-depth: 0
- - name: Set up Python
- uses: actions/setup-python@v4
+ - name: Install uv
+ uses: astral-sh/setup-uv@v4
with:
- python-version: '3.10'
+ version: "latest"
+
+ - name: Set up Python
+ run: uv python install 3.10
- name: Get App Version
id: hatch
@@ -127,16 +130,18 @@ jobs:
DEBIAN_FRONTEND: noninteractive
run: |
# install dependencies
- sudo apt update && sudo apt install -y git python3-pip libegl1 sqlite3 libsqlite3-dev libsqlite3-0 ffmpeg libsm6 libxext6
- # upgrade pip
- python -m ensurepip --upgrade && python -m pip install --upgrade pip
+ sudo apt update && sudo apt install -y git libegl1 sqlite3 libsqlite3-dev libsqlite3-0 ffmpeg libsm6 libxext6
# install terrarium for code sandbox
git clone https://github.com/khoj-ai/terrarium.git && cd terrarium && npm install --legacy-peer-deps && mkdir pyodide_cache
- name: ⬇️ Install Application
+ env:
+ UV_INDEX: "https://download.pytorch.org/whl/cpu"
+ UV_INDEX_STRATEGY: "unsafe-best-match"
+ CUDA_VISIBLE_DEVICES: ""
run: |
sed -i 's/dynamic = \["version"\]/version = "${{ steps.hatch.outputs.version }}"/' pyproject.toml
- pip install --upgrade .[dev]
+ uv sync --all-extras
- name: 📝 Run Eval
env:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8e04a8fc..c4c5628b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -54,10 +54,13 @@ jobs:
with:
fetch-depth: 0
- - name: Set up Python
- uses: actions/setup-python@v4
+ - name: Install uv
+ uses: astral-sh/setup-uv@v4
with:
- python-version: ${{ matrix.python_version }}
+ version: "latest"
+
+ - name: Set up Python ${{ matrix.python_version }}
+ run: uv python install ${{ matrix.python_version }}
- name: ⏬️ Install Dependencies
env:
@@ -71,17 +74,12 @@ jobs:
run : |
apt install -y postgresql postgresql-client && apt install -y postgresql-server-dev-16
- - name: ⬇️ Install pip
- run: |
- apt install -y python3-pip
- python3 -m ensurepip --upgrade
- python3 -m pip install --upgrade pip
-
- name: ⬇️ Install Application
env:
- PIP_EXTRA_INDEX_URL: "https://download.pytorch.org/whl/cpu https://abetlen.github.io/llama-cpp-python/whl/cpu"
+ UV_INDEX: "https://download.pytorch.org/whl/cpu"
+ UV_INDEX_STRATEGY: "unsafe-best-match"
CUDA_VISIBLE_DEVICES: ""
- run: sed -i 's/dynamic = \["version"\]/version = "0.0.0"/' pyproject.toml && pip install --break-system-packages --upgrade .[dev]
+ run: sed -i 's/dynamic = \["version"\]/version = "0.0.0"/' pyproject.toml && uv sync --all-extras
- name: 🧪 Test Application
env:
@@ -91,5 +89,5 @@ jobs:
POSTGRES_PASSWORD: postgres
POSTGRES_DB: postgres
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
- run: pytest
+ run: uv run pytest
timeout-minutes: 10
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index c3a2ae44..fbf5de8b 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,8 +1,12 @@
repos:
-- repo: https://github.com/psf/black
- rev: 23.1.0
+- repo: https://github.com/astral-sh/ruff-pre-commit
+ rev: v0.12.3
hooks:
- - id: black
+ - id: ruff-check
+ args: [ --fix ]
+ files: \.py$
+ - id: ruff-format
+ files: \.py$
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
@@ -16,13 +20,6 @@ repos:
- id: check-toml
- id: check-yaml
-- repo: https://github.com/pycqa/isort
- rev: 5.12.0
- hooks:
- - id: isort
- name: isort (python)
- args: ["--profile", "black", "--filter-files"]
-
- repo: https://github.com/pre-commit/mirrors-mypy
rev: v1.0.0
hooks:
diff --git a/Dockerfile b/Dockerfile
index fd37d15f..d133d8f9 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -35,17 +35,17 @@ RUN sed -i "s/dynamic = \\[\"version\"\\]/version = \"$VERSION\"/" pyproject.tom
pip install --no-cache-dir .
# Build Web App
-FROM node:23-alpine AS web-app
+FROM oven/bun:1-alpine AS web-app
# Set build optimization env vars
ENV NODE_ENV=production
ENV NEXT_TELEMETRY_DISABLED=1
WORKDIR /app/src/interface/web
# Install dependencies first (cache layer)
-COPY src/interface/web/package.json src/interface/web/yarn.lock ./
-RUN yarn install --frozen-lockfile
+COPY src/interface/web/package.json src/interface/web/bun.lock ./
+RUN bun install --frozen-lockfile
# Copy source and build
COPY src/interface/web/. ./
-RUN yarn build
+RUN bun run build
# Merge the Server and Web App into a Single Image
FROM base
diff --git a/computer.Dockerfile b/computer.Dockerfile
index 42b6b2df..dfa15064 100644
--- a/computer.Dockerfile
+++ b/computer.Dockerfile
@@ -72,39 +72,31 @@ RUN apt update \
&& apt remove -y light-locker xfce4-screensaver xfce4-power-manager || true
# Create Computer User
-ENV USERNAME=operator
+ENV USERNAME=khoj
ENV HOME=/home/$USERNAME
-RUN useradd -m -s /bin/bash -d $HOME -g $USERNAME $USERNAME && echo "${USERNAME} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+RUN groupadd $USERNAME && \
+ useradd -m -s /bin/bash -d $HOME -g $USERNAME $USERNAME && \
+ echo "${USERNAME} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
USER $USERNAME
WORKDIR $HOME
-# Setup Python
-RUN git clone https://github.com/pyenv/pyenv.git ~/.pyenv && \
- cd ~/.pyenv && src/configure && make -C src && cd .. && \
- echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc && \
- echo 'command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc && \
- echo 'eval "$(pyenv init -)"' >> ~/.bashrc
-ENV PYENV_ROOT="$HOME/.pyenv"
-ENV PATH="$PYENV_ROOT/bin:$PATH"
-ENV PYENV_VERSION_MAJOR=3
-ENV PYENV_VERSION_MINOR=11
-ENV PYENV_VERSION_PATCH=6
-ENV PYENV_VERSION=$PYENV_VERSION_MAJOR.$PYENV_VERSION_MINOR.$PYENV_VERSION_PATCH
-RUN eval "$(pyenv init -)" && \
- pyenv install $PYENV_VERSION && \
- pyenv global $PYENV_VERSION && \
- pyenv rehash
-ENV PATH="$HOME/.pyenv/shims:$HOME/.pyenv/bin:$PATH"
+# Install Python using uv and create a virtual environment
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+ENV PYTHON_VERSION=3.11.6
+RUN uv python pin $PYTHON_VERSION
+RUN uv venv $HOME/.venv --python $PYTHON_VERSION --seed
+RUN echo 'export PATH="$HOME/.venv/bin:$PATH"' >> "$HOME/.bashrc"
+ENV PATH="$HOME/.venv/bin:$PATH"
# Install Python Packages
-RUN python3 -m pip install --no-cache-dir \
+RUN uv pip install --no-cache-dir \
pyautogui \
Pillow \
pyperclip \
pygetwindow
# Setup VNC
-RUN x11vnc -storepasswd secret /home/operator/.vncpass
+RUN x11vnc -storepasswd secret /home/khoj/.vncpass
ARG WIDTH=1024
ARG HEIGHT=768
@@ -115,13 +107,22 @@ ENV DISPLAY_NUM=$DISPLAY_NUM
ENV DISPLAY=":$DISPLAY_NUM"
# Expose VNC on port 5900
-# run Xvfb, x11vnc, Xfce (no login manager)
EXPOSE 5900
-CMD ["/bin/sh", "-c", " export XDG_RUNTIME_DIR=/run/user/$(id -u); \
- mkdir -p $XDG_RUNTIME_DIR && chown $USERNAME:$USERNAME $XDG_RUNTIME_DIR && chmod 0700 $XDG_RUNTIME_DIR; \
+
+# Start Virtual Display (Xvfb), Desktop Manager (XFCE) and Remote Viewer (X11 VNC)
+CMD ["/bin/sh", "-c", " \
+ # Create and permission XDG_RUNTIME_DIR with sudo \n\
+ export XDG_RUNTIME_DIR=/run/user/$(id -u); \
+ sudo mkdir -p $XDG_RUNTIME_DIR && \
+ sudo chown $(id -u):$(id -g) $XDG_RUNTIME_DIR && \
+ sudo chmod 0700 $XDG_RUNTIME_DIR; \
+ \
+ # Start Virtual Display \n\
Xvfb $DISPLAY -screen 0 ${WIDTH}x${HEIGHT}x24 -dpi 96 -auth /home/$USERNAME/.Xauthority >/dev/null 2>&1 & \
sleep 1; \
xauth add $DISPLAY . $(mcookie); \
+ \
+ # Start VNC Server \n\
x11vnc -display $DISPLAY -forever -rfbauth /home/$USERNAME/.vncpass -listen 0.0.0.0 -rfbport 5900 >/dev/null 2>&1 & \
eval $(dbus-launch --sh-syntax) && \
startxfce4 & \
diff --git a/documentation/docs/contributing/development.mdx b/documentation/docs/contributing/development.mdx
index 3e299745..fda00573 100644
--- a/documentation/docs/contributing/development.mdx
+++ b/documentation/docs/contributing/development.mdx
@@ -30,7 +30,7 @@ git clone https://github.com/khoj-ai/khoj && cd khoj
python3 -m venv .venv && source .venv/bin/activate
# For MacOS or zsh users run this
-pip install -e '.[dev]'
+uv sync --all-extras
```
@@ -42,7 +42,7 @@ git clone https://github.com/khoj-ai/khoj && cd khoj
python3 -m venv .venv && .venv\Scripts\activate
# Install Khoj for Development
-pip install -e '.[dev]'
+uv sync --all-extras
```
@@ -54,7 +54,7 @@ git clone https://github.com/khoj-ai/khoj && cd khoj
python3 -m venv .venv && source .venv/bin/activate
# Install Khoj for Development
-pip install -e '.[dev]'
+uv sync --all-extras
```
@@ -129,7 +129,7 @@ Always run `yarn export` to test your front-end changes on http://localhost:4211
- Try reactivating the virtual environment and rerunning the `khoj` command.
- If it still doesn't work repeat the installation process.
2. Python Package Missing
- - Use `pip install xxx` and try running the `khoj` command.
+ - Use `uv add xxx` and try running the `khoj` command.
3. Command `createdb` Not Recognized
- make sure path to postgres binaries is included in environment variables. It usually looks something like
```
diff --git a/prod.Dockerfile b/prod.Dockerfile
index 015a5a3e..6337b0f7 100644
--- a/prod.Dockerfile
+++ b/prod.Dockerfile
@@ -34,17 +34,17 @@ RUN sed -i "s/dynamic = \\[\"version\"\\]/version = \"$VERSION\"/" pyproject.tom
pip install --no-cache-dir -e .[prod]
# Build Web App
-FROM node:20-alpine AS web-app
+FROM oven/bun:1-alpine AS web-app
# Set build optimization env vars
ENV NODE_ENV=production
ENV NEXT_TELEMETRY_DISABLED=1
WORKDIR /app/src/interface/web
# Install dependencies first (cache layer)
-COPY src/interface/web/package.json src/interface/web/yarn.lock ./
-RUN yarn install --frozen-lockfile
+COPY src/interface/web/package.json src/interface/web/bun.lock ./
+RUN bun install --frozen-lockfile
# Copy source and build
COPY src/interface/web/. ./
-RUN yarn build
+RUN bun run build
# Merge the Server and Web App into a Single Image
FROM base
diff --git a/pyproject.toml b/pyproject.toml
index 6491dc06..3f10ce66 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -50,6 +50,7 @@ dependencies = [
"pydantic[email] >= 2.0.0",
"pyyaml ~= 6.0",
"rich >= 13.3.1",
+ "click < 8.2.2",
"schedule == 1.1.0",
"sentence-transformers == 3.4.1",
"einops == 0.8.0",
@@ -122,7 +123,7 @@ dev = [
"freezegun >= 1.2.0",
"factory-boy >= 3.2.1",
"mypy >= 1.0.1",
- "black >= 23.1.0",
+ "ruff >= 0.12.0",
"pre-commit >= 3.0.4",
"gitpython ~= 3.1.43",
"datasets",
@@ -149,11 +150,31 @@ non_interactive = true
show_error_codes = true
warn_unused_ignores = false
-[tool.black]
+[tool.ruff]
line-length = 120
-[tool.isort]
-profile = "black"
+[tool.ruff.lint]
+select = ["E", "F", "I"] # Enable error, warning, and import checks
+ignore = [
+ "E501", # Ignore line length
+ "F405", # Ignore name not defined (e.g., from imports)
+ "E402", # Ignore module level import not at top of file
+]
+unfixable = ["F841"] # Don't auto-remove unused variables
+exclude = [ "tests/*.py" ]
+[tool.ruff.lint.per-file-ignores]
+"src/khoj/main.py" = [
+ "I001", # Ignore Import order
+ "I002", # Ignore Import not at top of file
+ "E402", # Ignore module level import not at top of file
+]
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+
+[tool.ruff.lint.isort]
+known-first-party = ["khoj"]
[tool.pytest.ini_options]
addopts = "--strict-markers"
diff --git a/scripts/dev_setup.sh b/scripts/dev_setup.sh
index 9a9afe03..937638cd 100755
--- a/scripts/dev_setup.sh
+++ b/scripts/dev_setup.sh
@@ -7,11 +7,11 @@ INSTALL_FULL=false
DEVCONTAINER=false
for arg in "$@"
do
- if [ "$arg" == "--full" ]
+ if [ "$arg" = "--full" ]
then
INSTALL_FULL=true
fi
- if [ "$arg" == "--devcontainer" ]
+ if [ "$arg" = "--devcontainer" ]
then
DEVCONTAINER=true
fi
@@ -24,26 +24,40 @@ if [ "$DEVCONTAINER" = true ]; then
# Use devcontainer launch.json
mkdir -p .vscode && cp .devcontainer/launch.json .vscode/launch.json
- # Activate the pre-installed venv (no need to create new one)
- echo "Using Python environment at /opt/venv"
- # PATH should already include /opt/venv/bin from Dockerfile
-
- # Install khoj in editable mode (dependencies already installed)
- python3 -m pip install -e '.[dev]'
+ # Install Server App using pre-installed dependencies
+ echo "Setup Server App with UV. Use pre-installed dependencies in $UV_PROJECT_ENVIRONMENT."
+ sed -i "s/dynamic = \\[\"version\"\\]/version = \"$VERSION\"/" pyproject.toml
+ cp /opt/uv.lock.linux uv.lock
+ uv sync --all-extras
# Install Web App using cached dependencies
- echo "Installing Web App using cached dependencies..."
+ echo "Setup Web App with Bun. Use pre-installed dependencies in /opt/khoj_web."
cd "$PROJECT_ROOT/src/interface/web"
- yarn install --cache-folder /opt/yarn-cache && yarn export
+ ln -sf /opt/khoj_web/node_modules node_modules
+ bun install && bun run ciexport
else
# Standard setup
echo "Installing Server App..."
cd "$PROJECT_ROOT"
- python3 -m venv .venv && . .venv/bin/activate && python3 -m pip install -e '.[dev]'
+ if command -v uv &> /dev/null
+ then
+ uv venv
+ uv sync --all-extras
+ else
+ python3 -m venv .venv && . .venv/bin/activate
+ python3 -m pip install -e '.[dev]'
+ fi
echo "Installing Web App..."
cd "$PROJECT_ROOT/src/interface/web"
- yarn install && yarn export
+ if command -v bun &> /dev/null
+ then
+ echo "using Bun."
+ bun install && bun run export
+ else
+ echo "using Yarn."
+ yarn install && yarn export
+ fi
fi
# Install Obsidian App
diff --git a/src/interface/web/.husky/pre-commit b/src/interface/web/.husky/pre-commit
index 053576eb..cc86a2ef 100755
--- a/src/interface/web/.husky/pre-commit
+++ b/src/interface/web/.husky/pre-commit
@@ -1,5 +1,5 @@
#!/usr/bin/env sh
. "$(dirname "$0")/_/husky.sh"
-yarn run lint-staged
-yarn test
+bun run lint-staged
+bun run test
diff --git a/src/interface/web/README.md b/src/interface/web/README.md
index 4196c5c7..70fd46ab 100644
--- a/src/interface/web/README.md
+++ b/src/interface/web/README.md
@@ -5,19 +5,19 @@ This is a [Next.js](https://nextjs.org/) project.
First, install the dependencies:
```bash
-yarn install
+bun install
```
In case you run into any dependency linking issues, you can try running:
```bash
-yarn add next
+bun add next
```
### Run the development server:
```bash
-yarn dev
+bun dev
```
Make sure the `rewrites` in `next.config.mjs` are set up correctly for your environment. The rewrites are used to proxy requests to the API server.
@@ -44,27 +44,30 @@ You can start editing the page by modifying any of the `.tsx` pages. The page au
We've setup a utility command for building and serving the built files. This is useful for testing the production build locally.
1. Exporting code
-To build the files once and serve them, run:
+ To build the files once and serve them, run:
+
```bash
-yarn export
+bun export
```
If you're using Windows:
-```bash
-yarn windowsexport
-```
+```bash
+bun windowsexport
+```
2. Continuously building code
To keep building the files and serving them, run:
+
```bash
-yarn watch
+bun watch
```
If you're using Windows:
+
```bash
-yarn windowswatch
+bun windowswatch
```
Now you should be able to load your custom pages from the Khoj app at http://localhost:42110/. To server any of the built files, you should update the routes in the `web_client.py` like so, where `new_file` is the new page you've added in this repo:
diff --git a/src/interface/web/app/agents/page.tsx b/src/interface/web/app/agents/page.tsx
index 35d23660..8b578de4 100644
--- a/src/interface/web/app/agents/page.tsx
+++ b/src/interface/web/app/agents/page.tsx
@@ -344,14 +344,16 @@ export default function Agents() {
/>
How it works Use any of these
specialized personas to tune your conversation to your needs.
- {
- !isSubscribed && (
-
- {" "}
- Upgrade your plan to leverage custom models. You will fallback to the default model when chatting.
-
- )
- }
+ {!isSubscribed && (
+
+ {" "}
+
+ Upgrade your plan
+ {" "}
+ to leverage custom models. You will fallback to the
+ default model when chatting.
+
+ )}
diff --git a/src/interface/web/app/common/auth.ts b/src/interface/web/app/common/auth.ts
index 4716fa13..defbf536 100644
--- a/src/interface/web/app/common/auth.ts
+++ b/src/interface/web/app/common/auth.ts
@@ -90,11 +90,9 @@ export interface UserConfig {
export function useUserConfig(detailed: boolean = false) {
const url = `/api/settings?detailed=${detailed}`;
- const {
- data,
- error,
- isLoading,
- } = useSWR
(url, fetcher, { revalidateOnFocus: false });
+ const { data, error, isLoading } = useSWR(url, fetcher, {
+ revalidateOnFocus: false,
+ });
if (error || !data || data?.detail === "Forbidden") {
return { data: null, error, isLoading };
diff --git a/src/interface/web/app/common/modelSelector.tsx b/src/interface/web/app/common/modelSelector.tsx
index 62485dd8..4075d487 100644
--- a/src/interface/web/app/common/modelSelector.tsx
+++ b/src/interface/web/app/common/modelSelector.tsx
@@ -1,12 +1,12 @@
-"use client"
+"use client";
-import * as React from "react"
+import * as React from "react";
import { useState, useEffect } from "react";
-import { PopoverProps } from "@radix-ui/react-popover"
+import { PopoverProps } from "@radix-ui/react-popover";
import { Check, CaretUpDown } from "@phosphor-icons/react";
-import { cn } from "@/lib/utils"
+import { cn } from "@/lib/utils";
import { useIsMobileWidth, useMutationObserver } from "@/app/common/utils";
import { Button } from "@/components/ui/button";
import {
@@ -17,11 +17,7 @@ import {
CommandItem,
CommandList,
} from "@/components/ui/command";
-import {
- Popover,
- PopoverContent,
- PopoverTrigger,
-} from "@/components/ui/popover";
+import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
import { ModelOptions, useUserConfig } from "./auth";
import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card";
@@ -35,7 +31,7 @@ interface ModelSelectorProps extends PopoverProps {
}
export function ModelSelector({ ...props }: ModelSelectorProps) {
- const [open, setOpen] = React.useState(false)
+ const [open, setOpen] = React.useState(false);
const [peekedModel, setPeekedModel] = useState(undefined);
const [selectedModel, setSelectedModel] = useState(undefined);
const { data: userConfig, error, isLoading: isLoadingUserConfig } = useUserConfig(true);
@@ -48,14 +44,18 @@ export function ModelSelector({ ...props }: ModelSelectorProps) {
if (userConfig) {
setModels(userConfig.chat_model_options);
if (!props.initialModel) {
- const selectedChatModelOption = userConfig.chat_model_options.find(model => model.id === userConfig.selected_chat_model_config);
+ const selectedChatModelOption = userConfig.chat_model_options.find(
+ (model) => model.id === userConfig.selected_chat_model_config,
+ );
if (!selectedChatModelOption && userConfig.chat_model_options.length > 0) {
setSelectedModel(userConfig.chat_model_options[0]);
} else {
setSelectedModel(selectedChatModelOption);
}
} else {
- const model = userConfig.chat_model_options.find(model => model.name === props.initialModel);
+ const model = userConfig.chat_model_options.find(
+ (model) => model.name === props.initialModel,
+ );
setSelectedModel(model);
}
}
@@ -68,15 +68,11 @@ export function ModelSelector({ ...props }: ModelSelectorProps) {
}, [selectedModel, userConfig, props.onSelect]);
if (isLoadingUserConfig) {
- return (
-
- );
+ return ;
}
if (error) {
- return (
- {error.message}
- );
+ return {error.message}
;
}
return (
@@ -92,30 +88,85 @@ export function ModelSelector({ ...props }: ModelSelectorProps) {
disabled={props.disabled ?? false}
>
- {selectedModel ? selectedModel.name?.substring(0, 20) : "Select a model..."}
+ {selectedModel
+ ? selectedModel.name?.substring(0, 20)
+ : "Select a model..."}
- {
- isMobileWidth ?
+ {isMobileWidth ? (
+
+
+
+
+ No Models found.
+
+ {models &&
+ models.length > 0 &&
+ models.map((model) => (
+ setPeekedModel(model)}
+ onSelect={() => {
+ setSelectedModel(model);
+ setOpen(false);
+ }}
+ isActive={props.isActive}
+ />
+ ))}
+
+
+
+
+ ) : (
+
+
+
+
+ {peekedModel?.name}
+
+
+ {peekedModel?.description}
+
+ {peekedModel?.strengths ? (
+
+
+ Strengths
+
+
+ {peekedModel.strengths}
+
+
+ ) : null}
+
+
+
No Models found.
- {models && models.length > 0 && models
- .map((model) => (
+ {models &&
+ models.length > 0 &&
+ models.map((model) => (
setPeekedModel(model)}
onSelect={() => {
- setSelectedModel(model)
- setOpen(false)
+ setSelectedModel(model);
+ setOpen(false);
}}
isActive={props.isActive}
/>
@@ -124,74 +175,24 @@ export function ModelSelector({ ...props }: ModelSelectorProps) {
- :
-
-
-
-
{peekedModel?.name}
-
- {peekedModel?.description}
-
- {peekedModel?.strengths ? (
-
-
- Strengths
-
-
- {peekedModel.strengths}
-
-
- ) : null}
-
-
-
-
-
-
-
- No Models found.
-
- {models && models.length > 0 && models
- .map((model) => (
- setPeekedModel(model)}
- onSelect={() => {
- setSelectedModel(model)
- setOpen(false)
- }}
- isActive={props.isActive}
- />
- ))}
-
-
-
-
-
- }
+
+ )}
- )
+ );
}
interface ModelItemProps {
- model: ModelOptions,
- isSelected: boolean,
- onSelect: () => void,
- onPeek: (model: ModelOptions) => void
- isActive?: boolean
+ model: ModelOptions;
+ isSelected: boolean;
+ onSelect: () => void;
+ onPeek: (model: ModelOptions) => void;
+ isActive?: boolean;
}
function ModelItem({ model, isSelected, onSelect, onPeek, isActive }: ModelItemProps) {
- const ref = React.useRef(null)
+ const ref = React.useRef(null);
useMutationObserver(ref, (mutations) => {
mutations.forEach((mutation) => {
@@ -200,10 +201,10 @@ function ModelItem({ model, isSelected, onSelect, onPeek, isActive }: ModelItemP
mutation.attributeName === "aria-selected" &&
ref.current?.getAttribute("aria-selected") === "true"
) {
- onPeek(model)
+ onPeek(model);
}
- })
- })
+ });
+ });
return (
- {model.name} {model.tier === "standard" && (Futurist)}
-
+ {model.name}{" "}
+ {model.tier === "standard" && (Futurist)}
+
- )
+ );
}
diff --git a/src/interface/web/app/common/utils.ts b/src/interface/web/app/common/utils.ts
index b3520c00..d5cba0cf 100644
--- a/src/interface/web/app/common/utils.ts
+++ b/src/interface/web/app/common/utils.ts
@@ -1,6 +1,6 @@
import { useEffect, useState } from "react";
import useSWR from "swr";
-import * as React from "react"
+import * as React from "react";
export interface LocationData {
city?: string;
@@ -78,16 +78,16 @@ export const useMutationObserver = (
characterData: true,
childList: true,
subtree: true,
- }
+ },
) => {
React.useEffect(() => {
if (ref.current) {
- const observer = new MutationObserver(callback)
- observer.observe(ref.current, options)
- return () => observer.disconnect()
+ const observer = new MutationObserver(callback);
+ observer.observe(ref.current, options);
+ return () => observer.disconnect();
}
- }, [ref, callback, options])
-}
+ }, [ref, callback, options]);
+};
export function useIsDarkMode() {
const [darkMode, setDarkMode] = useState(false);
diff --git a/src/interface/web/app/components/agentCard/agentCard.tsx b/src/interface/web/app/components/agentCard/agentCard.tsx
index da7e8889..8e739d57 100644
--- a/src/interface/web/app/components/agentCard/agentCard.tsx
+++ b/src/interface/web/app/components/agentCard/agentCard.tsx
@@ -1061,12 +1061,27 @@ export function AgentModificationForm(props: AgentModificationFormProps) {
className="h-6 px-2 text-xs"
onClick={(e) => {
e.stopPropagation();
- const filteredFiles = allFileOptions.filter(file =>
- file.toLowerCase().includes(fileSearchValue.toLowerCase())
+ const filteredFiles =
+ allFileOptions.filter((file) =>
+ file
+ .toLowerCase()
+ .includes(
+ fileSearchValue.toLowerCase(),
+ ),
+ );
+ const currentFiles =
+ props.form.getValues("files") ||
+ [];
+ const newFiles = [
+ ...new Set([
+ ...currentFiles,
+ ...filteredFiles,
+ ]),
+ ];
+ props.form.setValue(
+ "files",
+ newFiles,
);
- const currentFiles = props.form.getValues("files") || [];
- const newFiles = [...new Set([...currentFiles, ...filteredFiles])];
- props.form.setValue("files", newFiles);
}}
>
Select All
@@ -1078,12 +1093,28 @@ export function AgentModificationForm(props: AgentModificationFormProps) {
className="h-6 px-2 text-xs"
onClick={(e) => {
e.stopPropagation();
- const filteredFiles = allFileOptions.filter(file =>
- file.toLowerCase().includes(fileSearchValue.toLowerCase())
+ const filteredFiles =
+ allFileOptions.filter((file) =>
+ file
+ .toLowerCase()
+ .includes(
+ fileSearchValue.toLowerCase(),
+ ),
+ );
+ const currentFiles =
+ props.form.getValues("files") ||
+ [];
+ const newFiles =
+ currentFiles.filter(
+ (file) =>
+ !filteredFiles.includes(
+ file,
+ ),
+ );
+ props.form.setValue(
+ "files",
+ newFiles,
);
- const currentFiles = props.form.getValues("files") || [];
- const newFiles = currentFiles.filter(file => !filteredFiles.includes(file));
- props.form.setValue("files", newFiles);
}}
>
Deselect All
diff --git a/src/interface/web/app/components/allConversations/allConversations.tsx b/src/interface/web/app/components/allConversations/allConversations.tsx
index 0d1c1b88..a8ff3702 100644
--- a/src/interface/web/app/components/allConversations/allConversations.tsx
+++ b/src/interface/web/app/components/allConversations/allConversations.tsx
@@ -127,7 +127,7 @@ function renameConversation(conversationId: string, newTitle: string) {
},
})
.then((response) => response.json())
- .then((data) => { })
+ .then((data) => {})
.catch((err) => {
console.error(err);
return;
@@ -171,7 +171,7 @@ function deleteConversation(conversationId: string) {
mutate("/api/chat/sessions");
}
})
- .then((data) => { })
+ .then((data) => {})
.catch((err) => {
console.error(err);
return;
@@ -245,9 +245,7 @@ export function FilesMenu(props: FilesMenuProps) {
Context
- {
- error ? "Failed to load files" : "Failed to load selected files"
- }
+ {error ? "Failed to load files" : "Failed to load selected files"}
@@ -257,7 +255,7 @@ export function FilesMenu(props: FilesMenuProps) {
- )
+ );
}
if (!files) return ;
@@ -443,10 +441,7 @@ function SessionsAndFiles(props: SessionsAndFilesProps) {
{props.sideBarOpen && (
-
+
{props.subsetOrganizedData != null &&
Object.keys(props.subsetOrganizedData)
@@ -471,7 +466,9 @@ function SessionsAndFiles(props: SessionsAndFilesProps) {
agent_name={chatHistory.agent_name}
agent_color={chatHistory.agent_color}
agent_icon={chatHistory.agent_icon}
- agent_is_hidden={chatHistory.agent_is_hidden}
+ agent_is_hidden={
+ chatHistory.agent_is_hidden
+ }
/>
),
)}
@@ -709,7 +706,7 @@ function ChatSession(props: ChatHistory) {
className="flex items-center gap-2 no-underline"
>
{title}
diff --git a/src/interface/web/app/components/appSidebar/appSidebar.tsx b/src/interface/web/app/components/appSidebar/appSidebar.tsx
index 85901371..94b3b3ea 100644
--- a/src/interface/web/app/components/appSidebar/appSidebar.tsx
+++ b/src/interface/web/app/components/appSidebar/appSidebar.tsx
@@ -48,13 +48,12 @@ async function openChat(userData: UserProfile | null | undefined) {
}
}
-
// Menu items.
const items = [
{
title: "Home",
url: "/",
- icon: HouseSimple
+ icon: HouseSimple,
},
{
title: "Agents",
diff --git a/src/interface/web/app/components/chatHistory/chatHistory.tsx b/src/interface/web/app/components/chatHistory/chatHistory.tsx
index 18810a93..bb61c9f7 100644
--- a/src/interface/web/app/components/chatHistory/chatHistory.tsx
+++ b/src/interface/web/app/components/chatHistory/chatHistory.tsx
@@ -52,7 +52,7 @@ interface TrainOfThoughtFrame {
}
interface TrainOfThoughtGroup {
- type: 'video' | 'text';
+ type: "video" | "text";
frames?: TrainOfThoughtFrame[];
textEntries?: TrainOfThoughtObject[];
}
@@ -65,7 +65,9 @@ interface TrainOfThoughtComponentProps {
completed?: boolean;
}
-function extractTrainOfThoughtGroups(trainOfThought?: TrainOfThoughtObject[]): TrainOfThoughtGroup[] {
+function extractTrainOfThoughtGroups(
+ trainOfThought?: TrainOfThoughtObject[],
+): TrainOfThoughtGroup[] {
if (!trainOfThought) return [];
const groups: TrainOfThoughtGroup[] = [];
@@ -94,8 +96,8 @@ function extractTrainOfThoughtGroups(trainOfThought?: TrainOfThoughtObject[]): T
// If we have accumulated text entries, add them as a text group
if (currentTextEntries.length > 0) {
groups.push({
- type: 'text',
- textEntries: [...currentTextEntries]
+ type: "text",
+ textEntries: [...currentTextEntries],
});
currentTextEntries = [];
}
@@ -116,8 +118,8 @@ function extractTrainOfThoughtGroups(trainOfThought?: TrainOfThoughtObject[]): T
// If we have accumulated video frames, add them as a video group
if (currentVideoFrames.length > 0) {
groups.push({
- type: 'video',
- frames: [...currentVideoFrames]
+ type: "video",
+ frames: [...currentVideoFrames],
});
currentVideoFrames = [];
}
@@ -130,14 +132,14 @@ function extractTrainOfThoughtGroups(trainOfThought?: TrainOfThoughtObject[]): T
// Add any remaining frames/entries
if (currentVideoFrames.length > 0) {
groups.push({
- type: 'video',
- frames: currentVideoFrames
+ type: "video",
+ frames: currentVideoFrames,
});
}
if (currentTextEntries.length > 0) {
groups.push({
- type: 'text',
- textEntries: currentTextEntries
+ type: "text",
+ textEntries: currentTextEntries,
});
}
@@ -177,10 +179,10 @@ function TrainOfThoughtComponent(props: TrainOfThoughtComponentProps) {
// Convert string array to TrainOfThoughtObject array if needed
let trainOfThoughtObjects: TrainOfThoughtObject[];
- if (typeof props.trainOfThought[0] === 'string') {
+ if (typeof props.trainOfThought[0] === "string") {
trainOfThoughtObjects = (props.trainOfThought as string[]).map((data, index) => ({
- type: 'text',
- data: data
+ type: "text",
+ data: data,
}));
} else {
trainOfThoughtObjects = props.trainOfThought as TrainOfThoughtObject[];
@@ -221,28 +223,37 @@ function TrainOfThoughtComponent(props: TrainOfThoughtComponentProps) {
{trainOfThoughtGroups.map((group, groupIndex) => (
- {group.type === 'video' && group.frames && group.frames.length > 0 && (
-
- )}
- {group.type === 'text' && group.textEntries && group.textEntries.map((entry, entryIndex) => {
- const lastIndex = trainOfThoughtGroups.length - 1;
- const isLastGroup = groupIndex === lastIndex;
- const isLastEntry = entryIndex === group.textEntries!.length - 1;
- const isPrimaryEntry = isLastGroup && isLastEntry && props.lastMessage && !props.completed;
-
- return (
- 0 && (
+
- );
- })}
+ )}
+ {group.type === "text" &&
+ group.textEntries &&
+ group.textEntries.map((entry, entryIndex) => {
+ const lastIndex = trainOfThoughtGroups.length - 1;
+ const isLastGroup = groupIndex === lastIndex;
+ const isLastEntry =
+ entryIndex === group.textEntries!.length - 1;
+ const isPrimaryEntry =
+ isLastGroup &&
+ isLastEntry &&
+ props.lastMessage &&
+ !props.completed;
+
+ return (
+
+ );
+ })}
))}
@@ -300,7 +311,8 @@ export default function ChatHistory(props: ChatHistoryProps) {
// ResizeObserver to handle content height changes (e.g., images loading)
useEffect(() => {
const contentWrapper = scrollableContentWrapperRef.current;
- const scrollViewport = scrollAreaRef.current?.querySelector
(scrollAreaSelector);
+ const scrollViewport =
+ scrollAreaRef.current?.querySelector(scrollAreaSelector);
if (!contentWrapper || !scrollViewport) return;
@@ -308,14 +320,18 @@ export default function ChatHistory(props: ChatHistoryProps) {
// Check current scroll position to decide if auto-scroll is warranted
const { scrollTop, scrollHeight, clientHeight } = scrollViewport;
const bottomThreshold = 50;
- const currentlyNearBottom = (scrollHeight - (scrollTop + clientHeight)) <= bottomThreshold;
+ const currentlyNearBottom =
+ scrollHeight - (scrollTop + clientHeight) <= bottomThreshold;
if (currentlyNearBottom) {
// Only auto-scroll if there are incoming messages being processed
if (props.incomingMessages && props.incomingMessages.length > 0) {
const lastMessage = props.incomingMessages[props.incomingMessages.length - 1];
// If the last message is not completed, or it just completed (indicated by incompleteIncomingMessageIndex still being set)
- if (!lastMessage.completed || (lastMessage.completed && incompleteIncomingMessageIndex !== null)) {
+ if (
+ !lastMessage.completed ||
+ (lastMessage.completed && incompleteIncomingMessageIndex !== null)
+ ) {
scrollToBottom(true); // Use instant scroll
}
}
@@ -463,7 +479,12 @@ export default function ChatHistory(props: ChatHistoryProps) {
});
});
// Optimistically set, the scroll listener will verify
- if (instant || scrollAreaEl && (scrollAreaEl.scrollHeight - (scrollAreaEl.scrollTop + scrollAreaEl.clientHeight)) < 5) {
+ if (
+ instant ||
+ (scrollAreaEl &&
+ scrollAreaEl.scrollHeight - (scrollAreaEl.scrollTop + scrollAreaEl.clientHeight) <
+ 5)
+ ) {
setIsNearBottom(true);
}
};
@@ -626,16 +647,19 @@ export default function ChatHistory(props: ChatHistoryProps) {
conversationId={props.conversationId}
turnId={messageTurnId}
/>
- {message.trainOfThought && message.trainOfThought.length > 0 && (
- t.length).join('-')}`}
- keyId={`${index}trainOfThought`}
- completed={message.completed}
- />
- )}
+ {message.trainOfThought &&
+ message.trainOfThought.length > 0 && (
+ t.length).join("-")}`}
+ keyId={`${index}trainOfThought`}
+ completed={message.completed}
+ />
+ )}
((props, ref) =>
/>
)}
- {props.chatMessage.by === "khoj" && props.onRetryMessage && props.isLastMessage && (
-
+ )}
fetch(url).then((res) => res.json());
export function ChatSidebar({ ...props }: ChatSideBarProps) {
-
if (props.isMobileWidth) {
return (
-
-
+
+
@@ -110,14 +142,14 @@ function AgentCreationForm(props: IAgentCreationProps) {
fetch(createAgentUrl, {
method: "POST",
headers: {
- "Content-Type": "application/json"
+ "Content-Type": "application/json",
},
- body: JSON.stringify(data)
+ body: JSON.stringify(data),
})
.then((res) => res.json())
.then((data: AgentData | AgentError) => {
console.log("Success:", data);
- if ('detail' in data) {
+ if ("detail" in data) {
setError(`Error creating agent: ${data.detail}`);
setIsCreating(false);
return;
@@ -142,162 +174,151 @@ function AgentCreationForm(props: IAgentCreationProps) {
}, [customAgentName, customAgentIcon, customAgentColor]);
return (
-