diff --git a/.dockerignore b/.dockerignore
index 8802acb5..6668a7b5 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,10 +1,11 @@
-.git/
-.pytest_cache/
-.vscode/
-.venv/
-docs/
+.*
+**/__pycache__/
+*.egg-info/
+documentation/
tests/
build/
dist/
scripts/
-*.egg-info/
+src/interface/
+src/telemetry/
+!src/interface/web
diff --git a/.github/workflows/dockerize.yml b/.github/workflows/dockerize.yml
index 1b5473cc..0dc9a27e 100644
--- a/.github/workflows/dockerize.yml
+++ b/.github/workflows/dockerize.yml
@@ -73,7 +73,7 @@ jobs:
run: rm -rf /opt/hostedtoolcache
- name: 📦 Build and Push Docker Image
- uses: docker/build-push-action@v2
+ uses: docker/build-push-action@v4
if: (matrix.image == 'local' && github.event_name == 'workflow_dispatch') && github.event.inputs.khoj == 'true' || (matrix.image == 'local' && github.event_name == 'push')
with:
context: .
@@ -86,9 +86,11 @@ jobs:
build-args: |
VERSION=${{ steps.hatch.outputs.version }}
PORT=42110
+ cache-from: type=gha,scope=${{ matrix.image }}
+ cache-to: type=gha,mode=max,scope=${{ matrix.image }}
- name: 📦️⛅️ Build and Push Cloud Docker Image
- uses: docker/build-push-action@v2
+ uses: docker/build-push-action@v4
if: (matrix.image == 'cloud' && github.event_name == 'workflow_dispatch') && github.event.inputs.khoj-cloud == 'true' || (matrix.image == 'cloud' && github.event_name == 'push')
with:
context: .
@@ -101,3 +103,5 @@ jobs:
build-args: |
VERSION=${{ steps.hatch.outputs.version }}
PORT=42110
+ cache-from: type=gha,scope=${{ matrix.image }}
+ cache-to: type=gha,mode=max,scope=${{ matrix.image }}
diff --git a/Dockerfile b/Dockerfile
index b481a8e6..fab572cc 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,5 @@
# syntax=docker/dockerfile:1
-FROM ubuntu:jammy
+FROM ubuntu:jammy AS base
LABEL homepage="https://khoj.dev"
LABEL repository="https://github.com/khoj-ai/khoj"
LABEL org.opencontainers.image.source="https://github.com/khoj-ai/khoj"
@@ -10,44 +10,54 @@ RUN apt update -y && apt -y install \
python3-pip \
swig \
curl \
- # Required by llama-cpp-python pre-built wheels. See #1628
- musl-dev \
# Required by RapidOCR
libgl1 \
libglx-mesa0 \
- libglib2.0-0 && \
- # Required by Next.js Web app
- curl -sL https://deb.nodesource.com/setup_20.x | bash - && \
- curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \
- echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \
- apt update -y && apt -y --no-install-recommends install nodejs yarn && \
- apt clean && rm -rf /var/lib/apt/lists/* && \
+ libglib2.0-0 \
# Required by llama-cpp-python pre-built wheels. See #1628
- ln -s /usr/lib/x86_64-linux-musl/libc.so /lib/libc.musl-x86_64.so.1
+ musl-dev && \
+ ln -s /usr/lib/x86_64-linux-musl/libc.so /lib/libc.musl-x86_64.so.1 && \
+ # Clean up
+ apt clean && rm -rf /var/lib/apt/lists/*
-# Install Application
+# Build Server
+FROM base AS server-deps
WORKDIR /app
COPY pyproject.toml .
COPY README.md .
ARG VERSION=0.0.0
-ENV PIP_EXTRA_INDEX_URL=https://abetlen.github.io/llama-cpp-python/whl/cpu
+# use the pre-built llama-cpp-python, torch cpu wheel
+ENV PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu https://abetlen.github.io/llama-cpp-python/whl/cpu"
+# avoid downloading unused cuda specific python packages
+ENV CUDA_VISIBLE_DEVICES=""
RUN sed -i "s/dynamic = \\[\"version\"\\]/version = \"$VERSION\"/" pyproject.toml && \
pip install --no-cache-dir .
-# Copy Source Code
-COPY . .
-
-# Set the PYTHONPATH environment variable in order for it to find the Django app.
-ENV PYTHONPATH=/app/src:$PYTHONPATH
-
-# Go to the directory src/interface/web and export the built Next.js assets
+# Build Web App
+FROM node:20-alpine AS web-app
+# Set build optimization env vars
+ENV NODE_ENV=production
+ENV NEXT_TELEMETRY_DISABLED=1
WORKDIR /app/src/interface/web
-RUN bash -c "yarn install --frozen-lockfile && yarn ciexport && yarn cache clean"
+# Install dependencies first (cache layer)
+COPY src/interface/web/package.json src/interface/web/yarn.lock ./
+RUN yarn install --frozen-lockfile
+# Copy source and build
+COPY src/interface/web/. ./
+RUN yarn build
+
+# Merge the Server and Web App into a Single Image
+FROM base
+ENV PYTHONPATH=/app/src
WORKDIR /app
+COPY --from=server-deps /usr/local/lib/python3.10/dist-packages /usr/local/lib/python3.10/dist-packages
+COPY --from=web-app /app/src/interface/web/out ./src/khoj/interface/built
+COPY . .
+RUN cd src && python3 khoj/manage.py collectstatic --noinput
# Run the Application
# There are more arguments required for the application to run,
-# but these should be passed in through the docker-compose.yml file.
+# but those should be passed in through the docker-compose.yml file.
ARG PORT
EXPOSE ${PORT}
ENTRYPOINT ["python3", "src/khoj/main.py"]
diff --git a/prod.Dockerfile b/prod.Dockerfile
index 8c467b93..21468c91 100644
--- a/prod.Dockerfile
+++ b/prod.Dockerfile
@@ -1,5 +1,5 @@
# syntax=docker/dockerfile:1
-FROM ubuntu:jammy
+FROM ubuntu:jammy AS base
LABEL homepage="https://khoj.dev"
LABEL repository="https://github.com/khoj-ai/khoj"
LABEL org.opencontainers.image.source="https://github.com/khoj-ai/khoj"
@@ -16,38 +16,48 @@ RUN apt update -y && apt -y install \
curl \
# Required by llama-cpp-python pre-built wheels. See #1628
musl-dev && \
- # Required by Next.js Web app
- curl -sL https://deb.nodesource.com/setup_20.x | bash - && \
- curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \
- echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \
- apt update -y && apt -y --no-install-recommends install nodejs yarn && \
- apt clean && rm -rf /var/lib/apt/lists/* && \
- # Required by llama-cpp-python pre-built wheels. See #1628
- ln -s /usr/lib/x86_64-linux-musl/libc.so /lib/libc.musl-x86_64.so.1
+ ln -s /usr/lib/x86_64-linux-musl/libc.so /lib/libc.musl-x86_64.so.1 && \
+ # Clean up
+ apt clean && rm -rf /var/lib/apt/lists/*
-# Install Application
+# Build Server
+FROM base AS server-deps
WORKDIR /app
COPY pyproject.toml .
COPY README.md .
ARG VERSION=0.0.0
-ENV PIP_EXTRA_INDEX_URL=https://abetlen.github.io/llama-cpp-python/whl/cpu
+# use the pre-built llama-cpp-python, torch cpu wheel
+ENV PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu https://abetlen.github.io/llama-cpp-python/whl/cpu"
+# avoid downloading unused cuda specific python packages
+ENV CUDA_VISIBLE_DEVICES=""
RUN sed -i "s/dynamic = \\[\"version\"\\]/version = \"$VERSION\"/" pyproject.toml && \
- pip install --no-cache-dir -e .[prod]
+ pip install --no-cache-dir .[prod]
-# Copy Source Code
-COPY . .
-
-# Set the PYTHONPATH environment variable in order for it to find the Django app.
-ENV PYTHONPATH=/app/src:$PYTHONPATH
-
-# Go to the directory src/interface/web and export the built Next.js assets
+# Build Web App
+FROM node:20-alpine AS web-app
+# Set build optimization env vars
+ENV NODE_ENV=production
+ENV NEXT_TELEMETRY_DISABLED=1
WORKDIR /app/src/interface/web
-RUN bash -c "yarn install --frozen-lockfile && yarn ciexport && yarn cache clean"
+# Install dependencies first (cache layer)
+COPY src/interface/web/package.json src/interface/web/yarn.lock ./
+RUN yarn install --frozen-lockfile
+# Copy source and build
+COPY src/interface/web/. ./
+RUN yarn build
+
+# Merge the Server and Web App into a Single Image
+FROM base
+ENV PYTHONPATH=/app/src
WORKDIR /app
+COPY --from=server-deps /usr/local/lib/python3.10/dist-packages /usr/local/lib/python3.10/dist-packages
+COPY --from=web-app /app/src/interface/web/out ./src/khoj/interface/built
+COPY . .
+RUN cd src && python3 khoj/manage.py collectstatic --noinput
# Run the Application
# There are more arguments required for the application to run,
-# but these should be passed in through the docker-compose.yml file.
+# but those should be passed in through the docker-compose.yml file.
ARG PORT
EXPOSE ${PORT}
ENTRYPOINT ["gunicorn", "-c", "gunicorn-config.py", "src.khoj.main:app"]
diff --git a/src/interface/web/app/agents/layout.tsx b/src/interface/web/app/agents/layout.tsx
index b30c782d..c7779b06 100644
--- a/src/interface/web/app/agents/layout.tsx
+++ b/src/interface/web/app/agents/layout.tsx
@@ -1,9 +1,7 @@
import type { Metadata } from "next";
-import { Noto_Sans } from "next/font/google";
+import { noto_sans, noto_sans_arabic } from "@/app/fonts";
import "../globals.css";
-const inter = Noto_Sans({ subsets: ["latin"] });
-
export const metadata: Metadata = {
title: "Khoj AI - Agents",
description: "Find a specialized agent that can help you address more specific needs.",
@@ -33,7 +31,7 @@ export default function RootLayout({
children: React.ReactNode;
}>) {
return (
-
+
- {children}
+ {children}
);
}
diff --git a/src/interface/web/app/chat/layout.tsx b/src/interface/web/app/chat/layout.tsx
index 09c3afb7..4e4661f1 100644
--- a/src/interface/web/app/chat/layout.tsx
+++ b/src/interface/web/app/chat/layout.tsx
@@ -1,9 +1,7 @@
import type { Metadata } from "next";
-import { Noto_Sans } from "next/font/google";
+import { noto_sans, noto_sans_arabic } from "@/app/fonts";
import "../globals.css";
-const inter = Noto_Sans({ subsets: ["latin"] });
-
export const metadata: Metadata = {
title: "Khoj AI - Chat",
description:
@@ -34,7 +32,7 @@ export default function RootLayout({
children: React.ReactNode;
}>) {
return (
-
+
-
+
{children}