diff --git a/.github/workflows/dockerize.yml b/.github/workflows/dockerize.yml index 70098040..b9673e97 100644 --- a/.github/workflows/dockerize.yml +++ b/.github/workflows/dockerize.yml @@ -8,23 +8,47 @@ on: - master paths: - src/khoj/** - - config/** - pyproject.toml - Dockerfile + - prod.Dockerfile - docker-compose.yml - .github/workflows/dockerize.yml workflow_dispatch: + inputs: + tag: + description: 'Docker image tag' + default: 'dev' + khoj: + description: 'Build Khoj docker image' + type: boolean + default: true + khoj-cloud: + description: 'Build Khoj cloud docker image' + type: boolean + default: true env: - DOCKER_IMAGE_TAG: ${{ github.ref == 'refs/heads/master' && 'latest' || github.ref_name }} + # Tag Image with tag name on release + # else with user specified tag (default 'dev') if triggered via workflow + # else with 'pre' (if push to master) + DOCKER_IMAGE_TAG: ${{ github.ref_type == 'tag' && github.ref_name || github.event_name == 'workflow_dispatch' && github.event.inputs.tag || 'pre' }} jobs: build: - name: Build Docker Image, Push to Container Registry + name: Publish Khoj Docker Images runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image: + - 'local' + - 'cloud' steps: - name: Checkout Code uses: actions/checkout@v3 + with: + # Get all history to correctly infer Khoj version using hatch + fetch-depth: 0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -36,13 +60,36 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.PAT }} + - name: Get App Version + id: hatch + run: echo "version=$(pipx run hatch version)" >> $GITHUB_OUTPUT + - name: 📦 Build and Push Docker Image uses: docker/build-push-action@v2 + if: (matrix.image == 'local' && github.event_name == 'workflow_dispatch') && github.event.inputs.khoj == 'true' || (matrix.image == 'local' && github.event_name == 'push') with: context: . file: Dockerfile platforms: linux/amd64, linux/arm64 push: true - tags: ghcr.io/${{ github.repository }}:${{ env.DOCKER_IMAGE_TAG }} + tags: | + ghcr.io/${{ github.repository }}:${{ env.DOCKER_IMAGE_TAG }} + ${{ github.ref_type == 'tag' && format('ghcr.io/{0}:latest', github.repository) || '' }} build-args: | + VERSION=${{ steps.hatch.outputs.version }} + PORT=42110 + + - name: 📦️⛅️ Build and Push Cloud Docker Image + uses: docker/build-push-action@v2 + if: (matrix.image == 'cloud' && github.event_name == 'workflow_dispatch') && github.event.inputs.khoj-cloud == 'true' || (matrix.image == 'cloud' && github.event_name == 'push') + with: + context: . + file: prod.Dockerfile + platforms: linux/amd64 + push: true + tags: | + ghcr.io/${{ github.repository }}-cloud:${{ env.DOCKER_IMAGE_TAG }} + ${{ github.ref_type == 'tag' && format('ghcr.io/{0}-cloud:latest', github.repository) || '' }} + build-args: | + VERSION=${{ steps.hatch.outputs.version }} PORT=42110 diff --git a/.github/workflows/dockerize_production.yml b/.github/workflows/dockerize_production.yml deleted file mode 100644 index c4ed963e..00000000 --- a/.github/workflows/dockerize_production.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: dockerize production - -on: - pull_request: - paths: - - src/khoj/** - - pyproject.toml - - prod.Dockerfile - - .github/workflows/dockerize_production.yml - push: - tags: - - "*" - branches: - - master - paths: - - src/khoj/** - - pyproject.toml - - prod.Dockerfile - - .github/workflows/dockerize_production.yml - workflow_dispatch: - -env: - DOCKER_IMAGE_TAG: ${{ github.event_name == 'pull_request' && 'dev' || (github.ref == 'refs/heads/master' && 'latest' || github.ref_name) }} - -jobs: - build: - name: Build Production Docker Image, Push to Container Registry - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.PAT }} - - - name: 📦 Build and Push Docker Image - uses: docker/build-push-action@v2 - with: - context: . - file: prod.Dockerfile - platforms: linux/amd64 - push: true - tags: ghcr.io/${{ github.repository }}-cloud:${{ env.DOCKER_IMAGE_TAG }} - build-args: | - PORT=42110 diff --git a/Dockerfile b/Dockerfile index 9882a236..e6a6fb4a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,8 @@ WORKDIR /app # Install Application COPY pyproject.toml . COPY README.md . -RUN sed -i 's/dynamic = \["version"\]/version = "0.0.0"/' pyproject.toml && \ +ARG VERSION=0.0.0 +RUN sed -i "s/dynamic = \\[\"version\"\\]/version = \"$VERSION\"/" pyproject.toml && \ pip install --no-cache-dir . # Copy Source Code diff --git a/config/khoj_docker.yml b/config/khoj_docker.yml deleted file mode 100644 index 5fb89665..00000000 --- a/config/khoj_docker.yml +++ /dev/null @@ -1,51 +0,0 @@ -content-type: - # The /data/folder/ prefix to the folders is here because this is - # the directory to which the local files are copied in the docker-compose. - # If changing, the docker-compose volumes should also be changed to match. - org: - input-files: null - input-filter: ["/data/org/**/*.org"] - compressed-jsonl: "/data/embeddings/notes.jsonl.gz" - embeddings-file: "/data/embeddings/note_embeddings.pt" - index_heading_entries: false - - markdown: - input-files: null - input-filter: ["/data/markdown/**/*.markdown"] - compressed-jsonl: "/data/embeddings/markdown.jsonl.gz" - embeddings-file: "/data/embeddings/markdown_embeddings.pt" - - pdf: - input-files: null - input-filter: ["/data/pdf/**/*.pdf"] - compressed-jsonl: "/data/embeddings/pdf.jsonl.gz" - embeddings-file: "/data/embeddings/pdf_embeddings.pt" - - image: - input-directories: ["/data/images/"] - embeddings-file: "/data/embeddings/image_embeddings.pt" - batch-size: 50 - use-xmp-metadata: false - - notion: null - github: null - plugins: null - -search-type: - symmetric: null - asymmetric: - encoder: "sentence-transformers/multi-qa-MiniLM-L6-cos-v1" - cross-encoder: "cross-encoder/ms-marco-MiniLM-L-6-v2" - model_directory: "/data/models/asymmetric" - image: - encoder: "sentence-transformers/clip-ViT-B-32" - model_directory: "/data/models/image_encoder" - -processor: - conversation: - conversation-logfile: "/data/embeddings/conversation_logs.json" - enable-offline-chat: false - openai: null - -app: - should_log_telemetry: true diff --git a/config/khoj_sample.yml b/config/khoj_sample.yml deleted file mode 100644 index a30b02d9..00000000 --- a/config/khoj_sample.yml +++ /dev/null @@ -1,57 +0,0 @@ -content-type: - org: - input-files: # ["/path/to/org-file.org"] REQUIRED IF input-filter IS NOT SET OR - input-filter: # ["/path/to/org/*.org"] REQUIRED IF input-files IS NOT SET - compressed-jsonl: "~/.khoj/content/org/org.jsonl.gz" - embeddings-file: "~/.khoj/content/org/org_embeddings.pt" - index_heading_entries: false # Set to true to index entries with empty body - - markdown: - input-files: # ["/path/to/markdown-file.md"] REQUIRED IF input-filter IS NOT SET OR - input-filter: # ["/path/to/markdown/*.md"] REQUIRED IF input-files IS NOT SET - compressed-jsonl: "~/.khoj/content/markdown/markdown.jsonl.gz" - embeddings-file: "~/.khoj/content/markdown/markdown_embeddings.pt" - - ledger: - input-files: # ["/path/to/ledger-file.beancount"] REQUIRED IF input-filter is not set OR - input-filter: # ["/path/to/ledger/*.beancount"] REQUIRED IF input-files is not set - compressed-jsonl: "~/.khoj/content/ledger/ledger.jsonl.gz" - embeddings-file: "~/.khoj/content/ledger/ledger_embeddings.pt" - - image: - input-directories: # ["/path/to/images/"] REQUIRED IF input-filter IS NOT SET OR - input-filter: # ["/path/to/images/*.jpg"] REQUIRED IF input-directories IS NOT SET - embeddings-file: "~/.khoj/content/image/image_embeddings.pt" - batch-size: 50 - use-xmp-metadata: false - - music: - input-files: # ["/path/to/music-file.org"] REQUIRED IF input-filter IS NOT SET OR - input-filter: # ["/path/to/music/*.org"] REQUIRED IF input-files IS NOT SET - compressed-jsonl: "~/.khoj/content/music/music.jsonl.gz" - embeddings-file: "~/.khoj/content/music/music_embeddings.pt" - -search-type: - symmetric: - encoder: "sentence-transformers/all-MiniLM-L6-v2" - cross-encoder: "cross-encoder/ms-marco-MiniLM-L-6-v2" - encoder-type: sentence_transformers.SentenceTransformer - model_directory: "~/.khoj/search/symmetric/" - - asymmetric: - encoder: "sentence-transformers/multi-qa-MiniLM-L6-cos-v1" - cross-encoder: "cross-encoder/ms-marco-MiniLM-L-6-v2" - encoder-type: sentence_transformers.SentenceTransformer - model_directory: "~/.khoj/search/asymmetric/" - - image: - encoder: "sentence-transformers/clip-ViT-B-32" - encoder-type: sentence_transformers.SentenceTransformer - model_directory: "~/.khoj/search/image/" - -processor: - conversation: - openai-api-key: # "YOUR_OPENAI_API_KEY" - model: "text-davinci-003" - chat-model: "gpt-3.5-turbo" - conversation-logfile: "~/.khoj/processor/conversation/conversation_logs.json" diff --git a/prod.Dockerfile b/prod.Dockerfile index 693a3a8b..8b21cb66 100644 --- a/prod.Dockerfile +++ b/prod.Dockerfile @@ -11,7 +11,8 @@ WORKDIR /app # Install Application COPY pyproject.toml . COPY README.md . -RUN sed -i 's/dynamic = \["version"\]/version = "0.0.0"/' pyproject.toml && \ +ARG VERSION=0.0.0 +RUN sed -i "s/dynamic = \\[\"version\"\\]/version = \"$VERSION\"/" pyproject.toml && \ TMPDIR=/home/cache/ pip install --cache-dir=/home/cache/ -e . # Copy Source Code diff --git a/src/interface/desktop/chat.html b/src/interface/desktop/chat.html index b1d0ce48..fbd09daa 100644 --- a/src/interface/desktop/chat.html +++ b/src/interface/desktop/chat.html @@ -115,10 +115,10 @@ return referenceButton; } - function renderMessage(message, by, dt=null, annotations=null) { + function renderMessage(message, by, dt=null, annotations=null, raw=false) { let message_time = formatDate(dt ?? new Date()); let by_name = by == "khoj" ? "🏮 Khoj" : "🤔 You"; - let formattedMessage = formatHTMLMessage(message); + let formattedMessage = formatHTMLMessage(message, raw); let chatBody = document.getElementById("chat-body"); // Create a new div for the chat message @@ -248,7 +248,7 @@ renderMessage(message, by, dt, references); } - function formatHTMLMessage(htmlMessage) { + function formatHTMLMessage(htmlMessage, raw=false) { var md = window.markdownit(); let newHTML = htmlMessage; @@ -267,7 +267,7 @@ }; // Render markdown - newHTML = md.render(newHTML); + newHTML = raw ? newHTML : md.render(newHTML); // Get any elements with a class that starts with "language" let element = document.createElement('div'); element.innerHTML = newHTML; @@ -574,7 +574,7 @@ .trim() .replace(/(\r\n|\n|\r)/gm, ""); - renderMessage(first_run_message, "khoj"); + renderMessage(first_run_message, "khoj", null, null, true); // Disable chat input field and update placeholder text document.getElementById("chat-input").setAttribute("disabled", "disabled"); diff --git a/src/interface/obsidian/src/chat_modal.ts b/src/interface/obsidian/src/chat_modal.ts index 115f4c1f..6f990b74 100644 --- a/src/interface/obsidian/src/chat_modal.ts +++ b/src/interface/obsidian/src/chat_modal.ts @@ -41,20 +41,21 @@ export class KhojChatModal extends Modal { let chatBodyEl = contentEl.createDiv({ attr: { id: "khoj-chat-body", class: "khoj-chat-body" } }); // Get chat history from Khoj backend - await this.getChatHistory(chatBodyEl); + let getChatHistorySucessfully = await this.getChatHistory(chatBodyEl); + let placeholderText = getChatHistorySucessfully ? "Chat with Khoj [Hit Enter to send message]" : "Configure Khoj to enable chat"; // Add chat input field let inputRow = contentEl.createDiv("khoj-input-row"); - const chatInput = inputRow.createEl("input", - { - attr: { - type: "text", - id: "khoj-chat-input", - autofocus: "autofocus", - placeholder: "Chat with Khoj [Hit Enter to send message]", - class: "khoj-chat-input option" - } - }) + let chatInput = inputRow.createEl("input", { + attr: { + type: "text", + id: "khoj-chat-input", + autofocus: "autofocus", + placeholder: placeholderText, + class: "khoj-chat-input option", + disabled: !getChatHistorySucessfully ? "disabled" : null + }, + }) let transcribe = inputRow.createEl("button", { text: "Transcribe", @@ -162,7 +163,7 @@ export class KhojChatModal extends Modal { referenceExpandButton.innerHTML = expandButtonText; } - renderMessage(chatEl: Element, message: string, sender: string, dt?: Date): Element { + renderMessage(chatEl: Element, message: string, sender: string, dt?: Date, raw: boolean=false): Element { let message_time = this.formatDate(dt ?? new Date()); let emojified_sender = sender == "khoj" ? "🏮 Khoj" : "🤔 You"; @@ -177,8 +178,12 @@ export class KhojChatModal extends Modal { let chat_message_body_el = chatMessageEl.createDiv(); chat_message_body_el.addClasses(["khoj-chat-message-text", sender]); let chat_message_body_text_el = chat_message_body_el.createDiv(); - // @ts-ignore - MarkdownRenderer.renderMarkdown(message, chat_message_body_text_el, null, null); + if (raw) { + chat_message_body_text_el.innerHTML = message; + } else { + // @ts-ignore + MarkdownRenderer.renderMarkdown(message, chat_message_body_text_el, null, null); + } // Remove user-select: none property to make text selectable chatMessageEl.style.userSelect = "text"; @@ -212,11 +217,11 @@ export class KhojChatModal extends Modal { return chat_message_el } - renderIncrementalMessage(htmlElement: HTMLDivElement, additionalMessage: string) { + async renderIncrementalMessage(htmlElement: HTMLDivElement, additionalMessage: string) { this.result += additionalMessage; htmlElement.innerHTML = ""; // @ts-ignore - MarkdownRenderer.renderMarkdown(this.result, htmlElement, null, null); + await MarkdownRenderer.renderMarkdown(this.result, htmlElement, null, null); // Scroll to bottom of modal, till the send message input box this.modalEl.scrollTop = this.modalEl.scrollHeight; } @@ -228,15 +233,33 @@ export class KhojChatModal extends Modal { return `${time_string}, ${date_string}`; } - async getChatHistory(chatBodyEl: Element): Promise { + async getChatHistory(chatBodyEl: Element): Promise { // Get chat history from Khoj backend let chatUrl = `${this.setting.khojUrl}/api/chat/history?client=obsidian`; let headers = { "Authorization": `Bearer ${this.setting.khojApiKey}` }; - let response = await request({ url: chatUrl, headers: headers }); - let chatLogs = JSON.parse(response).response; - chatLogs.forEach((chatLog: any) => { - this.renderMessageWithReferences(chatBodyEl, chatLog.message, chatLog.by, chatLog.context, new Date(chatLog.created), chatLog.intent?.type); - }); + + try { + let response = await fetch(chatUrl, { method: "GET", headers: headers }); + let responseJson: any = await response.json(); + + if (responseJson.detail) { + // If the server returns error details in response, render a setup hint. + let setupMsg = "Hi 👋🏾, to start chatting add available chat models options via [the Django Admin panel](/server/admin) on the Server"; + this.renderMessage(chatBodyEl, setupMsg, "khoj", undefined, true); + + return false; + } else if (responseJson.response) { + let chatLogs = responseJson.response; + chatLogs.forEach((chatLog: any) => { + this.renderMessageWithReferences(chatBodyEl, chatLog.message, chatLog.by, chatLog.context, new Date(chatLog.created), chatLog.intent?.type); + }); + } + } catch (err) { + let errorMsg = "Unable to get response from Khoj server ❤️‍🩹. Ensure server is running or contact developers for help at [team@khoj.dev](mailto:team@khoj.dev) or in [Discord](https://discord.gg/BDgyabRM6e)"; + this.renderMessage(chatBodyEl, errorMsg, "khoj", undefined); + return false; + } + return true; } async getChatResponse(query: string | undefined | null): Promise { @@ -254,7 +277,7 @@ export class KhojChatModal extends Modal { // Temporary status message to indicate that Khoj is thinking this.result = ""; - this.renderIncrementalMessage(responseElement, "🤔"); + await this.renderIncrementalMessage(responseElement, "🤔"); let response = await fetch(chatUrl, { method: "GET", @@ -289,17 +312,17 @@ export class KhojChatModal extends Modal { // If the chunk is not a JSON object, just display it as is responseText = response.body.read().toString() } finally { - this.renderIncrementalMessage(responseElement, responseText); + await this.renderIncrementalMessage(responseElement, responseText); } } for await (const chunk of response.body) { let responseText = chunk.toString(); if (responseText.includes("### compiled references:")) { - const additionalResponse = responseText.split("### compiled references:")[0]; - this.renderIncrementalMessage(responseElement, additionalResponse); + const [additionalResponse, rawReference] = responseText.split("### compiled references:", 2); + await this.renderIncrementalMessage(responseElement, additionalResponse); + console.log(`Raw: ${responseText}\nResponse: ${additionalResponse}\nReferences: ${rawReference}`); - const rawReference = responseText.split("### compiled references:")[1]; const rawReferenceAsJson = JSON.parse(rawReference); let references = responseElement.createDiv(); references.classList.add("references"); @@ -337,17 +360,12 @@ export class KhojChatModal extends Modal { referenceExpandButton.innerHTML = expandButtonText; references.appendChild(referenceSection); } else { - if (responseText.startsWith("{") && responseText.endsWith("}")) { - } else { - // If the chunk is not a JSON object, just display it as is - continue; - } - - this.renderIncrementalMessage(responseElement, responseText); + await this.renderIncrementalMessage(responseElement, responseText); } } } catch (err) { - this.renderIncrementalMessage(responseElement, "Sorry, unable to get response from Khoj backend ❤️‍🩹. Contact developer for help at team@khoj.dev or in Discord") + let errorMsg = "Sorry, unable to get response from Khoj backend ❤️‍🩹. Contact developer for help at team@khoj.dev or [in Discord](https://discord.gg/BDgyabRM6e)"; + responseElement.innerHTML = errorMsg } } @@ -377,10 +395,11 @@ export class KhojChatModal extends Modal { // Throw error if conversation history isn't cleared throw new Error("Failed to clear conversation history"); } else { + let getChatHistoryStatus = await this.getChatHistory(chatBody); // If conversation history is cleared successfully, clear chat logs from modal - chatBody.innerHTML = ""; - await this.getChatHistory(chatBody); - this.flashStatusInChatInput(result.message); + if (getChatHistoryStatus) chatBody.innerHTML = ""; + let statusMsg = getChatHistoryStatus ? result.message : "Failed to clear conversation history"; + this.flashStatusInChatInput(statusMsg); } } catch (err) { this.flashStatusInChatInput("Failed to clear conversation history"); diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html index f17e9b11..a751ee0b 100644 --- a/src/khoj/interface/web/chat.html +++ b/src/khoj/interface/web/chat.html @@ -17,7 +17,7 @@ Hi, I am Khoj, your open, personal AI 👋🏽. I can help: - 💡 Be a sounding board for your ideas - 📜 Chat with your notes & documents -Download the [🖥️ Desktop app](https://khoj.dev/downloads) to chat with your computer docs. +Get the Khoj [Desktop](https://khoj.dev/downloads), [Obsidian](https://docs.khoj.dev/#/obsidian?id=setup) or [Emacs](https://docs.khoj.dev/#/emacs?id=setup) app to search, chat with your 🖥️ computer docs. To get started, just start typing below. You can also type / to see a list of commands. `.trim() @@ -124,10 +124,10 @@ To get started, just start typing below. You can also type / to see a list of co return referenceButton; } - function renderMessage(message, by, dt=null, annotations=null) { + function renderMessage(message, by, dt=null, annotations=null, raw=false) { let message_time = formatDate(dt ?? new Date()); let by_name = by == "khoj" ? "🏮 Khoj" : "🤔 You"; - let formattedMessage = formatHTMLMessage(message); + let formattedMessage = formatHTMLMessage(message, raw); let chatBody = document.getElementById("chat-body"); // Create a new div for the chat message @@ -257,7 +257,7 @@ To get started, just start typing below. You can also type / to see a list of co renderMessage(message, by, dt, references); } - function formatHTMLMessage(htmlMessage) { + function formatHTMLMessage(htmlMessage, raw=false) { var md = window.markdownit(); let newHTML = htmlMessage; @@ -276,7 +276,7 @@ To get started, just start typing below. You can also type / to see a list of co }; // Render markdown - newHTML = md.render(newHTML); + newHTML = raw ? newHTML : md.render(newHTML); // Get any elements with a class that starts with "language" let element = document.createElement('div'); element.innerHTML = newHTML; @@ -435,9 +435,9 @@ To get started, just start typing below. You can also type / to see a list of co numReferences = rawReferenceAsJson.length; rawReferenceAsJson.forEach((reference, index) => { - let polishedReference = generateReference(reference, index); - referenceSection.appendChild(polishedReference); - }); + let polishedReference = generateReference(reference, index); + referenceSection.appendChild(polishedReference); + }); } else { numReferences += processOnlineReferences(referenceSection, rawReferenceAsJson); } @@ -539,7 +539,8 @@ To get started, just start typing below. You can also type / to see a list of co .then(data => { if (data.detail) { // If the server returns a 500 error with detail, render a setup hint. - renderMessage("Hi 👋🏾, to start chatting add available chat models options via the Django Admin panel on the Server", "khoj"); + let setupMsg = "Hi 👋🏾, to start chatting add available chat models options via the Django Admin panel on the Server"; + renderMessage(setupMsg, "khoj", null, null, true); // Disable chat input field and update placeholder text document.getElementById("chat-input").setAttribute("disabled", "disabled"); diff --git a/src/khoj/interface/web/content_source_computer_input.html b/src/khoj/interface/web/content_source_computer_input.html index 72aa3810..6c245079 100644 --- a/src/khoj/interface/web/content_source_computer_input.html +++ b/src/khoj/interface/web/content_source_computer_input.html @@ -7,7 +7,7 @@ Files

Manage files from your computer

-

Download the Khoj Desktop app to sync documents from your computer

+

Get the Khoj Desktop, Obsidian or Emacs app to sync documents from your computer