From e04fe921eb47347e74c3afcfe86a6f59dc159941 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Mon, 18 Dec 2023 20:31:50 +0530 Subject: [PATCH 1/9] Fix first-run, chat error message in obsidian, desktop and web clients - Disable chat input field if getChatHistory had error as Khoj may not be setup correctly to chat --- src/interface/desktop/chat.html | 10 ++-- src/interface/obsidian/src/chat_modal.ts | 71 ++++++++++++++++-------- src/khoj/interface/web/chat.html | 11 ++-- 3 files changed, 59 insertions(+), 33 deletions(-) diff --git a/src/interface/desktop/chat.html b/src/interface/desktop/chat.html index b1d0ce48..fbd09daa 100644 --- a/src/interface/desktop/chat.html +++ b/src/interface/desktop/chat.html @@ -115,10 +115,10 @@ return referenceButton; } - function renderMessage(message, by, dt=null, annotations=null) { + function renderMessage(message, by, dt=null, annotations=null, raw=false) { let message_time = formatDate(dt ?? new Date()); let by_name = by == "khoj" ? "🏮 Khoj" : "🤔 You"; - let formattedMessage = formatHTMLMessage(message); + let formattedMessage = formatHTMLMessage(message, raw); let chatBody = document.getElementById("chat-body"); // Create a new div for the chat message @@ -248,7 +248,7 @@ renderMessage(message, by, dt, references); } - function formatHTMLMessage(htmlMessage) { + function formatHTMLMessage(htmlMessage, raw=false) { var md = window.markdownit(); let newHTML = htmlMessage; @@ -267,7 +267,7 @@ }; // Render markdown - newHTML = md.render(newHTML); + newHTML = raw ? newHTML : md.render(newHTML); // Get any elements with a class that starts with "language" let element = document.createElement('div'); element.innerHTML = newHTML; @@ -574,7 +574,7 @@ .trim() .replace(/(\r\n|\n|\r)/gm, ""); - renderMessage(first_run_message, "khoj"); + renderMessage(first_run_message, "khoj", null, null, true); // Disable chat input field and update placeholder text document.getElementById("chat-input").setAttribute("disabled", "disabled"); diff --git a/src/interface/obsidian/src/chat_modal.ts b/src/interface/obsidian/src/chat_modal.ts index 115f4c1f..100fd853 100644 --- a/src/interface/obsidian/src/chat_modal.ts +++ b/src/interface/obsidian/src/chat_modal.ts @@ -41,20 +41,21 @@ export class KhojChatModal extends Modal { let chatBodyEl = contentEl.createDiv({ attr: { id: "khoj-chat-body", class: "khoj-chat-body" } }); // Get chat history from Khoj backend - await this.getChatHistory(chatBodyEl); + let getChatHistorySucessfully = await this.getChatHistory(chatBodyEl); + let placeholderText = getChatHistorySucessfully ? "Chat with Khoj [Hit Enter to send message]" : "Configure Khoj to enable chat"; // Add chat input field let inputRow = contentEl.createDiv("khoj-input-row"); - const chatInput = inputRow.createEl("input", - { - attr: { - type: "text", - id: "khoj-chat-input", - autofocus: "autofocus", - placeholder: "Chat with Khoj [Hit Enter to send message]", - class: "khoj-chat-input option" - } - }) + let chatInput = inputRow.createEl("input", { + attr: { + type: "text", + id: "khoj-chat-input", + autofocus: "autofocus", + placeholder: placeholderText, + class: "khoj-chat-input option", + disabled: !getChatHistorySucessfully ? "disabled" : null + }, + }) let transcribe = inputRow.createEl("button", { text: "Transcribe", @@ -162,7 +163,7 @@ export class KhojChatModal extends Modal { referenceExpandButton.innerHTML = expandButtonText; } - renderMessage(chatEl: Element, message: string, sender: string, dt?: Date): Element { + renderMessage(chatEl: Element, message: string, sender: string, dt?: Date, raw: boolean=false): Element { let message_time = this.formatDate(dt ?? new Date()); let emojified_sender = sender == "khoj" ? "🏮 Khoj" : "🤔 You"; @@ -177,8 +178,12 @@ export class KhojChatModal extends Modal { let chat_message_body_el = chatMessageEl.createDiv(); chat_message_body_el.addClasses(["khoj-chat-message-text", sender]); let chat_message_body_text_el = chat_message_body_el.createDiv(); - // @ts-ignore - MarkdownRenderer.renderMarkdown(message, chat_message_body_text_el, null, null); + if (raw) { + chat_message_body_text_el.innerHTML = message; + } else { + // @ts-ignore + MarkdownRenderer.renderMarkdown(message, chat_message_body_text_el, null, null); + } // Remove user-select: none property to make text selectable chatMessageEl.style.userSelect = "text"; @@ -228,15 +233,33 @@ export class KhojChatModal extends Modal { return `${time_string}, ${date_string}`; } - async getChatHistory(chatBodyEl: Element): Promise { + async getChatHistory(chatBodyEl: Element): Promise { // Get chat history from Khoj backend let chatUrl = `${this.setting.khojUrl}/api/chat/history?client=obsidian`; let headers = { "Authorization": `Bearer ${this.setting.khojApiKey}` }; - let response = await request({ url: chatUrl, headers: headers }); - let chatLogs = JSON.parse(response).response; - chatLogs.forEach((chatLog: any) => { - this.renderMessageWithReferences(chatBodyEl, chatLog.message, chatLog.by, chatLog.context, new Date(chatLog.created), chatLog.intent?.type); - }); + + try { + let response = await fetch(chatUrl, { method: "GET", headers: headers }); + let responseJson: any = await response.json(); + + if (responseJson.detail) { + // If the server returns error details in response, render a setup hint. + let setupMsg = "Hi 👋🏾, to start chatting add available chat models options via the Django Admin panel on the Server"; + this.renderMessage(chatBodyEl, setupMsg, "khoj", undefined, true); + + return false; + } else if (responseJson.response) { + let chatLogs = responseJson.response; + chatLogs.forEach((chatLog: any) => { + this.renderMessageWithReferences(chatBodyEl, chatLog.message, chatLog.by, chatLog.context, new Date(chatLog.created), chatLog.intent?.type); + }); + } + } catch (err) { + let errorMsg = "Unable to get response from Khoj server ❤️‍🩹. Ensure server is running or contact developers for help at team@khoj.dev or on Discord"; + this.renderMessage(chatBodyEl, errorMsg, "khoj", undefined, true); + return false; + } + return true; } async getChatResponse(query: string | undefined | null): Promise { @@ -347,7 +370,8 @@ export class KhojChatModal extends Modal { } } } catch (err) { - this.renderIncrementalMessage(responseElement, "Sorry, unable to get response from Khoj backend ❤️‍🩹. Contact developer for help at team@khoj.dev or in Discord") + let errorMsg = "

Sorry, unable to get response from Khoj backend ❤️‍🩹. Contact developer for help at team@khoj.dev or in Discord

"; + responseElement.innerHTML = errorMsg } } @@ -379,8 +403,9 @@ export class KhojChatModal extends Modal { } else { // If conversation history is cleared successfully, clear chat logs from modal chatBody.innerHTML = ""; - await this.getChatHistory(chatBody); - this.flashStatusInChatInput(result.message); + let getChatHistoryStatus = await this.getChatHistory(chatBody); + let statusMsg = getChatHistoryStatus ? result.message : "Failed to clear conversation history"; + this.flashStatusInChatInput(statusMsg); } } catch (err) { this.flashStatusInChatInput("Failed to clear conversation history"); diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html index 2c64d96c..7e6b157f 100644 --- a/src/khoj/interface/web/chat.html +++ b/src/khoj/interface/web/chat.html @@ -124,10 +124,10 @@ To get started, just start typing below. You can also type / to see a list of co return referenceButton; } - function renderMessage(message, by, dt=null, annotations=null) { + function renderMessage(message, by, dt=null, annotations=null, raw=false) { let message_time = formatDate(dt ?? new Date()); let by_name = by == "khoj" ? "🏮 Khoj" : "🤔 You"; - let formattedMessage = formatHTMLMessage(message); + let formattedMessage = formatHTMLMessage(message, raw); let chatBody = document.getElementById("chat-body"); // Create a new div for the chat message @@ -257,7 +257,7 @@ To get started, just start typing below. You can also type / to see a list of co renderMessage(message, by, dt, references); } - function formatHTMLMessage(htmlMessage) { + function formatHTMLMessage(htmlMessage, raw=false) { var md = window.markdownit(); let newHTML = htmlMessage; @@ -276,7 +276,7 @@ To get started, just start typing below. You can also type / to see a list of co }; // Render markdown - newHTML = md.render(newHTML); + newHTML = raw ? newHTML : md.render(newHTML); // Get any elements with a class that starts with "language" let element = document.createElement('div'); element.innerHTML = newHTML; @@ -539,7 +539,8 @@ To get started, just start typing below. You can also type / to see a list of co .then(data => { if (data.detail) { // If the server returns a 500 error with detail, render a setup hint. - renderMessage("Hi 👋🏾, to start chatting add available chat models options via the Django Admin panel on the Server", "khoj"); + let setupMsg = "Hi 👋🏾, to start chatting add available chat models options via the Django Admin panel on the Server"; + renderMessage(setupMsg, "khoj", null, null, true); // Disable chat input field and update placeholder text document.getElementById("chat-input").setAttribute("disabled", "disabled"); From 447c1b90e74b0345ef94dbd615dd7a332b1dafac Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Wed, 20 Dec 2023 14:50:06 +0530 Subject: [PATCH 2/9] Fix streaming chat response in Obsidian client - Convert renderIncrementalMessage to an async method as MarkdownRenderer is an async method - Simplify code, remove unneeded JSON check --- src/interface/obsidian/src/chat_modal.ts | 22 ++++++++-------------- src/khoj/interface/web/chat.html | 6 +++--- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/interface/obsidian/src/chat_modal.ts b/src/interface/obsidian/src/chat_modal.ts index 100fd853..ae7eada7 100644 --- a/src/interface/obsidian/src/chat_modal.ts +++ b/src/interface/obsidian/src/chat_modal.ts @@ -217,11 +217,11 @@ export class KhojChatModal extends Modal { return chat_message_el } - renderIncrementalMessage(htmlElement: HTMLDivElement, additionalMessage: string) { + async renderIncrementalMessage(htmlElement: HTMLDivElement, additionalMessage: string) { this.result += additionalMessage; htmlElement.innerHTML = ""; // @ts-ignore - MarkdownRenderer.renderMarkdown(this.result, htmlElement, null, null); + await MarkdownRenderer.renderMarkdown(this.result, htmlElement, null, null); // Scroll to bottom of modal, till the send message input box this.modalEl.scrollTop = this.modalEl.scrollHeight; } @@ -277,7 +277,7 @@ export class KhojChatModal extends Modal { // Temporary status message to indicate that Khoj is thinking this.result = ""; - this.renderIncrementalMessage(responseElement, "🤔"); + await this.renderIncrementalMessage(responseElement, "🤔"); let response = await fetch(chatUrl, { method: "GET", @@ -312,17 +312,17 @@ export class KhojChatModal extends Modal { // If the chunk is not a JSON object, just display it as is responseText = response.body.read().toString() } finally { - this.renderIncrementalMessage(responseElement, responseText); + await this.renderIncrementalMessage(responseElement, responseText); } } for await (const chunk of response.body) { let responseText = chunk.toString(); if (responseText.includes("### compiled references:")) { - const additionalResponse = responseText.split("### compiled references:")[0]; - this.renderIncrementalMessage(responseElement, additionalResponse); + const [additionalResponse, rawReference] = responseText.split("### compiled references:", 2); + await this.renderIncrementalMessage(responseElement, additionalResponse); + console.log(`Raw: ${responseText}\nResponse: ${additionalResponse}\nReferences: ${rawReference}`); - const rawReference = responseText.split("### compiled references:")[1]; const rawReferenceAsJson = JSON.parse(rawReference); let references = responseElement.createDiv(); references.classList.add("references"); @@ -360,13 +360,7 @@ export class KhojChatModal extends Modal { referenceExpandButton.innerHTML = expandButtonText; references.appendChild(referenceSection); } else { - if (responseText.startsWith("{") && responseText.endsWith("}")) { - } else { - // If the chunk is not a JSON object, just display it as is - continue; - } - - this.renderIncrementalMessage(responseElement, responseText); + await this.renderIncrementalMessage(responseElement, responseText); } } } catch (err) { diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html index 7e6b157f..b07cd69c 100644 --- a/src/khoj/interface/web/chat.html +++ b/src/khoj/interface/web/chat.html @@ -435,9 +435,9 @@ To get started, just start typing below. You can also type / to see a list of co numReferences = rawReferenceAsJson.length; rawReferenceAsJson.forEach((reference, index) => { - let polishedReference = generateReference(reference, index); - referenceSection.appendChild(polishedReference); - }); + let polishedReference = generateReference(reference, index); + referenceSection.appendChild(polishedReference); + }); } else { numReferences += processOnlineReferences(referenceSection, rawReferenceAsJson); } From 70607cbbbb5127aba585c578fa5a148d62fdcfea Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Thu, 21 Dec 2023 15:23:47 +0530 Subject: [PATCH 3/9] Update FRE message to get any Khoj client to sync files with server --- src/khoj/interface/web/chat.html | 2 +- src/khoj/interface/web/content_source_computer_input.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html index b07cd69c..9878ca0f 100644 --- a/src/khoj/interface/web/chat.html +++ b/src/khoj/interface/web/chat.html @@ -17,7 +17,7 @@ Hi, I am Khoj, your open, personal AI 👋🏽. I can help: - 💡 Be a sounding board for your ideas - 📜 Chat with your notes & documents -Download the [🖥️ Desktop app](https://khoj.dev/downloads) to chat with your computer docs. +Get the Khoj [Desktop](https://khoj.dev/downloads), [Obsidian](https://docs.khoj.dev/#/obsidian?id=setup) or [Emacs](https://docs.khoj.dev/#/emacs?id=setup) app to search, chat with your 🖥️ computer docs. To get started, just start typing below. You can also type / to see a list of commands. `.trim() diff --git a/src/khoj/interface/web/content_source_computer_input.html b/src/khoj/interface/web/content_source_computer_input.html index 72aa3810..6c245079 100644 --- a/src/khoj/interface/web/content_source_computer_input.html +++ b/src/khoj/interface/web/content_source_computer_input.html @@ -7,7 +7,7 @@ Files

Manage files from your computer

-

Download the Khoj Desktop app to sync documents from your computer

+

Get the Khoj Desktop, Obsidian or Emacs app to sync documents from your computer

From d3d47dce0bb7a15a5d7ce04b679950c4d64cc9b1 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Thu, 21 Dec 2023 17:07:08 +0530 Subject: [PATCH 4/9] Allow setting Khoj app version during docker build via build-args This will allow troubleshooting by getting the actual khoj version being used. Previously it was always set to a static 0.0.0 version Command to build Khoj docker image with dynamically set current app version: `docker-compose build server --build-arg VERSION=$(pipx run hatch version)' --- Dockerfile | 3 ++- prod.Dockerfile | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9882a236..e6a6fb4a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,8 @@ WORKDIR /app # Install Application COPY pyproject.toml . COPY README.md . -RUN sed -i 's/dynamic = \["version"\]/version = "0.0.0"/' pyproject.toml && \ +ARG VERSION=0.0.0 +RUN sed -i "s/dynamic = \\[\"version\"\\]/version = \"$VERSION\"/" pyproject.toml && \ pip install --no-cache-dir . # Copy Source Code diff --git a/prod.Dockerfile b/prod.Dockerfile index 693a3a8b..8b21cb66 100644 --- a/prod.Dockerfile +++ b/prod.Dockerfile @@ -11,7 +11,8 @@ WORKDIR /app # Install Application COPY pyproject.toml . COPY README.md . -RUN sed -i 's/dynamic = \["version"\]/version = "0.0.0"/' pyproject.toml && \ +ARG VERSION=0.0.0 +RUN sed -i "s/dynamic = \\[\"version\"\\]/version = \"$VERSION\"/" pyproject.toml && \ TMPDIR=/home/cache/ pip install --cache-dir=/home/cache/ -e . # Copy Source Code From b5ae64cb3cda7900f4c168431cc43676c7957462 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Thu, 21 Dec 2023 17:16:05 +0530 Subject: [PATCH 5/9] Dynamically set Khoj app version in the Dockerization Github workflows --- .github/workflows/dockerize.yml | 5 +++++ .github/workflows/dockerize_production.yml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/dockerize.yml b/.github/workflows/dockerize.yml index 70098040..e0e63ccd 100644 --- a/.github/workflows/dockerize.yml +++ b/.github/workflows/dockerize.yml @@ -36,6 +36,10 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.PAT }} + - name: Get App Version + id: hatch + run: echo "version=$(pipx run hatch version)" >> $GITHUB_OUTPUT + - name: 📦 Build and Push Docker Image uses: docker/build-push-action@v2 with: @@ -45,4 +49,5 @@ jobs: push: true tags: ghcr.io/${{ github.repository }}:${{ env.DOCKER_IMAGE_TAG }} build-args: | + VERSION=${{ steps.hatch.outputs.version }} PORT=42110 diff --git a/.github/workflows/dockerize_production.yml b/.github/workflows/dockerize_production.yml index c4ed963e..a0b86981 100644 --- a/.github/workflows/dockerize_production.yml +++ b/.github/workflows/dockerize_production.yml @@ -40,6 +40,10 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.PAT }} + - name: Get App Version + id: hatch + run: echo "::set-output name=version::$(pipx run hatch version)" + - name: 📦 Build and Push Docker Image uses: docker/build-push-action@v2 with: @@ -49,4 +53,5 @@ jobs: push: true tags: ghcr.io/${{ github.repository }}-cloud:${{ env.DOCKER_IMAGE_TAG }} build-args: | + VERSION=${{ steps.hatch.outputs.version }} PORT=42110 From 8d1e9880592095464a538b18e6a142526ddd54f5 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Thu, 21 Dec 2023 17:29:37 +0530 Subject: [PATCH 6/9] Update tagging of the docker image on release, push to master & PR - Tag docker image with `tag_name' on release (i.e tag push) - Else tag with 'pre' on push to master - Else tag with 'dev' on push to PR branch - Only tag the latest release with release tag Previously the latest commit on master was being tagged with the latest tag. This doesn't sync with the release cadence of the rest of Khoj --- .github/workflows/dockerize.yml | 7 +++++-- .github/workflows/dockerize_production.yml | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dockerize.yml b/.github/workflows/dockerize.yml index e0e63ccd..bc2da688 100644 --- a/.github/workflows/dockerize.yml +++ b/.github/workflows/dockerize.yml @@ -16,7 +16,8 @@ on: workflow_dispatch: env: - DOCKER_IMAGE_TAG: ${{ github.ref == 'refs/heads/master' && 'latest' || github.ref_name }} + # Tag Image with tag name on release, else with 'pre' if push to master + DOCKER_IMAGE_TAG: ${{ github.ref_type == 'tag' && github.ref_name || 'pre' }} jobs: build: @@ -47,7 +48,9 @@ jobs: file: Dockerfile platforms: linux/amd64, linux/arm64 push: true - tags: ghcr.io/${{ github.repository }}:${{ env.DOCKER_IMAGE_TAG }} + tags: | + ghcr.io/${{ github.repository }}:${{ env.DOCKER_IMAGE_TAG }} + ${{ github.ref_type == 'tag' && 'ghcr.io/${{ github.repository }}-cloud:latest' || '' }} build-args: | VERSION=${{ steps.hatch.outputs.version }} PORT=42110 diff --git a/.github/workflows/dockerize_production.yml b/.github/workflows/dockerize_production.yml index a0b86981..a42aa8d9 100644 --- a/.github/workflows/dockerize_production.yml +++ b/.github/workflows/dockerize_production.yml @@ -20,7 +20,8 @@ on: workflow_dispatch: env: - DOCKER_IMAGE_TAG: ${{ github.event_name == 'pull_request' && 'dev' || (github.ref == 'refs/heads/master' && 'latest' || github.ref_name) }} + # Tag Image with tag name on release, else with 'pre' if push to master else with 'dev' if push to PR branch + DOCKER_IMAGE_TAG: ${{ github.ref_type == 'tag' && github.ref_name || github.event_name == 'pull_request' && 'dev' || 'pre' }} jobs: build: @@ -51,7 +52,9 @@ jobs: file: prod.Dockerfile platforms: linux/amd64 push: true - tags: ghcr.io/${{ github.repository }}-cloud:${{ env.DOCKER_IMAGE_TAG }} + tags: | + ghcr.io/${{ github.repository }}-cloud:${{ env.DOCKER_IMAGE_TAG }} + ${{ github.ref_type == 'tag' && 'ghcr.io/${{ github.repository }}-cloud:latest' || '' }} build-args: | VERSION=${{ steps.hatch.outputs.version }} PORT=42110 From 350fd89c8d7e6dfc0936d7983b232d587946c0ff Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Thu, 21 Dec 2023 17:47:08 +0530 Subject: [PATCH 7/9] Clear chat history html in Obsidian if getChatHistory works too --- src/interface/obsidian/src/chat_modal.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/interface/obsidian/src/chat_modal.ts b/src/interface/obsidian/src/chat_modal.ts index ae7eada7..5b849544 100644 --- a/src/interface/obsidian/src/chat_modal.ts +++ b/src/interface/obsidian/src/chat_modal.ts @@ -255,8 +255,8 @@ export class KhojChatModal extends Modal { }); } } catch (err) { - let errorMsg = "Unable to get response from Khoj server ❤️‍🩹. Ensure server is running or contact developers for help at team@khoj.dev or on Discord"; - this.renderMessage(chatBodyEl, errorMsg, "khoj", undefined, true); + let errorMsg = "Unable to get response from Khoj server ❤️‍🩹. Ensure server is running or contact developers for help at [team@khoj.dev](mailto:team@khoj.dev) or in [Discord](https://discord.gg/BDgyabRM6e)"; + this.renderMessage(chatBodyEl, errorMsg, "khoj", undefined); return false; } return true; @@ -395,9 +395,9 @@ export class KhojChatModal extends Modal { // Throw error if conversation history isn't cleared throw new Error("Failed to clear conversation history"); } else { - // If conversation history is cleared successfully, clear chat logs from modal - chatBody.innerHTML = ""; let getChatHistoryStatus = await this.getChatHistory(chatBody); + // If conversation history is cleared successfully, clear chat logs from modal + if (getChatHistoryStatus) chatBody.innerHTML = ""; let statusMsg = getChatHistoryStatus ? result.message : "Failed to clear conversation history"; this.flashStatusInChatInput(statusMsg); } From d1012979954869773f34484c5ae9c24be9cce2ce Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Fri, 22 Dec 2023 13:52:40 +0530 Subject: [PATCH 8/9] Use markdown formatted chat message in chat modal --- src/interface/obsidian/src/chat_modal.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/interface/obsidian/src/chat_modal.ts b/src/interface/obsidian/src/chat_modal.ts index 5b849544..6f990b74 100644 --- a/src/interface/obsidian/src/chat_modal.ts +++ b/src/interface/obsidian/src/chat_modal.ts @@ -244,7 +244,7 @@ export class KhojChatModal extends Modal { if (responseJson.detail) { // If the server returns error details in response, render a setup hint. - let setupMsg = "Hi 👋🏾, to start chatting add available chat models options via the Django Admin panel on the Server"; + let setupMsg = "Hi 👋🏾, to start chatting add available chat models options via [the Django Admin panel](/server/admin) on the Server"; this.renderMessage(chatBodyEl, setupMsg, "khoj", undefined, true); return false; @@ -364,7 +364,7 @@ export class KhojChatModal extends Modal { } } } catch (err) { - let errorMsg = "

Sorry, unable to get response from Khoj backend ❤️‍🩹. Contact developer for help at team@khoj.dev or in Discord

"; + let errorMsg = "Sorry, unable to get response from Khoj backend ❤️‍🩹. Contact developer for help at team@khoj.dev or [in Discord](https://discord.gg/BDgyabRM6e)"; responseElement.innerHTML = errorMsg } } From 074123b9b9fa2aa56aee457e9ef1e42639cb8c13 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Fri, 22 Dec 2023 16:22:42 +0530 Subject: [PATCH 9/9] Merge cloud, local dockerize workflows - Delete unused config directory --- .github/workflows/dockerize.yml | 49 ++++++++++++++++-- .github/workflows/dockerize_production.yml | 60 ---------------------- config/khoj_docker.yml | 51 ------------------ config/khoj_sample.yml | 57 -------------------- 4 files changed, 44 insertions(+), 173 deletions(-) delete mode 100644 .github/workflows/dockerize_production.yml delete mode 100644 config/khoj_docker.yml delete mode 100644 config/khoj_sample.yml diff --git a/.github/workflows/dockerize.yml b/.github/workflows/dockerize.yml index bc2da688..b9673e97 100644 --- a/.github/workflows/dockerize.yml +++ b/.github/workflows/dockerize.yml @@ -8,24 +8,47 @@ on: - master paths: - src/khoj/** - - config/** - pyproject.toml - Dockerfile + - prod.Dockerfile - docker-compose.yml - .github/workflows/dockerize.yml workflow_dispatch: + inputs: + tag: + description: 'Docker image tag' + default: 'dev' + khoj: + description: 'Build Khoj docker image' + type: boolean + default: true + khoj-cloud: + description: 'Build Khoj cloud docker image' + type: boolean + default: true env: - # Tag Image with tag name on release, else with 'pre' if push to master - DOCKER_IMAGE_TAG: ${{ github.ref_type == 'tag' && github.ref_name || 'pre' }} + # Tag Image with tag name on release + # else with user specified tag (default 'dev') if triggered via workflow + # else with 'pre' (if push to master) + DOCKER_IMAGE_TAG: ${{ github.ref_type == 'tag' && github.ref_name || github.event_name == 'workflow_dispatch' && github.event.inputs.tag || 'pre' }} jobs: build: - name: Build Docker Image, Push to Container Registry + name: Publish Khoj Docker Images runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image: + - 'local' + - 'cloud' steps: - name: Checkout Code uses: actions/checkout@v3 + with: + # Get all history to correctly infer Khoj version using hatch + fetch-depth: 0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -43,6 +66,7 @@ jobs: - name: 📦 Build and Push Docker Image uses: docker/build-push-action@v2 + if: (matrix.image == 'local' && github.event_name == 'workflow_dispatch') && github.event.inputs.khoj == 'true' || (matrix.image == 'local' && github.event_name == 'push') with: context: . file: Dockerfile @@ -50,7 +74,22 @@ jobs: push: true tags: | ghcr.io/${{ github.repository }}:${{ env.DOCKER_IMAGE_TAG }} - ${{ github.ref_type == 'tag' && 'ghcr.io/${{ github.repository }}-cloud:latest' || '' }} + ${{ github.ref_type == 'tag' && format('ghcr.io/{0}:latest', github.repository) || '' }} + build-args: | + VERSION=${{ steps.hatch.outputs.version }} + PORT=42110 + + - name: 📦️⛅️ Build and Push Cloud Docker Image + uses: docker/build-push-action@v2 + if: (matrix.image == 'cloud' && github.event_name == 'workflow_dispatch') && github.event.inputs.khoj-cloud == 'true' || (matrix.image == 'cloud' && github.event_name == 'push') + with: + context: . + file: prod.Dockerfile + platforms: linux/amd64 + push: true + tags: | + ghcr.io/${{ github.repository }}-cloud:${{ env.DOCKER_IMAGE_TAG }} + ${{ github.ref_type == 'tag' && format('ghcr.io/{0}-cloud:latest', github.repository) || '' }} build-args: | VERSION=${{ steps.hatch.outputs.version }} PORT=42110 diff --git a/.github/workflows/dockerize_production.yml b/.github/workflows/dockerize_production.yml deleted file mode 100644 index a42aa8d9..00000000 --- a/.github/workflows/dockerize_production.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: dockerize production - -on: - pull_request: - paths: - - src/khoj/** - - pyproject.toml - - prod.Dockerfile - - .github/workflows/dockerize_production.yml - push: - tags: - - "*" - branches: - - master - paths: - - src/khoj/** - - pyproject.toml - - prod.Dockerfile - - .github/workflows/dockerize_production.yml - workflow_dispatch: - -env: - # Tag Image with tag name on release, else with 'pre' if push to master else with 'dev' if push to PR branch - DOCKER_IMAGE_TAG: ${{ github.ref_type == 'tag' && github.ref_name || github.event_name == 'pull_request' && 'dev' || 'pre' }} - -jobs: - build: - name: Build Production Docker Image, Push to Container Registry - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.PAT }} - - - name: Get App Version - id: hatch - run: echo "::set-output name=version::$(pipx run hatch version)" - - - name: 📦 Build and Push Docker Image - uses: docker/build-push-action@v2 - with: - context: . - file: prod.Dockerfile - platforms: linux/amd64 - push: true - tags: | - ghcr.io/${{ github.repository }}-cloud:${{ env.DOCKER_IMAGE_TAG }} - ${{ github.ref_type == 'tag' && 'ghcr.io/${{ github.repository }}-cloud:latest' || '' }} - build-args: | - VERSION=${{ steps.hatch.outputs.version }} - PORT=42110 diff --git a/config/khoj_docker.yml b/config/khoj_docker.yml deleted file mode 100644 index 5fb89665..00000000 --- a/config/khoj_docker.yml +++ /dev/null @@ -1,51 +0,0 @@ -content-type: - # The /data/folder/ prefix to the folders is here because this is - # the directory to which the local files are copied in the docker-compose. - # If changing, the docker-compose volumes should also be changed to match. - org: - input-files: null - input-filter: ["/data/org/**/*.org"] - compressed-jsonl: "/data/embeddings/notes.jsonl.gz" - embeddings-file: "/data/embeddings/note_embeddings.pt" - index_heading_entries: false - - markdown: - input-files: null - input-filter: ["/data/markdown/**/*.markdown"] - compressed-jsonl: "/data/embeddings/markdown.jsonl.gz" - embeddings-file: "/data/embeddings/markdown_embeddings.pt" - - pdf: - input-files: null - input-filter: ["/data/pdf/**/*.pdf"] - compressed-jsonl: "/data/embeddings/pdf.jsonl.gz" - embeddings-file: "/data/embeddings/pdf_embeddings.pt" - - image: - input-directories: ["/data/images/"] - embeddings-file: "/data/embeddings/image_embeddings.pt" - batch-size: 50 - use-xmp-metadata: false - - notion: null - github: null - plugins: null - -search-type: - symmetric: null - asymmetric: - encoder: "sentence-transformers/multi-qa-MiniLM-L6-cos-v1" - cross-encoder: "cross-encoder/ms-marco-MiniLM-L-6-v2" - model_directory: "/data/models/asymmetric" - image: - encoder: "sentence-transformers/clip-ViT-B-32" - model_directory: "/data/models/image_encoder" - -processor: - conversation: - conversation-logfile: "/data/embeddings/conversation_logs.json" - enable-offline-chat: false - openai: null - -app: - should_log_telemetry: true diff --git a/config/khoj_sample.yml b/config/khoj_sample.yml deleted file mode 100644 index a30b02d9..00000000 --- a/config/khoj_sample.yml +++ /dev/null @@ -1,57 +0,0 @@ -content-type: - org: - input-files: # ["/path/to/org-file.org"] REQUIRED IF input-filter IS NOT SET OR - input-filter: # ["/path/to/org/*.org"] REQUIRED IF input-files IS NOT SET - compressed-jsonl: "~/.khoj/content/org/org.jsonl.gz" - embeddings-file: "~/.khoj/content/org/org_embeddings.pt" - index_heading_entries: false # Set to true to index entries with empty body - - markdown: - input-files: # ["/path/to/markdown-file.md"] REQUIRED IF input-filter IS NOT SET OR - input-filter: # ["/path/to/markdown/*.md"] REQUIRED IF input-files IS NOT SET - compressed-jsonl: "~/.khoj/content/markdown/markdown.jsonl.gz" - embeddings-file: "~/.khoj/content/markdown/markdown_embeddings.pt" - - ledger: - input-files: # ["/path/to/ledger-file.beancount"] REQUIRED IF input-filter is not set OR - input-filter: # ["/path/to/ledger/*.beancount"] REQUIRED IF input-files is not set - compressed-jsonl: "~/.khoj/content/ledger/ledger.jsonl.gz" - embeddings-file: "~/.khoj/content/ledger/ledger_embeddings.pt" - - image: - input-directories: # ["/path/to/images/"] REQUIRED IF input-filter IS NOT SET OR - input-filter: # ["/path/to/images/*.jpg"] REQUIRED IF input-directories IS NOT SET - embeddings-file: "~/.khoj/content/image/image_embeddings.pt" - batch-size: 50 - use-xmp-metadata: false - - music: - input-files: # ["/path/to/music-file.org"] REQUIRED IF input-filter IS NOT SET OR - input-filter: # ["/path/to/music/*.org"] REQUIRED IF input-files IS NOT SET - compressed-jsonl: "~/.khoj/content/music/music.jsonl.gz" - embeddings-file: "~/.khoj/content/music/music_embeddings.pt" - -search-type: - symmetric: - encoder: "sentence-transformers/all-MiniLM-L6-v2" - cross-encoder: "cross-encoder/ms-marco-MiniLM-L-6-v2" - encoder-type: sentence_transformers.SentenceTransformer - model_directory: "~/.khoj/search/symmetric/" - - asymmetric: - encoder: "sentence-transformers/multi-qa-MiniLM-L6-cos-v1" - cross-encoder: "cross-encoder/ms-marco-MiniLM-L-6-v2" - encoder-type: sentence_transformers.SentenceTransformer - model_directory: "~/.khoj/search/asymmetric/" - - image: - encoder: "sentence-transformers/clip-ViT-B-32" - encoder-type: sentence_transformers.SentenceTransformer - model_directory: "~/.khoj/search/image/" - -processor: - conversation: - openai-api-key: # "YOUR_OPENAI_API_KEY" - model: "text-davinci-003" - chat-model: "gpt-3.5-turbo" - conversation-logfile: "~/.khoj/processor/conversation/conversation_logs.json"