diff --git a/Dockerfile b/Dockerfile index 4512e884..9882a236 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ RUN sed -i 's/dynamic = \["version"\]/version = "0.0.0"/' pyproject.toml && \ COPY . . # Set the PYTHONPATH environment variable in order for it to find the Django app. -ENV PYTHONPATH=/app/src/khoj:$PYTHONPATH +ENV PYTHONPATH=/app/src:$PYTHONPATH # Run the Application # There are more arguments required for the application to run, diff --git a/docs/setup.md b/docs/setup.md index a1d2c17c..7c399cae 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -153,6 +153,9 @@ The optional steps below allow using Khoj from within an existing application li - **Khoj Emacs**:
[Install](/emacs?id=setup) khoj.el +#### Setup host URL +To configure your host URL on your clients when self-hosting, use `http://127.0.0.1:42110`. This is the default value for the `KHOJ_HOST` environment variable. Note that `localhost` will not work. + ### 5. Use Khoj 🚀 You can head to http://localhost:42110 to use the web interface. You can also use the desktop client to search and chat. diff --git a/manifest.json b/manifest.json index e0a0a9f5..1fcc5fa2 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "khoj", "name": "Khoj", - "version": "1.0.0", + "version": "1.0.1", "minAppVersion": "0.15.0", "description": "An AI copilot for your Second Brain", "author": "Khoj Inc.", diff --git a/src/interface/desktop/assets/icons/trash-solid.svg b/src/interface/desktop/assets/icons/trash-solid.svg new file mode 100644 index 00000000..768d80f8 --- /dev/null +++ b/src/interface/desktop/assets/icons/trash-solid.svg @@ -0,0 +1 @@ + diff --git a/src/interface/desktop/chat.html b/src/interface/desktop/chat.html index 6c6d1ca1..41e185d1 100644 --- a/src/interface/desktop/chat.html +++ b/src/interface/desktop/chat.html @@ -60,6 +60,52 @@ return referenceButton; } + function generateOnlineReference(reference, index) { + + // Generate HTML for Chat Reference + let title = reference.title; + let link = reference.link; + let snippet = reference.snippet; + let question = reference.question; + if (question) { + question = `Question: ${question}

`; + } else { + question = ""; + } + + let linkElement = document.createElement('a'); + linkElement.setAttribute('href', link); + linkElement.setAttribute('target', '_blank'); + linkElement.setAttribute('rel', 'noopener noreferrer'); + linkElement.classList.add("inline-chat-link"); + linkElement.classList.add("reference-link"); + linkElement.setAttribute('title', title); + linkElement.innerHTML = title; + + let referenceButton = document.createElement('button'); + referenceButton.innerHTML = linkElement.outerHTML; + referenceButton.id = `ref-${index}`; + referenceButton.classList.add("reference-button"); + referenceButton.classList.add("collapsed"); + referenceButton.tabIndex = 0; + + // Add event listener to toggle full reference on click + referenceButton.addEventListener('click', function() { + console.log(`Toggling ref-${index}`) + if (this.classList.contains("collapsed")) { + this.classList.remove("collapsed"); + this.classList.add("expanded"); + this.innerHTML = linkElement.outerHTML + `

${question + snippet}`; + } else { + this.classList.add("collapsed"); + this.classList.remove("expanded"); + this.innerHTML = linkElement.outerHTML; + } + }); + + return referenceButton; + } + function renderMessage(message, by, dt=null, annotations=null) { let message_time = formatDate(dt ?? new Date()); let by_name = by == "khoj" ? "🏮 Khoj" : "🤔 You"; @@ -90,8 +136,48 @@ chatBody.scrollTop = chatBody.scrollHeight; } - function renderMessageWithReference(message, by, context=null, dt=null) { - if (context == null || context.length == 0) { + function processOnlineReferences(referenceSection, onlineContext) { + let numOnlineReferences = 0; + for (let subquery in onlineContext) { + let onlineReference = onlineContext[subquery]; + if (onlineReference.organic && onlineReference.organic.length > 0) { + numOnlineReferences += onlineReference.organic.length; + for (let index in onlineReference.organic) { + let reference = onlineReference.organic[index]; + let polishedReference = generateOnlineReference(reference, index); + referenceSection.appendChild(polishedReference); + } + } + + if (onlineReference.knowledgeGraph && onlineReference.knowledgeGraph.length > 0) { + numOnlineReferences += onlineReference.knowledgeGraph.length; + for (let index in onlineReference.knowledgeGraph) { + let reference = onlineReference.knowledgeGraph[index]; + let polishedReference = generateOnlineReference(reference, index); + referenceSection.appendChild(polishedReference); + } + } + + if (onlineReference.peopleAlsoAsk && onlineReference.peopleAlsoAsk.length > 0) { + numOnlineReferences += onlineReference.peopleAlsoAsk.length; + for (let index in onlineReference.peopleAlsoAsk) { + let reference = onlineReference.peopleAlsoAsk[index]; + let polishedReference = generateOnlineReference(reference, index); + referenceSection.appendChild(polishedReference); + } + } + } + + return numOnlineReferences; + } + + function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null) { + if (context == null && onlineContext == null) { + renderMessage(message, by, dt); + return; + } + + if ((context && context.length == 0) && (onlineContext == null || (onlineContext && Object.keys(onlineContext).length == 0))) { renderMessage(message, by, dt); return; } @@ -100,8 +186,11 @@ let referenceExpandButton = document.createElement('button'); referenceExpandButton.classList.add("reference-expand-button"); - let expandButtonText = context.length == 1 ? "1 reference" : `${context.length} references`; - referenceExpandButton.innerHTML = expandButtonText; + let numReferences = 0; + + if (context) { + numReferences += context.length; + } references.appendChild(referenceExpandButton); @@ -127,6 +216,14 @@ referenceSection.appendChild(polishedReference); } } + + if (onlineContext) { + numReferences += processOnlineReferences(referenceSection, onlineContext); + } + + let expandButtonText = numReferences == 1 ? "1 reference" : `${numReferences} references`; + referenceExpandButton.innerHTML = expandButtonText; + references.appendChild(referenceSection); renderMessage(message, by, dt, references); @@ -140,6 +237,8 @@ newHTML = newHTML.replace(/__([\s\S]*?)__/g, '$1'); // Remove any text between [INST] and tags. These are spurious instructions for the AI chat model. newHTML = newHTML.replace(/\[INST\].+(<\/s>)?/g, ''); + // For any text that has single backticks, replace them with tags + newHTML = newHTML.replace(/`([^`]+)`/g, '$1'); return newHTML; } @@ -221,15 +320,28 @@ let referenceExpandButton = document.createElement('button'); referenceExpandButton.classList.add("reference-expand-button"); - let expandButtonText = rawReferenceAsJson.length == 1 ? "1 reference" : `${rawReferenceAsJson.length} references`; - referenceExpandButton.innerHTML = expandButtonText; - references.appendChild(referenceExpandButton); let referenceSection = document.createElement('div'); referenceSection.classList.add("reference-section"); referenceSection.classList.add("collapsed"); + let numReferences = 0; + + // If rawReferenceAsJson is a list, then count the length + if (Array.isArray(rawReferenceAsJson)) { + numReferences = rawReferenceAsJson.length; + + rawReferenceAsJson.forEach((reference, index) => { + let polishedReference = generateReference(reference, index); + referenceSection.appendChild(polishedReference); + }); + } else { + numReferences += processOnlineReferences(referenceSection, rawReferenceAsJson); + } + + references.appendChild(referenceExpandButton); + referenceExpandButton.addEventListener('click', function() { if (referenceSection.classList.contains("collapsed")) { referenceSection.classList.remove("collapsed"); @@ -240,10 +352,8 @@ } }); - rawReferenceAsJson.forEach((reference, index) => { - let polishedReference = generateReference(reference, index); - referenceSection.appendChild(polishedReference); - }); + let expandButtonText = numReferences == 1 ? "1 reference" : `${numReferences} references`; + referenceExpandButton.innerHTML = expandButtonText; references.appendChild(referenceSection); readStream(); } else { @@ -276,6 +386,9 @@ let chatInput = document.getElementById("chat-input"); chatInput.value = chatInput.value.trimStart(); + let questionStarterSuggestions = document.getElementById("question-starters"); + questionStarterSuggestions.style.display = "none"; + if (chatInput.value.startsWith("/") && chatInput.value.split(" ").length === 1) { let chatTooltip = document.getElementById("chat-tooltip"); chatTooltip.style.display = "block"; @@ -324,7 +437,7 @@ const khojToken = await window.tokenAPI.getToken(); const headers = { 'Authorization': `Bearer ${khojToken}` }; - fetch(`${hostURL}/api/chat/history?client=web`, { headers }) + fetch(`${hostURL}/api/chat/history?client=desktop`, { headers }) .then(response => response.json()) .then(data => { if (data.detail) { @@ -351,13 +464,38 @@ .then(response => { // Render conversation history, if any response.forEach(chat_log => { - renderMessageWithReference(chat_log.message, chat_log.by, chat_log.context, new Date(chat_log.created)); + renderMessageWithReference(chat_log.message, chat_log.by, chat_log.context, new Date(chat_log.created), chat_log.onlineContext); }); }) .catch(err => { return; }); + fetch(`${hostURL}/api/chat/starters?client=desktop`, { headers }) + .then(response => response.json()) + .then(data => { + // Render chat options, if any + if (data) { + let questionStarterSuggestions = document.getElementById("question-starters"); + for (let index in data) { + let questionStarter = data[index]; + let questionStarterButton = document.createElement('button'); + questionStarterButton.innerHTML = questionStarter; + questionStarterButton.classList.add("question-starter"); + questionStarterButton.addEventListener('click', function() { + questionStarterSuggestions.style.display = "none"; + document.getElementById("chat-input").value = questionStarter; + chat(); + }); + questionStarterSuggestions.appendChild(questionStarterButton); + } + questionStarterSuggestions.style.display = "grid"; + } + }) + .catch(err => { + return; + }); + fetch(`${hostURL}/api/chat/options`, { headers }) .then(response => response.json()) .then(data => { @@ -378,6 +516,32 @@ } } + async function clearConversationHistory() { + let chatInput = document.getElementById("chat-input"); + let originalPlaceholder = chatInput.placeholder; + let chatBody = document.getElementById("chat-body"); + + const hostURL = await window.hostURLAPI.getURL(); + const khojToken = await window.tokenAPI.getToken(); + const headers = { 'Authorization': `Bearer ${khojToken}` }; + + fetch(`${hostURL}/api/chat/history?client=desktop`, { method: "DELETE", headers }) + .then(response => response.ok ? response.json() : Promise.reject(response)) + .then(data => { + chatBody.innerHTML = ""; + loadChat(); + chatInput.placeholder = "Cleared conversation history"; + }) + .catch(err => { + chatInput.placeholder = "Failed to clear conversation history"; + }) + .finally(() => { + setTimeout(() => { + chatInput.placeholder = originalPlaceholder; + }, 2000); + }); + } + let mediaRecorder; async function speechToText() { const speakButton = document.getElementById('speak-button'); @@ -453,13 +617,19 @@
+ + + @@ -583,7 +753,7 @@ } #input-row { display: grid; - grid-template-columns: auto 32px; + grid-template-columns: auto 32px 32px; grid-column-gap: 10px; grid-row-gap: 10px; background: #f9fafc @@ -606,7 +776,7 @@ #chat-input:focus { outline: none !important; } - #speak-button { + .input-row-button { background: var(--background-color); border: none; border-radius: 5px; @@ -617,13 +787,13 @@ cursor: pointer; transition: background 0.3s ease-in-out; } - #speak-button:hover { + .input-row-button:hover { background: var(--primary-hover); } - #speak-button:active { + .input-row-button:active { background: var(--primary-active); } - #speak-button-img { + .input-row-button-img { width: 24px; } @@ -657,6 +827,38 @@ margin: 10px; } + div#question-starters { + grid-template-columns: repeat(auto-fit, minmax(100px, 1fr)); + grid-column-gap: 8px; + } + + button.question-starter { + background: var(--background-color); + color: var(--main-text-color); + border: 1px solid var(--main-text-color); + border-radius: 5px; + padding: 5px; + font-size: 14px; + font-weight: 300; + line-height: 1.5em; + cursor: pointer; + transition: background 0.2s ease-in-out; + text-align: left; + max-height: 75px; + transition: max-height 0.3s ease-in-out; + overflow: hidden; + } + + code.chat-response { + background: var(--primary-hover); + color: var(--primary-inverse); + border-radius: 5px; + padding: 5px; + font-size: 14px; + font-weight: 300; + line-height: 1.5em; + } + button.reference-button { background: var(--background-color); color: var(--main-text-color); diff --git a/src/interface/desktop/package.json b/src/interface/desktop/package.json index 04e1016b..03cee532 100644 --- a/src/interface/desktop/package.json +++ b/src/interface/desktop/package.json @@ -1,6 +1,6 @@ { "name": "Khoj", - "version": "1.0.0", + "version": "1.0.1", "description": "An AI copilot for your Second Brain", "author": "Saba Imran, Debanjum Singh Solanky ", "license": "GPL-3.0-or-later", diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el index f427c197..773f88d8 100644 --- a/src/interface/emacs/khoj.el +++ b/src/interface/emacs/khoj.el @@ -6,7 +6,7 @@ ;; Saba Imran ;; Description: An AI copilot for your Second Brain ;; Keywords: search, chat, org-mode, outlines, markdown, pdf, image -;; Version: 1.0.0 +;; Version: 1.0.1 ;; Package-Requires: ((emacs "27.1") (transient "0.3.0") (dash "2.19.1")) ;; URL: https://github.com/khoj-ai/khoj/tree/master/src/interface/emacs @@ -98,6 +98,11 @@ :group 'khoj :type 'string) +(defcustom khoj-auto-index t + "Should content be automatically re-indexed every `khoj-index-interval' seconds." + :group 'khoj + :type 'boolean) + (defcustom khoj-index-interval 3600 "Interval (in seconds) to wait before updating content index." :group 'khoj @@ -405,14 +410,16 @@ Auto invokes setup steps on calling main entrypoint." ;; render response from indexing API endpoint on server (lambda (status) (if (not status) - (message "khoj.el: %scontent index %supdated" (if content-type (format "%s " content-type) "") (if force "force " "")) - (with-current-buffer (current-buffer) - (goto-char "\n\n") - (message "khoj.el: Failed to %supdate %s content index. Status: %s. Response: %s" - (if force "force " "") - content-type - status - (string-trim (buffer-substring-no-properties (point) (point-max))))))) + (message "khoj.el: %scontent index %supdated" (if content-type (format "%s " content-type) "all ") (if force "force " "")) + (progn + (khoj--delete-open-network-connections-to-server) + (with-current-buffer (current-buffer) + (search-forward "\n\n" nil t) + (message "khoj.el: Failed to %supdate %s content index. Status: %s%s" + (if force "force " "") + (if content-type (format "%s " content-type) "all") + (string-trim (format "%s %s" (nth 1 (nth 1 status)) (nth 2 (nth 1 status)))) + (if (> (- (point-max) (point)) 0) (format ". Response: %s" (string-trim (buffer-substring-no-properties (point) (point-max)))) "")))))) nil t t))) (setq khoj--indexed-files files-to-index))) @@ -444,8 +451,9 @@ Use `BOUNDARY' to separate files. This is sent to Khoj server as a POST request. (when khoj--index-timer (cancel-timer khoj--index-timer)) ;; Send files to index on server every `khoj-index-interval' seconds -(setq khoj--index-timer - (run-with-timer 60 khoj-index-interval 'khoj--server-index-files)) +(when khoj-auto-index + (setq khoj--index-timer + (run-with-timer 60 khoj-index-interval 'khoj--server-index-files))) ;; ------------------------------------------- @@ -858,7 +866,7 @@ RECEIVE-DATE is the message receive date." (let ((proc-buf (buffer-name (process-buffer proc))) (khoj-network-proc-buf (string-join (split-string khoj-server-url "://") " "))) (when (string-match (format "%s" khoj-network-proc-buf) proc-buf) - (delete-process proc))))) + (ignore-errors (delete-process proc)))))) (defun khoj--teardown-incremental-search () "Teardown hooks used for incremental search." diff --git a/src/interface/obsidian/manifest.json b/src/interface/obsidian/manifest.json index e0a0a9f5..1fcc5fa2 100644 --- a/src/interface/obsidian/manifest.json +++ b/src/interface/obsidian/manifest.json @@ -1,7 +1,7 @@ { "id": "khoj", "name": "Khoj", - "version": "1.0.0", + "version": "1.0.1", "minAppVersion": "0.15.0", "description": "An AI copilot for your Second Brain", "author": "Khoj Inc.", diff --git a/src/interface/obsidian/package.json b/src/interface/obsidian/package.json index a0a0df3b..9ea5aeb1 100644 --- a/src/interface/obsidian/package.json +++ b/src/interface/obsidian/package.json @@ -1,6 +1,6 @@ { "name": "Khoj", - "version": "1.0.0", + "version": "1.0.1", "description": "An AI copilot for your Second Brain", "author": "Debanjum Singh Solanky, Saba Imran ", "license": "GPL-3.0-or-later", diff --git a/src/interface/obsidian/src/chat_modal.ts b/src/interface/obsidian/src/chat_modal.ts index a8008048..fc6d5a48 100644 --- a/src/interface/obsidian/src/chat_modal.ts +++ b/src/interface/obsidian/src/chat_modal.ts @@ -1,4 +1,4 @@ -import { App, Modal, request } from 'obsidian'; +import { App, Modal, request, setIcon } from 'obsidian'; import { KhojSetting } from 'src/settings'; import fetch from "node-fetch"; @@ -38,7 +38,8 @@ export class KhojChatModal extends Modal { await this.getChatHistory(); // Add chat input field - const chatInput = contentEl.createEl("input", + let inputRow = contentEl.createDiv("khoj-input-row"); + const chatInput = inputRow.createEl("input", { attr: { type: "text", @@ -50,6 +51,15 @@ export class KhojChatModal extends Modal { }) chatInput.addEventListener('change', (event) => { this.result = (event.target).value }); + let clearChat = inputRow.createEl("button", { + text: "Clear History", + attr: { + class: "khoj-input-row-button", + }, + }) + clearChat.addEventListener('click', async (_) => { await this.clearConversationHistory() }); + setIcon(clearChat, "trash"); + // Scroll to bottom of modal, till the send message input box this.modalEl.scrollTop = this.modalEl.scrollHeight; chatInput.focus(); @@ -194,4 +204,35 @@ export class KhojChatModal extends Modal { this.renderIncrementalMessage(responseElement, "Sorry, unable to get response from Khoj backend ❤️‍🩹. Contact developer for help at team@khoj.dev or in Discord") } } + + async clearConversationHistory() { + let chatInput = this.contentEl.getElementsByClassName("khoj-chat-input")[0]; + let originalPlaceholder = chatInput.placeholder; + let chatBody = this.contentEl.getElementsByClassName("khoj-chat-body")[0]; + + let response = await request({ + url: `${this.setting.khojUrl}/api/chat/history?client=web`, + method: "DELETE", + headers: { "Authorization": `Bearer ${this.setting.khojApiKey}` }, + }) + try { + let result = JSON.parse(response); + if (result.status !== "ok") { + // Throw error if conversation history isn't cleared + throw new Error("Failed to clear conversation history"); + } else { + // If conversation history is cleared successfully, clear chat logs from modal + chatBody.innerHTML = ""; + await this.getChatHistory(); + chatInput.placeholder = result.message; + } + } catch (err) { + chatInput.placeholder = "Failed to clear conversation history"; + } finally { + // Reset to original placeholder text after some time + setTimeout(() => { + chatInput.placeholder = originalPlaceholder; + }, 2000); + } + } } diff --git a/src/interface/obsidian/styles.css b/src/interface/obsidian/styles.css index d322804d..95a304f1 100644 --- a/src/interface/obsidian/styles.css +++ b/src/interface/obsidian/styles.css @@ -68,7 +68,7 @@ If your plugin does not need CSS, delete this file. } /* color chat bubble by khoj blue */ .khoj-chat-message-text.khoj { - color: var(--text-on-accent); + color: var(--khoj-chat-dark-grey); background: var(--khoj-chat-primary); margin-left: auto; white-space: pre-line; @@ -110,9 +110,12 @@ If your plugin does not need CSS, delete this file. grid-column-gap: 10px; grid-row-gap: 10px; } -#khoj-chat-footer > * { - padding: 15px; - background: #f9fafc +.khoj-input-row { + display: grid; + grid-template-columns: auto 32px; + grid-column-gap: 10px; + grid-row-gap: 10px; + background: var(--background-primary); } #khoj-chat-input.option:hover { box-shadow: 0 0 11px var(--background-modifier-box-shadow); @@ -121,6 +124,25 @@ If your plugin does not need CSS, delete this file. font-size: var(--font-ui-medium); padding: 25px 20px; } +.khoj-input-row-button { + background: var(--background-primary); + border: none; + border-radius: 5px; + padding: 5px; + --icon-size: var(--icon-size); + height: auto; + font-size: 14px; + font-weight: 300; + line-height: 1.5em; + cursor: pointer; + transition: background 0.3s ease-in-out; +} +.khoj-input-row-button:hover { + background: var(--background-modifier-hover); +} +.khoj-input-row-button:active { + background: var(--background-modifier-active); +} @media (pointer: coarse), (hover: none) { #khoj-chat-body.abbr[title] { diff --git a/src/interface/obsidian/versions.json b/src/interface/obsidian/versions.json index 06efeecb..fc1cfe72 100644 --- a/src/interface/obsidian/versions.json +++ b/src/interface/obsidian/versions.json @@ -27,5 +27,6 @@ "0.12.3": "0.15.0", "0.13.0": "0.15.0", "0.14.0": "0.15.0", - "1.0.0": "0.15.0" + "1.0.0": "0.15.0", + "1.0.1": "0.15.0" } diff --git a/src/khoj/app/settings.py b/src/khoj/app/settings.py index 427706e8..39af3d7a 100644 --- a/src/khoj/app/settings.py +++ b/src/khoj/app/settings.py @@ -70,7 +70,7 @@ MIDDLEWARE = [ "django.middleware.clickjacking.XFrameOptionsMiddleware", ] -ROOT_URLCONF = "app.urls" +ROOT_URLCONF = "khoj.app.urls" TEMPLATES = [ { diff --git a/src/khoj/app/wsgi.py b/src/khoj/app/wsgi.py index cbdf4342..03956936 100644 --- a/src/khoj/app/wsgi.py +++ b/src/khoj/app/wsgi.py @@ -11,6 +11,6 @@ import os from django.core.wsgi import get_wsgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "app.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "khoj.app.settings") application = get_wsgi_application() diff --git a/src/khoj/database/adapters/__init__.py b/src/khoj/database/adapters/__init__.py index 471f3605..eb143ab6 100644 --- a/src/khoj/database/adapters/__init__.py +++ b/src/khoj/database/adapters/__init__.py @@ -1,43 +1,44 @@ import math -from typing import Optional, Type, List -from datetime import date, datetime +import random import secrets -from typing import Type, List -from datetime import date, timezone +from datetime import date, datetime, timezone +from typing import List, Optional, Type -from django.db import models +from asgiref.sync import sync_to_async from django.contrib.sessions.backends.db import SessionStore -from pgvector.django import CosineDistance -from django.db.models.manager import BaseManager +from django.db import models from django.db.models import Q +from django.db.models.manager import BaseManager +from fastapi import HTTPException +from pgvector.django import CosineDistance from torch import Tensor -# Import sync_to_async from Django Channels -from asgiref.sync import sync_to_async - -from fastapi import HTTPException - from khoj.database.models import ( - KhojUser, + ChatModelOptions, + Conversation, + Entry, + GithubConfig, + GithubRepoConfig, GoogleUser, KhojApiUser, + KhojUser, NotionConfig, - GithubConfig, - Entry, - GithubRepoConfig, - Conversation, - ChatModelOptions, + OfflineChatProcessorConversationConfig, + OpenAIProcessorConversationConfig, SearchModelConfig, SpeechToTextModelOptions, Subscription, UserConversationConfig, OpenAIProcessorConversationConfig, OfflineChatProcessorConversationConfig, + ReflectiveQuestion, ) -from khoj.utils.helpers import generate_random_name -from khoj.search_filter.word_filter import WordFilter -from khoj.search_filter.file_filter import FileFilter from khoj.search_filter.date_filter import DateFilter +from khoj.search_filter.file_filter import FileFilter +from khoj.search_filter.word_filter import WordFilter +from khoj.utils import state +from khoj.utils.config import GPT4AllProcessorModel +from khoj.utils.helpers import generate_random_name async def set_notion_config(token: str, user: KhojUser): @@ -233,6 +234,10 @@ class ConversationAdapters: return await conversation.afirst() return await Conversation.objects.acreate(user=user) + @staticmethod + async def adelete_conversation_by_user(user: KhojUser): + return await Conversation.objects.filter(user=user).adelete() + @staticmethod def has_any_conversation_config(user: KhojUser): return ChatModelOptions.objects.filter(user=user).exists() @@ -344,6 +349,45 @@ class ConversationAdapters: async def get_speech_to_text_config(): return await SpeechToTextModelOptions.objects.filter().afirst() + @staticmethod + async def aget_conversation_starters(user: KhojUser): + all_questions = [] + if await ReflectiveQuestion.objects.filter(user=user).aexists(): + all_questions = await sync_to_async(ReflectiveQuestion.objects.filter(user=user).values_list)( + "question", flat=True + ) + + all_questions = await sync_to_async(ReflectiveQuestion.objects.filter(user=None).values_list)( + "question", flat=True + ) + + max_results = 3 + all_questions = await sync_to_async(list)(all_questions) + if len(all_questions) < max_results: + return all_questions + + return random.sample(all_questions, max_results) + + @staticmethod + def get_valid_conversation_config(user: KhojUser): + offline_chat_config = ConversationAdapters.get_offline_chat_conversation_config() + conversation_config = ConversationAdapters.get_conversation_config(user) + if conversation_config is None: + conversation_config = ConversationAdapters.get_default_conversation_config() + + if offline_chat_config and offline_chat_config.enabled and conversation_config.model_type == "offline": + if state.gpt4all_processor_config is None or state.gpt4all_processor_config.loaded_model is None: + state.gpt4all_processor_config = GPT4AllProcessorModel(conversation_config.chat_model) + + return conversation_config + + openai_chat_config = ConversationAdapters.get_openai_conversation_config() + if openai_chat_config and conversation_config.model_type == "openai": + return conversation_config + + else: + raise ValueError("Invalid conversation config - either configure offline chat or openai chat") + class EntryAdapters: word_filer = WordFilter() diff --git a/src/khoj/database/admin.py b/src/khoj/database/admin.py index 4383056f..2213fb6e 100644 --- a/src/khoj/database/admin.py +++ b/src/khoj/database/admin.py @@ -11,6 +11,7 @@ from khoj.database.models import ( SearchModelConfig, SpeechToTextModelOptions, Subscription, + ReflectiveQuestion, ) admin.site.register(KhojUser, UserAdmin) @@ -21,3 +22,4 @@ admin.site.register(OpenAIProcessorConversationConfig) admin.site.register(OfflineChatProcessorConversationConfig) admin.site.register(SearchModelConfig) admin.site.register(Subscription) +admin.site.register(ReflectiveQuestion) diff --git a/src/khoj/database/migrations/0020_reflectivequestion.py b/src/khoj/database/migrations/0020_reflectivequestion.py new file mode 100644 index 00000000..aefb73bf --- /dev/null +++ b/src/khoj/database/migrations/0020_reflectivequestion.py @@ -0,0 +1,36 @@ +# Generated by Django 4.2.7 on 2023-11-20 01:13 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("database", "0019_alter_googleuser_family_name_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="ReflectiveQuestion", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("question", models.CharField(max_length=500)), + ( + "user", + models.ForeignKey( + blank=True, + default=None, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "abstract": False, + }, + ), + ] diff --git a/src/khoj/database/migrations/0021_merge_20231126_0650.py b/src/khoj/database/migrations/0021_merge_20231126_0650.py new file mode 100644 index 00000000..579c0072 --- /dev/null +++ b/src/khoj/database/migrations/0021_merge_20231126_0650.py @@ -0,0 +1,12 @@ +# Generated by Django 4.2.7 on 2023-11-26 06:50 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("database", "0020_reflectivequestion"), + ("database", "0020_speechtotextmodeloptions_and_more"), + ] + + operations = [] diff --git a/src/khoj/database/models/__init__.py b/src/khoj/database/models/__init__.py index 8098a731..7b28521b 100644 --- a/src/khoj/database/models/__init__.py +++ b/src/khoj/database/models/__init__.py @@ -150,6 +150,11 @@ class Conversation(BaseModel): conversation_log = models.JSONField(default=dict) +class ReflectiveQuestion(BaseModel): + question = models.CharField(max_length=500) + user = models.ForeignKey(KhojUser, on_delete=models.CASCADE, default=None, null=True, blank=True) + + class Entry(BaseModel): class EntryType(models.TextChoices): IMAGE = "image" diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html index d346294f..7ca51fcc 100644 --- a/src/khoj/interface/web/chat.html +++ b/src/khoj/interface/web/chat.html @@ -9,14 +9,15 @@ +