From e8176b41ef1ee128a8faad474d71476ceaaf5dd6 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Tue, 16 Jul 2024 13:26:09 +0530 Subject: [PATCH 01/11] Reuse get config data logic across config pages on web client - Put logic to get config data, detailed or basic into router helpers module - Use the get config func across the config pages on web clients - Put configure content and get_notion_auth_url funcs in router helper module to avoid circular import --- src/khoj/routers/helpers.py | 301 +++++++++++++++++++++++++++++++++ src/khoj/routers/indexer.py | 195 +-------------------- src/khoj/routers/notion.py | 8 +- src/khoj/routers/web_client.py | 291 +++++-------------------------- 4 files changed, 347 insertions(+), 448 deletions(-) diff --git a/src/khoj/routers/helpers.py b/src/khoj/routers/helpers.py index e0f91df7..f497e6b4 100644 --- a/src/khoj/routers/helpers.py +++ b/src/khoj/routers/helpers.py @@ -5,6 +5,7 @@ import io import json import logging import math +import os import re from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timedelta, timezone @@ -35,6 +36,7 @@ from PIL import Image from starlette.authentication import has_required_scope from starlette.requests import URL +from khoj.database import adapters from khoj.database.adapters import ( AgentAdapters, AutomationAdapters, @@ -42,18 +44,30 @@ from khoj.database.adapters import ( EntryAdapters, create_khoj_token, get_khoj_tokens, + get_user_name, + get_user_subscription_state, run_with_process_lock, ) from khoj.database.models import ( ChatModelOptions, ClientApplication, Conversation, + GithubConfig, KhojUser, + NotionConfig, ProcessLock, Subscription, TextToImageModelConfig, UserRequests, ) +from khoj.processor.content.docx.docx_to_entries import DocxToEntries +from khoj.processor.content.github.github_to_entries import GithubToEntries +from khoj.processor.content.images.image_to_entries import ImageToEntries +from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntries +from khoj.processor.content.notion.notion_to_entries import NotionToEntries +from khoj.processor.content.org_mode.org_to_entries import OrgToEntries +from khoj.processor.content.pdf.pdf_to_entries import PdfToEntries +from khoj.processor.content.plaintext.plaintext_to_entries import PlaintextToEntries from khoj.processor.conversation import prompts from khoj.processor.conversation.anthropic.anthropic_chat import ( anthropic_send_message_to_model, @@ -69,11 +83,15 @@ from khoj.processor.conversation.utils import ( generate_chatml_messages_with_context, save_to_conversation_log, ) +from khoj.processor.speech.text_to_speech import is_eleven_labs_enabled from khoj.routers.email import is_resend_enabled, send_task_email from khoj.routers.storage import upload_image +from khoj.routers.twilio import is_twilio_enabled +from khoj.search_type import text_search from khoj.utils import state from khoj.utils.config import OfflineChatProcessorModel from khoj.utils.helpers import ( + LRU, ConversationCommand, ImageIntentType, is_none_or_empty, @@ -1186,3 +1204,286 @@ def construct_automation_created_message(automation: Job, crontime: str, query_t Manage your automations [here](/automations). """.strip() + + +def get_user_config(user: KhojUser, request: Request, is_detailed: bool = False): + user_picture = request.session.get("user", {}).get("picture") + is_active = has_required_scope(request, ["premium"]) + has_documents = EntryAdapters.user_has_entries(user=user) + + if not is_detailed: + return { + "request": request, + "username": user.username if user else None, + "user_photo": user_picture, + "is_active": is_active, + "has_documents": has_documents, + "khoj_version": state.khoj_version, + } + + user_subscription_state = get_user_subscription_state(user.email) + user_subscription = adapters.get_user_subscription(user.email) + subscription_renewal_date = ( + user_subscription.renewal_date.strftime("%d %b %Y") + if user_subscription and user_subscription.renewal_date + else (user_subscription.created_at + timedelta(days=7)).strftime("%d %b %Y") + ) + given_name = get_user_name(user) + + enabled_content_source = set(EntryAdapters.get_unique_file_sources(user)) + successfully_configured = { + "computer": ("computer" in enabled_content_source), + "github": ("github" in enabled_content_source), + "notion": ("notion" in enabled_content_source), + } + + selected_conversation_config = ConversationAdapters.get_conversation_config(user) + conversation_options = ConversationAdapters.get_conversation_processor_options().all() + all_conversation_options = list() + for conversation_option in conversation_options: + all_conversation_options.append({"chat_model": conversation_option.chat_model, "id": conversation_option.id}) + + search_model_options = adapters.get_or_create_search_models().all() + all_search_model_options = list() + for search_model_option in search_model_options: + all_search_model_options.append({"name": search_model_option.name, "id": search_model_option.id}) + + current_search_model_option = adapters.get_user_search_model_or_default(user) + + selected_paint_model_config = ConversationAdapters.get_user_text_to_image_model_config(user) + paint_model_options = ConversationAdapters.get_text_to_image_model_options().all() + all_paint_model_options = list() + for paint_model in paint_model_options: + all_paint_model_options.append({"model_name": paint_model.model_name, "id": paint_model.id}) + + notion_oauth_url = get_notion_auth_url(user) + + eleven_labs_enabled = is_eleven_labs_enabled() + + voice_models = ConversationAdapters.get_voice_model_options() + voice_model_options = list() + for voice_model in voice_models: + voice_model_options.append({"name": voice_model.name, "id": voice_model.model_id}) + + if len(voice_model_options) == 0: + eleven_labs_enabled = False + + selected_voice_config = ConversationAdapters.get_voice_model_config(user) + + return { + "request": request, + "username": user.username if user else None, + "user_photo": user_picture, + "is_active": is_active, + "has_documents": has_documents, + "khoj_version": state.khoj_version, + "current_model_state": successfully_configured, + "anonymous_mode": state.anonymous_mode, + "given_name": given_name, + "search_model_options": all_search_model_options, + "selected_search_model_config": current_search_model_option.id, + "conversation_options": all_conversation_options, + "selected_conversation_config": selected_conversation_config.id if selected_conversation_config else None, + "paint_model_options": all_paint_model_options, + "selected_paint_model_config": selected_paint_model_config.id if selected_paint_model_config else None, + "user_photo": user_picture, + "billing_enabled": state.billing_enabled, + "subscription_state": user_subscription_state, + "subscription_renewal_date": subscription_renewal_date, + "khoj_cloud_subscription_url": os.getenv("KHOJ_CLOUD_SUBSCRIPTION_URL"), + "is_twilio_enabled": is_twilio_enabled(), + "is_eleven_labs_enabled": eleven_labs_enabled, + "voice_model_options": voice_model_options, + "selected_voice_config": selected_voice_config.model_id if selected_voice_config else None, + "phone_number": user.phone_number, + "is_phone_number_verified": user.verified_phone_number, + "notion_oauth_url": notion_oauth_url, + } + + +def configure_content( + files: Optional[dict[str, dict[str, str]]], + regenerate: bool = False, + t: Optional[state.SearchType] = state.SearchType.All, + full_corpus: bool = True, + user: KhojUser = None, +) -> bool: + success = True + if t == None: + t = state.SearchType.All + + if t is not None and t in [type.value for type in state.SearchType]: + t = state.SearchType(t) + + if t is not None and not t.value in [type.value for type in state.SearchType]: + logger.warning(f"🚨 Invalid search type: {t}") + return False + + search_type = t.value if t else None + + no_documents = all([not files.get(file_type) for file_type in files]) + + if files is None: + logger.warning(f"🚨 No files to process for {search_type} search.") + return True + + try: + # Initialize Org Notes Search + if (search_type == state.SearchType.All.value or search_type == state.SearchType.Org.value) and files["org"]: + logger.info("🦄 Setting up search for orgmode notes") + # Extract Entries, Generate Notes Embeddings + text_search.setup( + OrgToEntries, + files.get("org"), + regenerate=regenerate, + full_corpus=full_corpus, + user=user, + ) + except Exception as e: + logger.error(f"🚨 Failed to setup org: {e}", exc_info=True) + success = False + + try: + # Initialize Markdown Search + if (search_type == state.SearchType.All.value or search_type == state.SearchType.Markdown.value) and files[ + "markdown" + ]: + logger.info("💎 Setting up search for markdown notes") + # Extract Entries, Generate Markdown Embeddings + text_search.setup( + MarkdownToEntries, + files.get("markdown"), + regenerate=regenerate, + full_corpus=full_corpus, + user=user, + ) + + except Exception as e: + logger.error(f"🚨 Failed to setup markdown: {e}", exc_info=True) + success = False + + try: + # Initialize PDF Search + if (search_type == state.SearchType.All.value or search_type == state.SearchType.Pdf.value) and files["pdf"]: + logger.info("🖨️ Setting up search for pdf") + # Extract Entries, Generate PDF Embeddings + text_search.setup( + PdfToEntries, + files.get("pdf"), + regenerate=regenerate, + full_corpus=full_corpus, + user=user, + ) + + except Exception as e: + logger.error(f"🚨 Failed to setup PDF: {e}", exc_info=True) + success = False + + try: + # Initialize Plaintext Search + if (search_type == state.SearchType.All.value or search_type == state.SearchType.Plaintext.value) and files[ + "plaintext" + ]: + logger.info("📄 Setting up search for plaintext") + # Extract Entries, Generate Plaintext Embeddings + text_search.setup( + PlaintextToEntries, + files.get("plaintext"), + regenerate=regenerate, + full_corpus=full_corpus, + user=user, + ) + + except Exception as e: + logger.error(f"🚨 Failed to setup plaintext: {e}", exc_info=True) + success = False + + try: + if no_documents: + github_config = GithubConfig.objects.filter(user=user).prefetch_related("githubrepoconfig").first() + if ( + search_type == state.SearchType.All.value or search_type == state.SearchType.Github.value + ) and github_config is not None: + logger.info("🐙 Setting up search for github") + # Extract Entries, Generate Github Embeddings + text_search.setup( + GithubToEntries, + None, + regenerate=regenerate, + full_corpus=full_corpus, + user=user, + config=github_config, + ) + + except Exception as e: + logger.error(f"🚨 Failed to setup GitHub: {e}", exc_info=True) + success = False + + try: + if no_documents: + # Initialize Notion Search + notion_config = NotionConfig.objects.filter(user=user).first() + if ( + search_type == state.SearchType.All.value or search_type == state.SearchType.Notion.value + ) and notion_config: + logger.info("🔌 Setting up search for notion") + text_search.setup( + NotionToEntries, + None, + regenerate=regenerate, + full_corpus=full_corpus, + user=user, + config=notion_config, + ) + + except Exception as e: + logger.error(f"🚨 Failed to setup Notion: {e}", exc_info=True) + success = False + + try: + # Initialize Image Search + if (search_type == state.SearchType.All.value or search_type == state.SearchType.Image.value) and files[ + "image" + ]: + logger.info("🖼️ Setting up search for images") + # Extract Entries, Generate Image Embeddings + text_search.setup( + ImageToEntries, + files.get("image"), + regenerate=regenerate, + full_corpus=full_corpus, + user=user, + ) + except Exception as e: + logger.error(f"🚨 Failed to setup images: {e}", exc_info=True) + success = False + try: + if (search_type == state.SearchType.All.value or search_type == state.SearchType.Docx.value) and files["docx"]: + logger.info("📄 Setting up search for docx") + text_search.setup( + DocxToEntries, + files.get("docx"), + regenerate=regenerate, + full_corpus=full_corpus, + user=user, + ) + except Exception as e: + logger.error(f"🚨 Failed to setup docx: {e}", exc_info=True) + success = False + + # Invalidate Query Cache + if user: + state.query_cache[user.uuid] = LRU() + + return success + + +NOTION_OAUTH_CLIENT_ID = os.getenv("NOTION_OAUTH_CLIENT_ID") +NOTION_OAUTH_CLIENT_SECRET = os.getenv("NOTION_OAUTH_CLIENT_SECRET") +NOTION_REDIRECT_URI = os.getenv("NOTION_REDIRECT_URI") + + +def get_notion_auth_url(user: KhojUser): + if not NOTION_OAUTH_CLIENT_ID or not NOTION_OAUTH_CLIENT_SECRET or not NOTION_REDIRECT_URI: + return None + return f"https://api.notion.com/v1/oauth/authorize?client_id={NOTION_OAUTH_CLIENT_ID}&redirect_uri={NOTION_REDIRECT_URI}&response_type=code&state={user.uuid}" diff --git a/src/khoj/routers/indexer.py b/src/khoj/routers/indexer.py index 2046ad41..5c080cd4 100644 --- a/src/khoj/routers/indexer.py +++ b/src/khoj/routers/indexer.py @@ -6,20 +6,14 @@ from fastapi import APIRouter, Depends, Header, Request, Response, UploadFile from pydantic import BaseModel from starlette.authentication import requires -from khoj.database.models import GithubConfig, KhojUser, NotionConfig -from khoj.processor.content.docx.docx_to_entries import DocxToEntries -from khoj.processor.content.github.github_to_entries import GithubToEntries -from khoj.processor.content.images.image_to_entries import ImageToEntries -from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntries -from khoj.processor.content.notion.notion_to_entries import NotionToEntries -from khoj.processor.content.org_mode.org_to_entries import OrgToEntries -from khoj.processor.content.pdf.pdf_to_entries import PdfToEntries -from khoj.processor.content.plaintext.plaintext_to_entries import PlaintextToEntries -from khoj.routers.helpers import ApiIndexedDataLimiter, update_telemetry_state -from khoj.search_type import text_search +from khoj.routers.helpers import ( + ApiIndexedDataLimiter, + configure_content, + update_telemetry_state, +) from khoj.utils import constants, state from khoj.utils.config import SearchModels -from khoj.utils.helpers import LRU, get_file_type +from khoj.utils.helpers import get_file_type from khoj.utils.rawconfig import ContentConfig, FullConfig, SearchConfig from khoj.utils.yaml import save_config_to_file_updated_state @@ -170,180 +164,3 @@ def configure_search(search_models: SearchModels, search_config: Optional[Search search_models = SearchModels() return search_models - - -def configure_content( - files: Optional[dict[str, dict[str, str]]], - regenerate: bool = False, - t: Optional[state.SearchType] = state.SearchType.All, - full_corpus: bool = True, - user: KhojUser = None, -) -> bool: - success = True - if t == None: - t = state.SearchType.All - - if t is not None and t in [type.value for type in state.SearchType]: - t = state.SearchType(t) - - if t is not None and not t.value in [type.value for type in state.SearchType]: - logger.warning(f"🚨 Invalid search type: {t}") - return False - - search_type = t.value if t else None - - no_documents = all([not files.get(file_type) for file_type in files]) - - if files is None: - logger.warning(f"🚨 No files to process for {search_type} search.") - return True - - try: - # Initialize Org Notes Search - if (search_type == state.SearchType.All.value or search_type == state.SearchType.Org.value) and files["org"]: - logger.info("🦄 Setting up search for orgmode notes") - # Extract Entries, Generate Notes Embeddings - text_search.setup( - OrgToEntries, - files.get("org"), - regenerate=regenerate, - full_corpus=full_corpus, - user=user, - ) - except Exception as e: - logger.error(f"🚨 Failed to setup org: {e}", exc_info=True) - success = False - - try: - # Initialize Markdown Search - if (search_type == state.SearchType.All.value or search_type == state.SearchType.Markdown.value) and files[ - "markdown" - ]: - logger.info("💎 Setting up search for markdown notes") - # Extract Entries, Generate Markdown Embeddings - text_search.setup( - MarkdownToEntries, - files.get("markdown"), - regenerate=regenerate, - full_corpus=full_corpus, - user=user, - ) - - except Exception as e: - logger.error(f"🚨 Failed to setup markdown: {e}", exc_info=True) - success = False - - try: - # Initialize PDF Search - if (search_type == state.SearchType.All.value or search_type == state.SearchType.Pdf.value) and files["pdf"]: - logger.info("🖨️ Setting up search for pdf") - # Extract Entries, Generate PDF Embeddings - text_search.setup( - PdfToEntries, - files.get("pdf"), - regenerate=regenerate, - full_corpus=full_corpus, - user=user, - ) - - except Exception as e: - logger.error(f"🚨 Failed to setup PDF: {e}", exc_info=True) - success = False - - try: - # Initialize Plaintext Search - if (search_type == state.SearchType.All.value or search_type == state.SearchType.Plaintext.value) and files[ - "plaintext" - ]: - logger.info("📄 Setting up search for plaintext") - # Extract Entries, Generate Plaintext Embeddings - text_search.setup( - PlaintextToEntries, - files.get("plaintext"), - regenerate=regenerate, - full_corpus=full_corpus, - user=user, - ) - - except Exception as e: - logger.error(f"🚨 Failed to setup plaintext: {e}", exc_info=True) - success = False - - try: - if no_documents: - github_config = GithubConfig.objects.filter(user=user).prefetch_related("githubrepoconfig").first() - if ( - search_type == state.SearchType.All.value or search_type == state.SearchType.Github.value - ) and github_config is not None: - logger.info("🐙 Setting up search for github") - # Extract Entries, Generate Github Embeddings - text_search.setup( - GithubToEntries, - None, - regenerate=regenerate, - full_corpus=full_corpus, - user=user, - config=github_config, - ) - - except Exception as e: - logger.error(f"🚨 Failed to setup GitHub: {e}", exc_info=True) - success = False - - try: - if no_documents: - # Initialize Notion Search - notion_config = NotionConfig.objects.filter(user=user).first() - if ( - search_type == state.SearchType.All.value or search_type == state.SearchType.Notion.value - ) and notion_config: - logger.info("🔌 Setting up search for notion") - text_search.setup( - NotionToEntries, - None, - regenerate=regenerate, - full_corpus=full_corpus, - user=user, - config=notion_config, - ) - - except Exception as e: - logger.error(f"🚨 Failed to setup Notion: {e}", exc_info=True) - success = False - - try: - # Initialize Image Search - if (search_type == state.SearchType.All.value or search_type == state.SearchType.Image.value) and files[ - "image" - ]: - logger.info("🖼️ Setting up search for images") - # Extract Entries, Generate Image Embeddings - text_search.setup( - ImageToEntries, - files.get("image"), - regenerate=regenerate, - full_corpus=full_corpus, - user=user, - ) - except Exception as e: - logger.error(f"🚨 Failed to setup images: {e}", exc_info=True) - success = False - try: - if (search_type == state.SearchType.All.value or search_type == state.SearchType.Docx.value) and files["docx"]: - logger.info("📄 Setting up search for docx") - text_search.setup( - DocxToEntries, - files.get("docx"), - regenerate=regenerate, - full_corpus=full_corpus, - user=user, - ) - except Exception as e: - logger.error(f"🚨 Failed to setup docx: {e}", exc_info=True) - success = False - - # Invalidate Query Cache - if user: - state.query_cache[user.uuid] = LRU() - - return success diff --git a/src/khoj/routers/notion.py b/src/khoj/routers/notion.py index 8190a617..9f5d803f 100644 --- a/src/khoj/routers/notion.py +++ b/src/khoj/routers/notion.py @@ -11,7 +11,7 @@ from starlette.responses import RedirectResponse from khoj.database.adapters import aget_user_by_uuid from khoj.database.models import KhojUser, NotionConfig -from khoj.routers.indexer import configure_content +from khoj.routers.helpers import configure_content from khoj.utils.state import SearchType NOTION_OAUTH_CLIENT_ID = os.getenv("NOTION_OAUTH_CLIENT_ID") @@ -25,12 +25,6 @@ executor = ThreadPoolExecutor() logger = logging.getLogger(__name__) -def get_notion_auth_url(user: KhojUser): - if not NOTION_OAUTH_CLIENT_ID or not NOTION_OAUTH_CLIENT_SECRET or not NOTION_REDIRECT_URI: - return None - return f"https://api.notion.com/v1/oauth/authorize?client_id={NOTION_OAUTH_CLIENT_ID}&redirect_uri={NOTION_REDIRECT_URI}&response_type=code&state={user.uuid}" - - async def run_in_executor(func, *args): loop = asyncio.get_event_loop() return await loop.run_in_executor(executor, func, *args) diff --git a/src/khoj/routers/web_client.py b/src/khoj/routers/web_client.py index b44c374c..33747dbf 100644 --- a/src/khoj/routers/web_client.py +++ b/src/khoj/routers/web_client.py @@ -1,30 +1,21 @@ # System Packages import json import os -from datetime import timedelta from typing import Optional from fastapi import APIRouter, Request from fastapi.responses import FileResponse, HTMLResponse, RedirectResponse from fastapi.templating import Jinja2Templates -from starlette.authentication import has_required_scope, requires +from starlette.authentication import requires -from khoj.database import adapters from khoj.database.adapters import ( AgentAdapters, - ConversationAdapters, - EntryAdapters, PublicConversationAdapters, get_user_github_config, - get_user_name, get_user_notion_config, - get_user_subscription_state, ) from khoj.database.models import KhojUser -from khoj.processor.speech.text_to_speech import is_eleven_labs_enabled -from khoj.routers.helpers import get_next_url -from khoj.routers.notion import get_notion_auth_url -from khoj.routers.twilio import is_twilio_enabled +from khoj.routers.helpers import get_next_url, get_user_config from khoj.utils import constants, state from khoj.utils.rawconfig import ( GithubContentConfig, @@ -42,80 +33,36 @@ templates = Jinja2Templates([constants.web_directory, constants.next_js_director @requires(["authenticated"], redirect="login_page") def index(request: Request): user = request.user.object - user_picture = request.session.get("user", {}).get("picture") - has_documents = EntryAdapters.user_has_entries(user=user) + user_config = get_user_config(user, request) - return templates.TemplateResponse( - "chat.html", - context={ - "request": request, - "username": user.username, - "user_photo": user_picture, - "is_active": has_required_scope(request, ["premium"]), - "has_documents": has_documents, - "khoj_version": state.khoj_version, - }, - ) + return templates.TemplateResponse("chat.html", context=user_config) @web_client.post("/", response_class=FileResponse) @requires(["authenticated"], redirect="login_page") def index_post(request: Request): user = request.user.object - user_picture = request.session.get("user", {}).get("picture") - has_documents = EntryAdapters.user_has_entries(user=user) + user_config = get_user_config(user, request) - return templates.TemplateResponse( - "chat.html", - context={ - "request": request, - "username": user.username, - "user_photo": user_picture, - "is_active": has_required_scope(request, ["premium"]), - "has_documents": has_documents, - "khoj_version": state.khoj_version, - }, - ) + return templates.TemplateResponse("chat.html", context=user_config) @web_client.get("/search", response_class=FileResponse) @requires(["authenticated"], redirect="login_page") def search_page(request: Request): user = request.user.object - user_picture = request.session.get("user", {}).get("picture") - has_documents = EntryAdapters.user_has_entries(user=user) + user_config = get_user_config(user, request) - return templates.TemplateResponse( - "search.html", - context={ - "request": request, - "username": user.username, - "user_photo": user_picture, - "is_active": has_required_scope(request, ["premium"]), - "has_documents": has_documents, - "khoj_version": state.khoj_version, - }, - ) + return templates.TemplateResponse("search.html", context=user_config) @web_client.get("/chat", response_class=FileResponse) @requires(["authenticated"], redirect="login_page") def chat_page(request: Request): user = request.user.object - user_picture = request.session.get("user", {}).get("picture") - has_documents = EntryAdapters.user_has_entries(user=user) + user_config = get_user_config(user, request) - return templates.TemplateResponse( - "chat.html", - context={ - "request": request, - "username": user.username, - "user_photo": user_picture, - "is_active": has_required_scope(request, ["premium"]), - "has_documents": has_documents, - "khoj_version": state.khoj_version, - }, - ) + return templates.TemplateResponse("chat.html", context=user_config) @web_client.get("/experimental", response_class=FileResponse) @@ -169,25 +116,14 @@ def agents_page(request: Request): @web_client.get("/agent/{agent_slug}", response_class=HTMLResponse) def agent_page(request: Request, agent_slug: str): user: KhojUser = request.user.object if request.user.is_authenticated else None - user_picture = request.session.get("user", {}).get("picture") if user else None - + user_config = get_user_config(user, request) agent = AgentAdapters.get_agent_by_slug(agent_slug) - has_documents = EntryAdapters.user_has_entries(user=user) if agent == None: - return templates.TemplateResponse( - "404.html", - context={ - "request": request, - "khoj_version": state.khoj_version, - "username": user.username if user else None, - "has_documents": False, - "is_active": has_required_scope(request, ["premium"]), - "user_photo": user_picture, - }, - ) + user_config["has_documents"] = False + return templates.TemplateResponse("404.html", context=user_config) - agent_metadata = { + user_config["agent"] = { "slug": agent.slug, "avatar": agent.avatar, "name": agent.name, @@ -199,115 +135,23 @@ def agent_page(request: Request, agent_slug: str): "creator_not_self": agent.creator != user, } - return templates.TemplateResponse( - "agent.html", - context={ - "request": request, - "agent": agent_metadata, - "khoj_version": state.khoj_version, - "username": user.username if user else None, - "has_documents": has_documents, - "is_active": has_required_scope(request, ["premium"]), - "user_photo": user_picture, - }, - ) + return templates.TemplateResponse("agent.html", context=user_config) @web_client.get("/config", response_class=HTMLResponse) @requires(["authenticated"], redirect="login_page") def config_page(request: Request): user: KhojUser = request.user.object - user_picture = request.session.get("user", {}).get("picture") - has_documents = EntryAdapters.user_has_entries(user=user) + user_config = get_user_config(user, request, is_detailed=True) - user_subscription_state = get_user_subscription_state(user.email) - user_subscription = adapters.get_user_subscription(user.email) - subscription_renewal_date = ( - user_subscription.renewal_date.strftime("%d %b %Y") - if user_subscription and user_subscription.renewal_date - else (user_subscription.created_at + timedelta(days=7)).strftime("%d %b %Y") - ) - given_name = get_user_name(user) - - enabled_content_source = set(EntryAdapters.get_unique_file_sources(user)) - successfully_configured = { - "computer": ("computer" in enabled_content_source), - "github": ("github" in enabled_content_source), - "notion": ("notion" in enabled_content_source), - } - - selected_conversation_config = ConversationAdapters.get_conversation_config(user) - conversation_options = ConversationAdapters.get_conversation_processor_options().all() - all_conversation_options = list() - for conversation_option in conversation_options: - all_conversation_options.append({"chat_model": conversation_option.chat_model, "id": conversation_option.id}) - - search_model_options = adapters.get_or_create_search_models().all() - all_search_model_options = list() - for search_model_option in search_model_options: - all_search_model_options.append({"name": search_model_option.name, "id": search_model_option.id}) - - current_search_model_option = adapters.get_user_search_model_or_default(user) - - selected_paint_model_config = ConversationAdapters.get_user_text_to_image_model_config(user) - paint_model_options = ConversationAdapters.get_text_to_image_model_options().all() - all_paint_model_options = list() - for paint_model in paint_model_options: - all_paint_model_options.append({"model_name": paint_model.model_name, "id": paint_model.id}) - - notion_oauth_url = get_notion_auth_url(user) - - eleven_labs_enabled = is_eleven_labs_enabled() - - voice_models = ConversationAdapters.get_voice_model_options() - voice_model_options = list() - for voice_model in voice_models: - voice_model_options.append({"name": voice_model.name, "id": voice_model.model_id}) - - if len(voice_model_options) == 0: - eleven_labs_enabled = False - - selected_voice_config = ConversationAdapters.get_voice_model_config(user) - - return templates.TemplateResponse( - "config.html", - context={ - "request": request, - "current_model_state": successfully_configured, - "anonymous_mode": state.anonymous_mode, - "username": user.username, - "given_name": given_name, - "search_model_options": all_search_model_options, - "selected_search_model_config": current_search_model_option.id, - "conversation_options": all_conversation_options, - "selected_conversation_config": selected_conversation_config.id if selected_conversation_config else None, - "paint_model_options": all_paint_model_options, - "selected_paint_model_config": selected_paint_model_config.id if selected_paint_model_config else None, - "user_photo": user_picture, - "billing_enabled": state.billing_enabled, - "subscription_state": user_subscription_state, - "subscription_renewal_date": subscription_renewal_date, - "khoj_cloud_subscription_url": os.getenv("KHOJ_CLOUD_SUBSCRIPTION_URL"), - "is_active": has_required_scope(request, ["premium"]), - "has_documents": has_documents, - "is_twilio_enabled": is_twilio_enabled(), - "is_eleven_labs_enabled": eleven_labs_enabled, - "voice_model_options": voice_model_options, - "selected_voice_config": selected_voice_config.model_id if selected_voice_config else None, - "phone_number": user.phone_number, - "is_phone_number_verified": user.verified_phone_number, - "khoj_version": state.khoj_version, - "notion_oauth_url": notion_oauth_url, - }, - ) + return templates.TemplateResponse("config.html", context=user_config) @web_client.get("/config/content-source/github", response_class=HTMLResponse) @requires(["authenticated"], redirect="login_page") def github_config_page(request: Request): user = request.user.object - user_picture = request.session.get("user", {}).get("picture") - has_documents = EntryAdapters.user_has_entries(user=user) + user_config = get_user_config(user, request) current_github_config = get_user_github_config(user) if current_github_config: @@ -329,66 +173,32 @@ def github_config_page(request: Request): else: current_config = {} # type: ignore - return templates.TemplateResponse( - "content_source_github_input.html", - context={ - "request": request, - "current_config": current_config, - "username": user.username, - "user_photo": user_picture, - "is_active": has_required_scope(request, ["premium"]), - "has_documents": has_documents, - "khoj_version": state.khoj_version, - }, - ) + user_config["current_config"] = current_config + return templates.TemplateResponse("content_source_github_input.html", context=user_config) @web_client.get("/config/content-source/notion", response_class=HTMLResponse) @requires(["authenticated"], redirect="login_page") def notion_config_page(request: Request): user = request.user.object - user_picture = request.session.get("user", {}).get("picture") - has_documents = EntryAdapters.user_has_entries(user=user) + user_config = get_user_config(user, request) + current_notion_config = get_user_notion_config(user) - - current_config = NotionContentConfig( - token=current_notion_config.token if current_notion_config else "", - ) - + token = current_notion_config.token if current_notion_config else "" + current_config = NotionContentConfig(token=token) current_config = json.loads(current_config.model_dump_json()) - return templates.TemplateResponse( - "content_source_notion_input.html", - context={ - "request": request, - "current_config": current_config, - "username": user.username, - "user_photo": user_picture, - "is_active": has_required_scope(request, ["premium"]), - "has_documents": has_documents, - "khoj_version": state.khoj_version, - }, - ) + user_config["current_config"] = current_config + return templates.TemplateResponse("content_source_notion_input.html", context=user_config) @web_client.get("/config/content-source/computer", response_class=HTMLResponse) @requires(["authenticated"], redirect="login_page") def computer_config_page(request: Request): user = request.user.object if request.user.is_authenticated else None - user_picture = request.session.get("user", {}).get("picture") if user else None - has_documents = EntryAdapters.user_has_entries(user=user) if user else False + user_config = get_user_config(user, request) - return templates.TemplateResponse( - "content_source_computer_input.html", - context={ - "request": request, - "username": user.username, - "user_photo": user_picture, - "is_active": has_required_scope(request, ["premium"]), - "has_documents": has_documents, - "khoj_version": state.khoj_version, - }, - ) + return templates.TemplateResponse("content_source_computer_input.html", context=user_config) @web_client.get("/share/chat/{public_conversation_slug}", response_class=HTMLResponse) @@ -404,8 +214,9 @@ def view_public_conversation(request: Request): }, ) user = request.user.object if request.user.is_authenticated else None - user_picture = request.session.get("user", {}).get("picture") if user else None - has_documents = EntryAdapters.user_has_entries(user=user) if user else False + user_config = get_user_config(user, request) + user_config["public_conversation_slug"] = public_conversation_slug + user_config["google_client_id"] = os.environ.get("GOOGLE_CLIENT_ID") all_agents = AgentAdapters.get_all_accessible_agents(request.user.object if request.user.is_authenticated else None) @@ -420,28 +231,15 @@ def view_public_conversation(request: Request): "name": agent.name, } ) + user_config["agents"] = agents_packet - google_client_id = os.environ.get("GOOGLE_CLIENT_ID") redirect_uri = str(request.app.url_path_for("auth")) next_url = str( request.app.url_path_for("view_public_conversation", public_conversation_slug=public_conversation_slug) ) + user_config["redirect_uri"] = f"{redirect_uri}?next={next_url}" - return templates.TemplateResponse( - "public_conversation.html", - context={ - "request": request, - "username": user.username if user else None, - "user_photo": user_picture, - "is_active": has_required_scope(request, ["premium"]), - "has_documents": has_documents, - "khoj_version": state.khoj_version, - "public_conversation_slug": public_conversation_slug, - "agents": agents_packet, - "google_client_id": google_client_id, - "redirect_uri": f"{redirect_uri}?next={next_url}", - }, - ) + return templates.TemplateResponse("public_conversation.html", context=user_config) @web_client.get("/automations", response_class=HTMLResponse) @@ -452,20 +250,9 @@ def automations_config_page( queryToRun: Optional[str] = None, ): user = request.user.object if request.user.is_authenticated else None - user_picture = request.session.get("user", {}).get("picture") - has_documents = EntryAdapters.user_has_entries(user=user) if user else False + user_config = get_user_config(user, request) + user_config["subject"] = subject if subject else "" + user_config["crontime"] = crontime if crontime else "" + user_config["queryToRun"] = queryToRun if queryToRun else "" - return templates.TemplateResponse( - "config_automation.html", - context={ - "request": request, - "username": user.username if user else None, - "user_photo": user_picture, - "is_active": has_required_scope(request, ["premium"]), - "has_documents": has_documents, - "khoj_version": state.khoj_version, - "subject": subject if subject else "", - "crontime": crontime if crontime else "", - "queryToRun": queryToRun if queryToRun else "", - }, - ) + return templates.TemplateResponse("config_automation.html", context=user_config) From dd31936746b9ac6ee5d9eed3a5d1c2bf5e58d1f9 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Tue, 16 Jul 2024 15:03:52 +0530 Subject: [PATCH 02/11] Make config api endpoint urls consistent - Consistently use /content/ for data. Remove content-source from path - Remove unnecessary /data/ prefix for API endpoints under /config --- .../components/modelPicker/modelPicker.tsx | 6 ++-- .../sidePanel/chatHistorySidePanel.tsx | 2 +- src/khoj/interface/web/chat.html | 2 +- src/khoj/interface/web/config.html | 16 ++++----- .../web/content_source_computer_input.html | 6 ++-- .../web/content_source_github_input.html | 2 +- .../web/content_source_notion_input.html | 2 +- src/khoj/routers/api_config.py | 36 +++++++++---------- 8 files changed, 36 insertions(+), 36 deletions(-) diff --git a/src/interface/web/app/components/modelPicker/modelPicker.tsx b/src/interface/web/app/components/modelPicker/modelPicker.tsx index 3691cf6a..40854d93 100644 --- a/src/interface/web/app/components/modelPicker/modelPicker.tsx +++ b/src/interface/web/app/components/modelPicker/modelPicker.tsx @@ -68,8 +68,8 @@ interface ModelPickerProps { } export const ModelPicker: React.FC = (props: ModelPickerProps) => { - const { data: models } = useOptionsRequest('/api/config/data/conversation/model/options'); - const { data: selectedModel } = useSelectedModel('/api/config/data/conversation/model'); + const { data: models } = useOptionsRequest('/api/config/chat/model/options'); + const { data: selectedModel } = useSelectedModel('/api/config/chat/model'); const [openLoginDialog, setOpenLoginDialog] = React.useState(false); let userData = useAuthenticatedData(); @@ -94,7 +94,7 @@ export const ModelPicker: React.FC = (props: ModelPickerProps) => { props.setModelUsed(model); } - fetch('/api/config/data/conversation/model' + '?id=' + String(model.id), { method: 'POST', body: JSON.stringify(model) }) + fetch('/api/config/chat/model' + '?id=' + String(model.id), { method: 'POST', body: JSON.stringify(model) }) .then((response) => { if (!response.ok) { throw new Error('Failed to select model'); diff --git a/src/interface/web/app/components/sidePanel/chatHistorySidePanel.tsx b/src/interface/web/app/components/sidePanel/chatHistorySidePanel.tsx index ced5c4e4..7232089f 100644 --- a/src/interface/web/app/components/sidePanel/chatHistorySidePanel.tsx +++ b/src/interface/web/app/components/sidePanel/chatHistorySidePanel.tsx @@ -148,7 +148,7 @@ interface FilesMenuProps { function FilesMenu(props: FilesMenuProps) { // Use SWR to fetch files - const { data: files, error } = useSWR(props.conversationId ? '/api/config/data/computer' : null, fetcher); + const { data: files, error } = useSWR(props.conversationId ? '/api/config/content/computer' : null, fetcher); const { data: selectedFiles, error: selectedFilesError } = useSWR(props.conversationId ? `/api/chat/conversation/file-filters/${props.conversationId}` : null, fetcher); const [isOpen, setIsOpen] = useState(false); const [unfilteredFiles, setUnfilteredFiles] = useState([]); diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html index ad8ced27..7a819633 100644 --- a/src/khoj/interface/web/chat.html +++ b/src/khoj/interface/web/chat.html @@ -1954,7 +1954,7 @@ To get started, just start typing below. You can also type / to see a list of co } var allFiles; function renderAllFiles() { - fetch('/api/config/data/computer') + fetch('/api/config/content/computer') .then(response => response.json()) .then(data => { var indexedFiles = document.getElementsByClassName("indexed-files")[0]; diff --git a/src/khoj/interface/web/config.html b/src/khoj/interface/web/config.html index be47660f..b21fa3ac 100644 --- a/src/khoj/interface/web/config.html +++ b/src/khoj/interface/web/config.html @@ -421,7 +421,7 @@ saveVoiceModelButton.disabled = true; saveVoiceModelButton.textContent = "Saving..."; - fetch('/api/config/data/voice/model?id=' + voiceModel, { + fetch('/api/config/voice/model?id=' + voiceModel, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -455,7 +455,7 @@ saveModelButton.innerHTML = ""; saveModelButton.textContent = "Saving..."; - fetch('/api/config/data/conversation/model?id=' + chatModel, { + fetch('/api/config/chat/model?id=' + chatModel, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -494,7 +494,7 @@ saveSearchModelButton.disabled = true; saveSearchModelButton.textContent = "Saving..."; - fetch('/api/config/data/search/model?id=' + searchModel, { + fetch('/api/config/search/model?id=' + searchModel, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -526,7 +526,7 @@ saveModelButton.disabled = true; saveModelButton.innerHTML = "Saving..."; - fetch('/api/config/data/paint/model?id=' + paintModel, { + fetch('/api/config/paint/model?id=' + paintModel, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -553,7 +553,7 @@ }; function clearContentType(content_source) { - fetch('/api/config/data/content-source/' + content_source, { + fetch('/api/config/content/' + content_source, { method: 'DELETE', headers: { 'Content-Type': 'application/json', @@ -676,7 +676,7 @@ content_sources = ["computer", "github", "notion"]; content_sources.forEach(content_source => { - fetch(`/api/config/data/${content_source}`, { + fetch(`/api/config/content/${content_source}`, { method: 'GET', headers: { 'Content-Type': 'application/json', @@ -807,7 +807,7 @@ function getIndexedDataSize() { document.getElementById("indexed-data-size").textContent = "Calculating..."; - fetch('/api/config/index/size') + fetch('/api/config/content/size') .then(response => response.json()) .then(data => { document.getElementById("indexed-data-size").textContent = data.indexed_data_size_in_mb + " MB used"; @@ -815,7 +815,7 @@ } function removeFile(path) { - fetch('/api/config/data/file?filename=' + path, { + fetch('/api/config/content/file?filename=' + path, { method: 'DELETE', headers: { 'Content-Type': 'application/json', diff --git a/src/khoj/interface/web/content_source_computer_input.html b/src/khoj/interface/web/content_source_computer_input.html index ac3209c3..f9b02d57 100644 --- a/src/khoj/interface/web/content_source_computer_input.html +++ b/src/khoj/interface/web/content_source_computer_input.html @@ -32,7 +32,7 @@