Merge pull request #732 from khoj-ai/fit-and-finish/schedule-tasks

Fixes and improves for scheduled tasks

src/interface/desktop/chat.html

@@ -1509,7 +1509,7 @@
         #chat-input {
             font-family: var(--font-family);
             font-size: small;
-            height: 36px;
+            height: 48px;
             border-radius: 16px;
             resize: none;
             overflow-y: hidden;

src/khoj/interface/email/task.html

@@ -11,20 +11,18 @@
 </a>
 <div class="calls-to-action" style="margin-top: 20px;">
     <div>
-        <h1 style="color: #333; font-size: large; font-weight: bold; margin: 0; line-height: 1.5; background-color: #fee285; padding: 8px; box-shadow: 6px 6px rgba(0, 0, 0, 1.5);">Your Open, Personal AI</h1>
+        <h1 style="color: #333; font-size: large; font-weight: bold; margin: 0; line-height: 1.5; background-color: #fee285; padding: 8px; box-shadow: 6px 6px rgba(0, 0, 0, 1.5);">Your Automation, From Your Personal AI</h1>
-        <p style="color: #333; font-size: medium; margin-top: 20px; padding: 0; line-height: 1.5;">Hey {{name}}! </p>
-        <p style="color: #333; font-size: medium; margin-top: 20px; padding: 0; line-height: 1.5;">I've shared your automation results below:</p>
 
         <div style="display: grid; grid-template-columns: 1fr 1fr; grid-gap: 12px; margin-top: 20px;">
             <div style="border: 1px solid black; border-radius: 8px; padding: 8px; box-shadow: 6px 6px rgba(0, 0, 0, 1.0); margin-top: 20px;">
-                <a href="https://app.khoj.dev/config#tasks" style="text-decoration: none; text-decoration: underline dotted;">
+                <a href="https://app.khoj.dev/automations" style="text-decoration: none; text-decoration: underline dotted;">
                     <h3 style="color: #333; font-size: large; margin: 0; padding: 0; line-height: 2.0; background-color: #b8f1c7; padding: 8px; ">{{subject}}</h3>
                 </a>
                 <p style="color: #333; font-size: medium; margin-top: 20px; padding: 0; line-height: 1.5;">{{result}}</p>
             </div>
         </div>
-        <p style="color: #333; font-size: medium; margin-top: 20px; padding: 0; line-height: 1.5;">The automation query I ran on your behalf: {{query}}</p>
+        <p style="color: #333; font-size: medium; margin-top: 20px; padding: 0; line-height: 1.5;">The automation I ran on your behalf: {{query}}</p>
-        <p style="color: #333; font-size: medium; margin-top: 20px; padding: 0; line-height: 1.5;">You can view, delete your automations via <a href="https://app.khoj.dev/configure#tasks">the settings page</a></p>
+        <p style="color: #333; font-size: medium; margin-top: 20px; padding: 0; line-height: 1.5;">You can manage your automations via <a href="https://app.khoj.dev/automations">the settings page</a>.</p>
     </div>
 </div>
 <p style="color: #333; font-size: large; margin-top: 20px; padding: 0; line-height: 1.5;">- Khoj</p>

src/khoj/interface/web/chat.html

@@ -1170,7 +1170,7 @@ To get started, just start typing below. You can also type / to see a list of co
                                 chat_log.message,
                                 chat_log.by,
                                 chat_log.context,
-                                new Date(chat_log.created),
+                                new Date(chat_log.created + "Z"),
                                 chat_log.onlineContext,
                                 chat_log.intent?.type,
                                 chat_log.intent?.["inferred-queries"]);
@@ -1265,7 +1265,7 @@ To get started, just start typing below. You can also type / to see a list of co
                             chat_log.message,
                             chat_log.by,
                             chat_log.context,
-                            new Date(chat_log.created),
+                            new Date(chat_log.created + "Z"),
                             chat_log.onlineContext,
                             chat_log.intent?.type,
                             chat_log.intent?.["inferred-queries"]
@@ -2164,7 +2164,7 @@ To get started, just start typing below. You can also type / to see a list of co
         #chat-input {
             font-family: var(--font-family);
             font-size: medium;
-            height: 36px;
+            height: 48px;
             border-radius: 16px;
             resize: none;
             overflow-y: hidden;

src/khoj/interface/web/config_automation.html

@@ -6,14 +6,13 @@
             <img class="card-icon" src="/static/assets/icons/automation.svg?v={{ khoj_version }}" alt="Automate">
             <span class="card-title-text">Automate</span>
             <div class="instructions">
-                <a href="https://docs.khoj.dev/features/automations">ⓘ Help</a>
+                You can automate queries to run on a schedule using Khoj's automations. Results will be sent straight to your inbox.
             </div>
         </h2>
         <div class="section-body">
-            <h4>Automations</h4>
             <button id="create-automation-button" type="button" class="positive-button">
                <img class="automation-action-icon" src="/static/assets/icons/new.svg" alt="Automations">
-               <span id="create-automation-button-text">Create</span>
+               <span id="create-automation-button-text">Build</span>
             </button>
             <div id="automations" class="section-cards"></div>
         </div>
@@ -28,12 +27,15 @@
         width: 100%;
         height: 100%;
         grid-template-rows: none;
+        background-color: var(--frosted-background-color);
+        padding: 12px;
     }
     #create-automation-button {
         width: auto;
     }
     div#automations {
         margin-bottom: 12px;
+        grid-template-columns: 1fr;
     }
     button.negative-button {
         background-color: gainsboro;
@@ -44,6 +46,34 @@
     .positive-button:hover {
         background-color: var(--summer-sun);
     }
+
+    div.automation-buttons {
+        display: grid;
+        grid-gap: 8px;
+        grid-template-columns: 1fr 3fr;
+    }
+
+    button.save-automation-button {
+        background-color: var(--summer-sun);
+    }
+
+    button.save-automation-button:hover {
+        background-color: var(--primary-hover);
+    }
+
+    div.new-automation {
+        background-color: var(--frosted-background-color);
+        border-radius: 10px;
+        box-shadow: 0 4px 6px 0 hsla(0, 0%, 0%, 0.2);
+        margin-bottom: 20px;
+        transition: box-shadow 0.3s ease, transform 0.3s ease;
+    }
+
+    div.new-automation:hover {
+        box-shadow: 0 10px 15px 0 hsla(0, 0%, 0%, 0.1);
+        transform: translateY(-5px);
+    }
+
 </style>
 <script>
     function deleteAutomation(automationId) {
@@ -84,13 +114,12 @@
         let automationEl = document.createElement("div");
         automationEl.innerHTML = `
             <div class="card automation" id="automation-card-${automationId}">
-                <label for="subject">Subject</label>
                 <input type="text"
                        id="automation-subject-${automationId}"
                        name="subject"
                        data-original="${automation.subject}"
                        value="${automation.subject}">
-                <label for="query-to-run">Query to Run</label>
+                <label for="query-to-run">Your automation</label>
                 <textarea id="automation-queryToRun-${automationId}"
                           data-original="${automation.query_to_run}"
                           name="query-to-run">${automation.query_to_run}</textarea>
@@ -102,12 +131,14 @@
                        data-original="${automation.schedule}"
                        title="${automationNextRun}"
                        value="${automation.schedule}">
-                <button type="button"
+                <div class="automation-buttons">
-                        class="save-automation-button positive-button"
+                    <button type="button"
-                        id="save-automation-button-${automationId}">Save</button>
+                            class="delete-automation-button negative-button"
-                <button type="button"
+                            id="delete-automation-button-${automationId}">Delete</button>
-                        class="delete-automation-button negative-button"
+                    <button type="button"
-                        id="delete-automation-button-${automationId}">Delete</button>
+                            class="save-automation-button positive-button"
+                            id="save-automation-button-${automationId}">Save</button>
+                </div>
                 <div id="automation-success-${automationId}" style="display: none;"></div>
             </div>
         `;
@@ -155,14 +186,13 @@
     }
 
     async function saveAutomation(automationId, create=false) {
-        const subject = encodeURIComponent(document.getElementById(`automation-subject-${automationId}`).value);
         const queryToRun = encodeURIComponent(document.getElementById(`automation-queryToRun-${automationId}`).value);
         const scheduleEl = document.getElementById(`automation-schedule-${automationId}`);
         const notificationEl = document.getElementById(`automation-success-${automationId}`);
         const saveButtonEl = document.getElementById(`save-automation-button-${automationId}`);
         const actOn = create ? "Create" : "Save";
 
-        if (subject === "" || queryToRun == "" || scheduleEl.value == "") {
+        if (queryToRun == "" || scheduleEl.value == "") {
             return;
         }
 
@@ -186,10 +216,13 @@
         const encodedCrontime = encodeURIComponent(crontime);
 
         // Construct query string and select method for API call
-        let query_string = `q=${queryToRun}&subject=${subject}&crontime=${encodedCrontime}&city=${ip_data.city}&region=${ip_data.region}&country=${ip_data.country_name}&timezone=${ip_data.timezone}`;
+        let query_string = `q=${queryToRun}&crontime=${encodedCrontime}&city=${ip_data.city}&region=${ip_data.region}&country=${ip_data.country_name}&timezone=${ip_data.timezone}`;
+
         let method = "POST";
         if (!create) {
+            const subject = encodeURIComponent(document.getElementById(`automation-subject-${automationId}`).value);
             query_string += `&automation_id=${automationId}`;
+            query_string += `&subject=${subject}`;
             method = "PUT"
         }
 
@@ -231,29 +264,27 @@
         var automationEl = document.createElement("div");
         automationEl.classList.add("card");
         automationEl.classList.add("automation");
+        automationEl.classList.add("new-automation")
         const placeholderId = Date.now();
         automationEl.id = "automation-card-" + placeholderId;
         automationEl.innerHTML = `
-            <label for="subject">Subject</label>
+            <label for="query-to-run">Your new automation</label>
-            <input type="text"
-                   id="automation-subject-${placeholderId}"
-                   name="subject"
-                   placeholder="My Personal Newsletter">
-            <label for="query-to-run">Query to Run</label>
             <textarea id="automation-queryToRun-${placeholderId}" placeholder="Share a Newsletter including: 1. Weather forecast for this Week. 2. A Book Highlight from my Notes. 3. Recap News from Last Week"></textarea>
             <label for="schedule">Schedule</label>
             <input type="text"
                    id="automation-schedule-${placeholderId}"
                    name="schedule"
                    placeholder="9AM every morning">
-            <button type="button"
+            <div class="automation-buttons">
-                    class="save-automation-button"
+                <button type="button"
-                    onclick="saveAutomation(${placeholderId}, true)"
+                        class="delete-automation-button negative-button"
-                    id="save-automation-button-${placeholderId}">Create</button>
+                        onclick="deleteAutomation(${placeholderId}, true)"
-            <button type="button"
+                        id="delete-automation-button-${placeholderId}">Cancel</button>
-                    class="delete-automation-button"
+                <button type="button"
-                    onclick="deleteAutomation(${placeholderId}, true)"
+                        class="save-automation-button"
-                    id="delete-automation-button-${placeholderId}">Delete</button>
+                        onclick="saveAutomation(${placeholderId}, true)"
+                        id="save-automation-button-${placeholderId}">Create</button>
+            </div>
             <div id="automation-success-${placeholderId}" style="display: none;"></div>
         `;
         document.getElementById("automations").insertBefore(automationEl, document.getElementById("automations").firstChild);

src/khoj/main.py

@@ -6,6 +6,7 @@ from contextlib import redirect_stdout
 import logging
 import io
 import os
+import atexit
 import sys
 import locale
 
@@ -93,6 +94,11 @@ from khoj.utils.cli import cli
 from khoj.utils.initialization import initialization
 
 
+def shutdown_scheduler():
+    logger.info("🌑 Shutting down Khoj")
+    state.scheduler.shutdown()
+
+
 def run(should_start_server=True):
     # Turn Tokenizers Parallelism Off. App does not support it.
     os.environ["TOKENIZERS_PARALLELISM"] = "false"
@@ -158,9 +164,8 @@ def run(should_start_server=True):
     # If the server is started through gunicorn (external to the script), don't start the server
     if should_start_server:
         start_server(app, host=args.host, port=args.port, socket=args.socket)
-
+        # Teardown
-    # Teardown
+        shutdown_scheduler()
-    state.scheduler.shutdown()
 
 
 def set_state(args):
@@ -202,3 +207,4 @@ if __name__ == "__main__":
     run()
 else:
     run(should_start_server=False)
+    atexit.register(shutdown_scheduler)

src/khoj/processor/conversation/prompts.py

@@ -575,6 +575,26 @@ Khoj:
 """.strip()
 )
 
+subject_generation = PromptTemplate.from_template(
+    """
+You are an extremely smart and helpful title generator assistant. Given a user query, extract the subject or title of the task to be performed.
+- Use the user query to infer the subject or title of the task.
+
+# Examples:
+User: Show a new Calvin and Hobbes quote every morning at 9am. My Current Location: Shanghai, China
+Khoj: Your daily Calvin and Hobbes Quote
+
+User: Notify me when version 2.0.0 of the sentence transformers python package is released. My Current Location: Mexico City, Mexico
+Khoj: Sentence Transformers Python Package Version 2.0.0 Release
+
+User: Gather the latest tech news on the first sunday of every month.
+Khoj: Your Monthly Dose of Tech News
+
+User Query: {query}
+Khoj:
+""".strip()
+)
+
 to_notify_or_not = PromptTemplate.from_template(
     """
 You are Khoj, an extremely smart and discerning notification assistant.

src/khoj/routers/api.py

@@ -34,6 +34,7 @@ from khoj.routers.helpers import (
     ApiUserRateLimiter,
     CommonQueryParams,
     ConversationCommandRateLimiter,
+    acreate_title_from_query,
     schedule_automation,
     update_telemetry_state,
 )
@@ -425,7 +426,6 @@ def delete_automation(request: Request, automation_id: str) -> Response:
 async def post_automation(
     request: Request,
     q: str,
-    subject: str,
     crontime: str,
     city: Optional[str] = None,
     region: Optional[str] = None,
@@ -435,8 +435,8 @@ async def post_automation(
     user: KhojUser = request.user.object
 
     # Perform validation checks
-    if is_none_or_empty(q) or is_none_or_empty(subject) or is_none_or_empty(crontime):
+    if is_none_or_empty(q) or is_none_or_empty(crontime):
-        return Response(content="A query, subject and crontime is required", status_code=400)
+        return Response(content="A query and crontime is required", status_code=400)
     if not cron_descriptor.get_description(crontime):
         return Response(content="Invalid crontime", status_code=400)
 
@@ -452,7 +452,7 @@ async def post_automation(
         crontime = " ".join(crontime.split(" ")[:5])
     # Convert crontime to standard unix crontime
     crontime = crontime.replace("?", "*")
-    subject = subject.strip()
+    subject = await acreate_title_from_query(q)
 
     # Schedule automation with query_to_run, timezone, subject directly provided by user
     try:

src/khoj/routers/email.py

@@ -44,7 +44,7 @@ async def send_welcome_email(name, email):
         {
             "from": "team@khoj.dev",
             "to": email,
-            "subject": f"Welcome to Khoj, {name}!" if name else "Welcome to Khoj!",
+            "subject": f"{name}, four ways to use Khoj!" if name else "Four ways to use Khoj!",
             "html": html_content,
         }
     )
@@ -55,6 +55,8 @@ def send_task_email(name, email, query, result, subject):
         logger.debug("Email sending disabled")
         return
 
+    logger.info(f"Sending email to {email} for task {subject}")
+
     template = env.get_template("task.html")
 
     html_result = markdown_it.MarkdownIt().render(result)

src/khoj/routers/helpers.py

@@ -187,6 +187,18 @@ async def agenerate_chat_response(*args):
     return await loop.run_in_executor(executor, generate_chat_response, *args)
 
 
+async def acreate_title_from_query(query: str) -> str:
+    """
+    Create a title from the given query
+    """
+    title_generation_prompt = prompts.subject_generation.format(query=query)
+
+    with timer("Chat actor: Generate title from query", logger):
+        response = await send_message_to_model_wrapper(title_generation_prompt)
+
+    return response.strip()
+
+
 async def aget_relevant_information_sources(query: str, conversation_history: dict, is_task: bool):
     """
     Given a query, determine which of the available tools the agent should use in order to answer appropriately.
@@ -913,7 +925,7 @@ def scheduled_chat(query_to_run: str, scheduling_request: str, subject: str, use
     # Notify user if the AI response is satisfactory
     if should_notify(original_query=scheduling_request, executed_query=cleaned_query, ai_response=ai_response):
         if is_resend_enabled():
-            send_task_email(user.get_short_name(), user.email, scheduling_request, ai_response, subject)
+            send_task_email(user.get_short_name(), user.email, cleaned_query, ai_response, subject)
         else:
             return raw_response