Move automations into independent page. Allow direct automation

- Previously it was a section in the settings page. Move it to
  independent, top-level page to improve visibility of feature

- Calculate crontime from natural language on web client before
  sending it to to server for saving new/updated schedule to disk.
  - Avoids round-trip of call to chat model

- Convert POST /api/automation API endpoint into a direct request for
  automation with query_to_run, subject and schedule provided via the
  automation page. This allows more granular control to create automation
  - Make the POST automations endpoint more robust; runs validation
    checks, normalizes parameters

src/khoj/interface/web/config.html

@@ -272,39 +272,6 @@
         {% endif %}
     </div>
     {% endif %}
-    <div id="automations" class="section">
-        <h2 class="section-title">Automations</h2>
-        <div id="automations" class="api-settings">
-            <div class="card-title-row">
-                <img class="card-icon" src="/static/assets/icons/automation.svg" alt="Automations">
-                <h3 class="card-title">
-                    <span>Automations</span>
-                </h3>
-            </div>
-            <div class="card-description-row">
-                <p id="tasks-settings-card-description" class="card-description">Manage your automations</p>
-            </div>
-            <table id="automations-table">
-                <thead>
-                <tr>
-                    <th scope="col">Name</th>
-                    <th scope="col">Scheduling Request</th>
-                    <th scope="col">Query to Run</th>
-                    <th scope="col">Schedule</th>
-                    <th scope="col">Actions</th>
-                </tr>
-                </thead>
-                <tbody id="automations-list"></tbody>
-            </table>
-            <div class="card-action-row">
-                <button class="card-button happy" id="create-automation">
-                    <img class="automation-action-icon" src="/static/assets/icons/new.svg" alt="Automations">
-                    Create Automation
-                </button>
-            </div>
-        </div>
-    </div>
-
     {% if billing_enabled %}
     <div id="billing" class="section">
         <h2 class="section-title">Billing</h2>

src/khoj/interface/web/config_automation.html

@@ -0,0 +1,251 @@
+{% extends "base_config.html" %}
+{% block content %}
+<div class="page">
+    <div class="section">
+        <h2 class="section-title">
+            <img class="card-icon" src="/static/assets/icons/automation.svg?v={{ khoj_version }}" alt="Automate">
+            <span class="card-title-text">Automate</span>
+            <div class="instructions">
+                <a href="https://docs.khoj.dev/features/automations">ⓘ Help</a>
+            </div>
+        </h2>
+        <div class="section-body">
+            <h4>Automations</h4>
+            <button id="create-automation-button" type="button" class="positive-button">
+               <img class="automation-action-icon" src="/static/assets/icons/new.svg" alt="Automations">
+               <span id="create-automation-button-text">Create</span>
+            </button>
+            <div id="automations" class="section-cards"></div>
+        </div>
+    </div>
+</div>
+<script src="/static/assets/natural-cron.min.js"></script>
+<style>
+    td {
+        padding: 10px 0;
+    }
+    div.automation {
+        width: 100%;
+        height: 100%;
+        grid-template-rows: none;
+    }
+    #create-automation-button {
+        width: auto;
+    }
+    div#automations {
+        margin-bottom: 12px;
+    }
+    button.negative-button {
+        background-color: gainsboro;
+    }
+    .positive-button {
+        background-color: var(--primary-hover);
+    }
+    .positive-button:hover {
+        background-color: var(--summer-sun);
+    }
+</style>
+<script>
+    function deleteAutomation(automationId) {
+        const AutomationList = document.getElementById("automations");
+        fetch(`/api/automation?automation_id=${automationId}`, {
+            method: 'DELETE',
+        })
+        .then(response => {
+            if (response.status == 200 || response.status == 204) {
+                const AutomationItem = document.getElementById(`automation-card-${automationId}`);
+                AutomationList.removeChild(AutomationItem);
+            }
+        });
+    }
+
+    function updateAutomationRow(automation) {
+        let automationId = automation.id;
+        let automationNextRun = `Next run at ${automation.next}\nCron: ${automation.crontime}`;
+
+        let scheduleEl = document.getElementById(`automation-schedule-${automationId}`);
+        scheduleEl.setAttribute('data-original', automation.schedule);
+        scheduleEl.setAttribute('data-cron', automation.crontime);
+        scheduleEl.setAttribute('title', automationNextRun);
+        scheduleEl.value = automation.schedule;
+
+        let subjectEl = document.getElementById(`automation-subject-${automationId}`);
+        subjectEl.setAttribute('data-original', automation.subject);
+        subjectEl.value = automation.subject;
+
+        let queryEl = document.getElementById(`automation-queryToRun-${automationId}`);
+        queryEl.setAttribute('data-original', automation.query_to_run);
+        queryEl.value = automation.query_to_run;
+    }
+
+    function generateAutomationRow(automation) {
+        let automationId = automation.id;
+        let automationNextRun = `Next run at ${automation.next}\nCron: ${automation.crontime}`;
+        let automationEl = document.createElement("div");
+        automationEl.innerHTML = `
+            <div class="card automation" id="automation-card-${automationId}">
+                <label for="subject">Subject</label>
+                <input type="text"
+                       id="automation-subject-${automationId}"
+                       name="subject"
+                       data-original="${automation.subject}"
+                       value="${automation.subject}">
+                <label for="query-to-run">Query to Run</label>
+                <textarea id="automation-queryToRun-${automationId}"
+                          data-original="${automation.query_to_run}"
+                          name="query-to-run">${automation.query_to_run}</textarea>
+                <label for="schedule">Schedule</label>
+                <input type="text"
+                       id="automation-schedule-${automationId}"
+                       name="schedule"
+                       data-cron="${automation.crontime}"
+                       data-original="${automation.schedule}"
+                       title="${automationNextRun}"
+                       value="${automation.schedule}">
+                <button type="button"
+                        class="save-automation-button positive-button"
+                        id="save-automation-button-${automationId}">Save</button>
+                <button type="button"
+                        class="delete-automation-button negative-button"
+                        id="delete-automation-button-${automationId}">Delete</button>
+                <div id="automation-success-${automationId}" style="display: none;"></div>
+            </div>
+        `;
+
+        let saveAutomationButtonEl = automationEl.querySelector(`#save-automation-button-${automation.id}`);
+        saveAutomationButtonEl.addEventListener("click", async () => { await saveAutomation(automation.id); });
+        let deleteAutomationButtonEl = automationEl.querySelector(`#delete-automation-button-${automation.id}`);
+        deleteAutomationButtonEl.addEventListener("click", () => { deleteAutomation(automation.id); });
+
+        return automationEl.firstElementChild;
+    }
+
+    function listAutomations() {
+        const AutomationsList = document.getElementById("automations");
+        fetch('/api/automations')
+            .then(response => response.json())
+            .then(automations => {
+                if (!automations?.length > 0) return;
+                AutomationsList.innerHTML = ''; // Clear existing content
+                AutomationsList.append(...automations.map(automation => generateAutomationRow(automation)))
+            });
+    }
+    listAutomations();
+
+    function enableSaveOnlyWhenInputsChanged() {
+        const inputs = document.querySelectorAll('input[name="schedule"], textarea[name="query-to-run"], input[name="subject"]');
+        inputs.forEach(input => {
+            input.addEventListener('change', function() {
+                // Get automation id by splitting the id by "-" and taking all elements after the second one
+                const automationId = this.id.split("-").slice(2).join("-");
+                let anyChanged = false;
+                let inputNameStubs = ["subject", "query-to-run", "schedule"]
+                for (let stub of inputNameStubs) {
+                    let el = document.getElementById(`automation-${stub}-${automationId}`);
+                    let originalValue = el.getAttribute('data-original');
+                    let currentValue = el.value;
+                    if (originalValue !== currentValue) {
+                        anyChanged = true;
+                        break;
+                    }
+                }
+                document.getElementById(`save-automation-button-${automationId}`).disabled = !anyChanged;
+            });
+        });
+    }
+
+    async function saveAutomation(automationId, create=false) {
+        const subject = encodeURIComponent(document.getElementById(`automation-subject-${automationId}`).value);
+        const queryToRun = encodeURIComponent(document.getElementById(`automation-queryToRun-${automationId}`).value);
+        const scheduleEl = document.getElementById(`automation-schedule-${automationId}`);
+        const notificationEl = document.getElementById(`automation-success-${automationId}`);
+        const saveButtonEl = document.getElementById(`save-automation-button-${automationId}`);
+        const actOn = create ? "Create" : "Save";
+
+        if (subject === "" || queryToRun == "" || scheduleEl.value == "") {
+            return;
+        }
+
+        // Get client location information from IP
+        const ip_response = await fetch("https://ipapi.co/json")
+        const ip_data = await ip_response.json();
+
+        // Get cron string from natural language user schedule, if changed
+        const crontime = scheduleEl.getAttribute('data-original') !== scheduleEl.value ? getCronString(scheduleEl.value) : scheduleEl.getAttribute('data-cron');
+        const encodedCrontime = encodeURIComponent(crontime);
+
+        // Construct query string and select method for API call
+        let query_string = `q=${queryToRun}&subject=${subject}&crontime=${encodedCrontime}&city=${ip_data.city}&region=${ip_data.region}&country=${ip_data.country_name}&timezone=${ip_data.timezone}`;
+        let method = "POST";
+        if (!create) {
+            query_string += `&automation_id=${automationId}`;
+            method = "PUT"
+        }
+
+        fetch(`/api/automation?${query_string}`, {
+            method: method,
+            headers: {
+                'Content-Type': 'application/json',
+            },
+        })
+        .then(response => response.ok ? response.json() : Promise.reject(data))
+        .then(automation => {
+            if (create) {
+                const automationEl = document.getElementById(`automation-card-${automationId}`);
+                automationEl.replaceWith(generateAutomationRow(automation));
+            } else {
+                updateAutomationRow(automation);
+            }
+
+            notificationEl.style.display = "none";
+            saveButtonEl.textContent = `✅ Automation ${actOn}d`;
+            setTimeout(function() {
+                saveButtonEl.textContent = "Save";
+            }, 2000);
+        })
+        .catch(error => {
+            notificationEl.textContent = `⚠️ Failed to ${actOn.toLowerCase()} automations.`;
+            notificationEl.style.display = "block";
+            saveButtonEl.textContent = `⚠️ Failed to ${actOn.toLowerCase()} automations`;
+            setTimeout(function() {
+                saveButtonEl.textContent = actOn;
+            }, 2000);
+            return;
+        });
+    }
+
+    const create_automation_button = document.getElementById("create-automation-button");
+    create_automation_button.addEventListener("click", function(event) {
+        event.preventDefault();
+        var automationEl = document.createElement("div");
+        automationEl.classList.add("card");
+        automationEl.classList.add("automation");
+        const placeholderId = Date.now();
+        automationEl.id = "automation-card-" + placeholderId;
+        automationEl.innerHTML = `
+            <label for="subject">Subject</label>
+            <input type="text"
+                   id="automation-subject-${placeholderId}"
+                   name="subject"
+                   placeholder="My Personal Newsletter">
+            <label for="query-to-run">Query to Run</label>
+            <textarea id="automation-queryToRun-${placeholderId}" placeholder="Share a Newsletter including: 1. Weather forecast for this Week. 2. A Book Highlight from my Notes. 3. Recap News from Last Week"></textarea>
+            <label for="schedule">Schedule</label>
+            <input type="text"
+                   id="automation-schedule-${placeholderId}"
+                   name="schedule"
+                   placeholder="9AM every morning">
+            <button type="button"
+                    class="save-automation-button"
+                    onclick="saveAutomation(${placeholderId}, true)"
+                    id="save-automation-button-${placeholderId}">Create</button>
+            <button type="button"
+                    class="delete-automation-button"
+                    onclick="deleteAutomation(${placeholderId}, true)"
+                    id="delete-automation-button-${placeholderId}">Delete</button>
+            <div id="automation-success-${placeholderId}" style="display: none;"></div>
+        `;
+        document.getElementById("automations").insertBefore(automationEl, document.getElementById("automations").firstChild);
+    })
+</script>
+{% endblock %}

src/khoj/routers/api.py

@@ -8,7 +8,9 @@ import uuid
 from typing import Any, Callable, List, Optional, Union
 
 import cron_descriptor
+import pytz
 from apscheduler.job import Job
+from apscheduler.triggers.cron import CronTrigger
 from asgiref.sync import sync_to_async
 from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile
 from fastapi.requests import Request
@@ -33,6 +35,7 @@ from khoj.routers.helpers import (
     CommonQueryParams,
     ConversationCommandRateLimiter,
     create_automation,
+    schedule_automation,
     update_telemetry_state,
 )
 from khoj.search_filter.date_filter import DateFilter
@@ -41,7 +44,7 @@ from khoj.search_filter.word_filter import WordFilter
 from khoj.search_type import text_search
 from khoj.utils import state
 from khoj.utils.config import OfflineChatProcessorModel
-from khoj.utils.helpers import ConversationCommand, timer
+from khoj.utils.helpers import ConversationCommand, is_none_or_empty, timer
 from khoj.utils.rawconfig import LocationData, SearchResponse
 from khoj.utils.state import SearchType
 
@@ -411,8 +414,8 @@ def delete_automation(request: Request, automation_id: str) -> Response:
 
     try:
         automation_info = AutomationAdapters.delete_automation(user, automation_id)
-    except ValueError as e:
+    except ValueError:
-        return Response(content="Could not find automation", status_code=403)
+        return Response(status_code=204)
 
     # Return deleted automation information as a JSON response
     return Response(content=json.dumps(automation_info), media_type="application/json", status_code=200)
@@ -420,21 +423,44 @@ def delete_automation(request: Request, automation_id: str) -> Response:
 
 @api.post("/automation", response_class=Response)
 @requires(["authenticated"])
-async def make_automation(
+async def post_automation(
     request: Request,
     q: str,
+    subject: str,
+    crontime: str,
     city: Optional[str] = None,
     region: Optional[str] = None,
     country: Optional[str] = None,
     timezone: Optional[str] = None,
 ) -> Response:
     user: KhojUser = request.user.object
-    if city or region or country:
-        location = LocationData(city=city, region=region, country=country)
 
-    # Create automation with scheduling query and location data
+    # Perform validation checks
+    if is_none_or_empty(q) or is_none_or_empty(subject) or is_none_or_empty(crontime):
+        return Response(content="A query, subject and crontime is required", status_code=400)
+    if not cron_descriptor.get_description(crontime):
+        return Response(content="Invalid crontime", status_code=400)
+
+    # Normalize query parameters
+    # Add /automated_task prefix to query if not present
+    q = q.strip()
+    if not q.startswith("/automated_task"):
+        query_to_run = f"/automated_task {q}"
+    # Normalize crontime for AP Scheduler CronTrigger
+    crontime = crontime.strip()
+    if len(crontime.split(" ")) > 5:
+        # Truncate crontime to 5 fields
+        crontime = " ".join(crontime.split(" ")[:5])
+    # Convert crontime to standard unix crontime
+    crontime = crontime.replace("?", "*")
+    subject = subject.strip()
+
+    # Schedule automation with query_to_run, timezone, subject directly provided by user
     try:
-        automation, crontime, query_to_run, subject = await create_automation(q, location, timezone, user, request.url)
+        # Get user timezone
+        user_timezone = pytz.timezone(timezone)
+        # Use the query to run as the scheduling request if the scheduling request is unset
+        automation = await schedule_automation(query_to_run, subject, crontime, user_timezone, q, user, request.url)
     except Exception as e:
         logger.error(f"Error creating automation {q} for {user.email}: {e}")
         return Response(
@@ -449,25 +475,36 @@ async def make_automation(
         "id": automation.id,
         "subject": subject,
         "query_to_run": query_to_run,
-        "scheduling_request": crontime,
+        "scheduling_request": query_to_run,
         "schedule": schedule,
+        "crontime": crontime,
         "next": automation.next_run_time.strftime("%Y-%m-%d %I:%M %p %Z"),
     }
+
     # Return information about the created automation as a JSON response
     return Response(content=json.dumps(automation_info), media_type="application/json", status_code=200)
 
 
-@api.patch("/automation", response_class=Response)
+@api.put("/automation", response_class=Response)
 @requires(["authenticated"])
 def edit_job(
-    request: Request, automation_id: str, query_to_run: Optional[str] = None, crontime: Optional[str] = None
+    request: Request,
+    automation_id: str,
+    q: Optional[str],
+    subject: Optional[str],
+    crontime: Optional[str],
+    city: Optional[str] = None,
+    region: Optional[str] = None,
+    country: Optional[str] = None,
+    timezone: Optional[str] = None,
 ) -> Response:
     user: KhojUser = request.user.object
 
     # Perform validation checks
-    # Check at least one of query or crontime is provided
+    if is_none_or_empty(q) or is_none_or_empty(subject) or is_none_or_empty(crontime):
-    if not query_to_run and not crontime:
+        return Response(content="A query, subject and crontime is required", status_code=400)
-        return Response(content="A query or crontime is required", status_code=400)
+    if not cron_descriptor.get_description(crontime):
+        return Response(content="Invalid crontime", status_code=400)
 
     # Check, get automation to edit
     try:
@@ -475,14 +512,31 @@ def edit_job(
     except ValueError as e:
         return Response(content="Invalid automation", status_code=403)
 
+    # Normalize query parameters
     # Add /automated_task prefix to query if not present
-    if not query_to_run.startswith("/automated_task"):
+    q = q.strip()
-        query_to_run = f"/automated_task {query_to_run}"
+    if not q.startswith("/automated_task"):
+        query_to_run = f"/automated_task {q}"
+    # Normalize crontime for AP Scheduler CronTrigger
+    crontime = crontime.strip()
+    if len(crontime.split(" ")) > 5:
+        # Truncate crontime to 5 fields
+        crontime = " ".join(crontime.split(" ")[:5])
+    # Convert crontime to standard unix crontime
+    crontime = crontime.replace("?", "*")
 
-    # Update automation with new query
+    # Construct updated automation metadata
     automation_metadata = json.loads(automation.name)
     automation_metadata["query_to_run"] = query_to_run
-    automation.modify(kwargs={"query_to_run": query_to_run}, name=json.dumps(automation_metadata))
+    automation_metadata["subject"] = subject.strip()
+
+    # Modify automation with updated query, subject, crontime
+    automation.modify(kwargs={"query_to_run": query_to_run, "subject": subject}, name=json.dumps(automation_metadata))
+
+    # Reschedule automation if crontime updated
+    trigger = CronTrigger.from_crontab(crontime)
+    if automation.trigger != trigger:
+        automation.reschedule(trigger=trigger)
 
     # Collate info about the modified user automation
     automation_info = {

src/khoj/routers/helpers.py

@@ -922,14 +922,32 @@ async def create_automation(
     q: str, location: LocationData, timezone: str, user: KhojUser, calling_url: URL, meta_log: dict = {}
 ):
     user_timezone = pytz.timezone(timezone)
-    crontime_string, query_to_run, subject = await schedule_query(q, location, meta_log)
+    crontime, query_to_run, subject = await schedule_query(q, location, meta_log)
-    trigger = CronTrigger.from_crontab(crontime_string, user_timezone)
+    job = await schedule_automation(query_to_run, subject, crontime, user_timezone, q, user, calling_url)
+    return job, crontime, query_to_run, subject
+
+
+async def schedule_automation(
+    query_to_run: str,
+    subject: str,
+    crontime: str,
+    user_timezone,
+    scheduling_request: str,
+    user: KhojUser,
+    calling_url: URL,
+):
+    trigger = CronTrigger.from_crontab(crontime, user_timezone)
     # Generate id and metadata used by task scheduler and process locks for the task runs
     job_metadata = json.dumps(
-        {"query_to_run": query_to_run, "scheduling_request": q, "subject": subject, "crontime": crontime_string}
+        {
+            "query_to_run": query_to_run,
+            "scheduling_request": scheduling_request,
+            "subject": subject,
+            "crontime": crontime,
+        }
     )
-    query_id = hashlib.md5(f"{query_to_run}{crontime_string}".encode("utf-8")).hexdigest()
+    query_id = hashlib.md5(f"{query_to_run}_{crontime}".encode("utf-8")).hexdigest()
-    job_id = f"automation_{user.uuid}_{crontime_string}_{query_id}"
+    job_id = f"automation_{user.uuid}_{query_id}"
     job = await sync_to_async(state.scheduler.add_job)(
         run_with_process_lock,
         trigger=trigger,
@@ -939,7 +957,7 @@ async def create_automation(
         ),
         kwargs={
             "query_to_run": query_to_run,
-            "scheduling_request": q,
+            "scheduling_request": scheduling_request,
             "subject": subject,
             "user": user,
             "calling_url": calling_url,
@@ -949,7 +967,7 @@ async def create_automation(
         max_instances=2,  # Allow second instance to kill any previous instance with stale lock
         jitter=30,
     )
-    return job, crontime_string, query_to_run, subject
+    return job
 
 
 def construct_automation_created_message(automation: Job, crontime: str, query_to_run: str, subject: str, url: URL):
@@ -968,5 +986,5 @@ def construct_automation_created_message(automation: Job, crontime: str, query_t
 - Schedule: `{schedule}`
 - Next Run At: {next_run_time}
 
-Manage your tasks [here](/config#automations).
+Manage your automations [here](/automations).
     """.strip()

src/khoj/routers/web_client.py

@@ -11,6 +11,7 @@ from starlette.authentication import has_required_scope, requires
 from khoj.database import adapters
 from khoj.database.adapters import (
     AgentAdapters,
+    AutomationAdapters,
     ConversationAdapters,
     EntryAdapters,
     get_user_github_config,
@@ -364,3 +365,23 @@ def computer_config_page(request: Request):
             "khoj_version": state.khoj_version,
         },
     )
+
+
+@web_client.get("/automations", response_class=HTMLResponse)
+@requires(["authenticated"], redirect="login_page")
+def automations_config_page(request: Request):
+    user = request.user.object
+    user_picture = request.session.get("user", {}).get("picture")
+    has_documents = EntryAdapters.user_has_entries(user=user)
+
+    return templates.TemplateResponse(
+        "config_automation.html",
+        context={
+            "request": request,
+            "username": user.username,
+            "user_photo": user_picture,
+            "is_active": has_required_scope(request, ["premium"]),
+            "has_documents": has_documents,
+            "khoj_version": state.khoj_version,
+        },
+    )