From bd5008136a90cb7c2527f4a641783928c745ad79 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Tue, 30 Apr 2024 19:12:04 +0530 Subject: [PATCH] Move automations into independent page. Allow direct automation - Previously it was a section in the settings page. Move it to independent, top-level page to improve visibility of feature - Calculate crontime from natural language on web client before sending it to to server for saving new/updated schedule to disk. - Avoids round-trip of call to chat model - Convert POST /api/automation API endpoint into a direct request for automation with query_to_run, subject and schedule provided via the automation page. This allows more granular control to create automation - Make the POST automations endpoint more robust; runs validation checks, normalizes parameters --- .../interface/web/assets/natural-cron.min.js | 1 + src/khoj/interface/web/config.html | 33 --- src/khoj/interface/web/config_automation.html | 251 ++++++++++++++++++ src/khoj/routers/api.py | 90 +++++-- src/khoj/routers/helpers.py | 34 ++- src/khoj/routers/web_client.py | 21 ++ 6 files changed, 371 insertions(+), 59 deletions(-) create mode 100644 src/khoj/interface/web/assets/natural-cron.min.js create mode 100644 src/khoj/interface/web/config_automation.html diff --git a/src/khoj/interface/web/assets/natural-cron.min.js b/src/khoj/interface/web/assets/natural-cron.min.js new file mode 100644 index 00000000..d974dc4f --- /dev/null +++ b/src/khoj/interface/web/assets/natural-cron.min.js @@ -0,0 +1 @@ +!function(e){"object"==typeof exports&&"undefined"!=typeof module?module.exports=e():"function"==typeof define&&define.amd?define([],e):("undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:this).getCronString=e()}(function(){return function r(a,o,s){function u(n,e){if(!o[n]){if(!a[n]){var t="function"==typeof require&&require;if(!e&&t)return t(n,!0);if(i)return i(n,!0);throw(t=new Error("Cannot find module '"+n+"'")).code="MODULE_NOT_FOUND",t}t=o[n]={exports:{}},a[n][0].call(t.exports,function(e){return u(a[n][1][e]||e)},t,t.exports,r,a,o,s)}return o[n].exports}for(var i="function"==typeof require&&require,e=0;e {% endif %} -
-

Automations

-
-
- Automations -

- Automations -

-
-
-

Manage your automations

-
- - - - - - - - - - - -
NameScheduling RequestQuery to RunScheduleActions
-
- -
-
-
- {% if billing_enabled %}

Billing

diff --git a/src/khoj/interface/web/config_automation.html b/src/khoj/interface/web/config_automation.html new file mode 100644 index 00000000..89074796 --- /dev/null +++ b/src/khoj/interface/web/config_automation.html @@ -0,0 +1,251 @@ +{% extends "base_config.html" %} +{% block content %} +
+
+

+ Automate + Automate +
+ ⓘ Help +
+

+
+

Automations

+ +
+
+
+
+ + + +{% endblock %} diff --git a/src/khoj/routers/api.py b/src/khoj/routers/api.py index 0c22e5fe..d3a0ef1d 100644 --- a/src/khoj/routers/api.py +++ b/src/khoj/routers/api.py @@ -8,7 +8,9 @@ import uuid from typing import Any, Callable, List, Optional, Union import cron_descriptor +import pytz from apscheduler.job import Job +from apscheduler.triggers.cron import CronTrigger from asgiref.sync import sync_to_async from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile from fastapi.requests import Request @@ -33,6 +35,7 @@ from khoj.routers.helpers import ( CommonQueryParams, ConversationCommandRateLimiter, create_automation, + schedule_automation, update_telemetry_state, ) from khoj.search_filter.date_filter import DateFilter @@ -41,7 +44,7 @@ from khoj.search_filter.word_filter import WordFilter from khoj.search_type import text_search from khoj.utils import state from khoj.utils.config import OfflineChatProcessorModel -from khoj.utils.helpers import ConversationCommand, timer +from khoj.utils.helpers import ConversationCommand, is_none_or_empty, timer from khoj.utils.rawconfig import LocationData, SearchResponse from khoj.utils.state import SearchType @@ -411,8 +414,8 @@ def delete_automation(request: Request, automation_id: str) -> Response: try: automation_info = AutomationAdapters.delete_automation(user, automation_id) - except ValueError as e: - return Response(content="Could not find automation", status_code=403) + except ValueError: + return Response(status_code=204) # Return deleted automation information as a JSON response return Response(content=json.dumps(automation_info), media_type="application/json", status_code=200) @@ -420,21 +423,44 @@ def delete_automation(request: Request, automation_id: str) -> Response: @api.post("/automation", response_class=Response) @requires(["authenticated"]) -async def make_automation( +async def post_automation( request: Request, q: str, + subject: str, + crontime: str, city: Optional[str] = None, region: Optional[str] = None, country: Optional[str] = None, timezone: Optional[str] = None, ) -> Response: user: KhojUser = request.user.object - if city or region or country: - location = LocationData(city=city, region=region, country=country) - # Create automation with scheduling query and location data + # Perform validation checks + if is_none_or_empty(q) or is_none_or_empty(subject) or is_none_or_empty(crontime): + return Response(content="A query, subject and crontime is required", status_code=400) + if not cron_descriptor.get_description(crontime): + return Response(content="Invalid crontime", status_code=400) + + # Normalize query parameters + # Add /automated_task prefix to query if not present + q = q.strip() + if not q.startswith("/automated_task"): + query_to_run = f"/automated_task {q}" + # Normalize crontime for AP Scheduler CronTrigger + crontime = crontime.strip() + if len(crontime.split(" ")) > 5: + # Truncate crontime to 5 fields + crontime = " ".join(crontime.split(" ")[:5]) + # Convert crontime to standard unix crontime + crontime = crontime.replace("?", "*") + subject = subject.strip() + + # Schedule automation with query_to_run, timezone, subject directly provided by user try: - automation, crontime, query_to_run, subject = await create_automation(q, location, timezone, user, request.url) + # Get user timezone + user_timezone = pytz.timezone(timezone) + # Use the query to run as the scheduling request if the scheduling request is unset + automation = await schedule_automation(query_to_run, subject, crontime, user_timezone, q, user, request.url) except Exception as e: logger.error(f"Error creating automation {q} for {user.email}: {e}") return Response( @@ -449,25 +475,36 @@ async def make_automation( "id": automation.id, "subject": subject, "query_to_run": query_to_run, - "scheduling_request": crontime, + "scheduling_request": query_to_run, "schedule": schedule, + "crontime": crontime, "next": automation.next_run_time.strftime("%Y-%m-%d %I:%M %p %Z"), } + # Return information about the created automation as a JSON response return Response(content=json.dumps(automation_info), media_type="application/json", status_code=200) -@api.patch("/automation", response_class=Response) +@api.put("/automation", response_class=Response) @requires(["authenticated"]) def edit_job( - request: Request, automation_id: str, query_to_run: Optional[str] = None, crontime: Optional[str] = None + request: Request, + automation_id: str, + q: Optional[str], + subject: Optional[str], + crontime: Optional[str], + city: Optional[str] = None, + region: Optional[str] = None, + country: Optional[str] = None, + timezone: Optional[str] = None, ) -> Response: user: KhojUser = request.user.object # Perform validation checks - # Check at least one of query or crontime is provided - if not query_to_run and not crontime: - return Response(content="A query or crontime is required", status_code=400) + if is_none_or_empty(q) or is_none_or_empty(subject) or is_none_or_empty(crontime): + return Response(content="A query, subject and crontime is required", status_code=400) + if not cron_descriptor.get_description(crontime): + return Response(content="Invalid crontime", status_code=400) # Check, get automation to edit try: @@ -475,14 +512,31 @@ def edit_job( except ValueError as e: return Response(content="Invalid automation", status_code=403) + # Normalize query parameters # Add /automated_task prefix to query if not present - if not query_to_run.startswith("/automated_task"): - query_to_run = f"/automated_task {query_to_run}" + q = q.strip() + if not q.startswith("/automated_task"): + query_to_run = f"/automated_task {q}" + # Normalize crontime for AP Scheduler CronTrigger + crontime = crontime.strip() + if len(crontime.split(" ")) > 5: + # Truncate crontime to 5 fields + crontime = " ".join(crontime.split(" ")[:5]) + # Convert crontime to standard unix crontime + crontime = crontime.replace("?", "*") - # Update automation with new query + # Construct updated automation metadata automation_metadata = json.loads(automation.name) automation_metadata["query_to_run"] = query_to_run - automation.modify(kwargs={"query_to_run": query_to_run}, name=json.dumps(automation_metadata)) + automation_metadata["subject"] = subject.strip() + + # Modify automation with updated query, subject, crontime + automation.modify(kwargs={"query_to_run": query_to_run, "subject": subject}, name=json.dumps(automation_metadata)) + + # Reschedule automation if crontime updated + trigger = CronTrigger.from_crontab(crontime) + if automation.trigger != trigger: + automation.reschedule(trigger=trigger) # Collate info about the modified user automation automation_info = { diff --git a/src/khoj/routers/helpers.py b/src/khoj/routers/helpers.py index fda45469..7a8d869c 100644 --- a/src/khoj/routers/helpers.py +++ b/src/khoj/routers/helpers.py @@ -922,14 +922,32 @@ async def create_automation( q: str, location: LocationData, timezone: str, user: KhojUser, calling_url: URL, meta_log: dict = {} ): user_timezone = pytz.timezone(timezone) - crontime_string, query_to_run, subject = await schedule_query(q, location, meta_log) - trigger = CronTrigger.from_crontab(crontime_string, user_timezone) + crontime, query_to_run, subject = await schedule_query(q, location, meta_log) + job = await schedule_automation(query_to_run, subject, crontime, user_timezone, q, user, calling_url) + return job, crontime, query_to_run, subject + + +async def schedule_automation( + query_to_run: str, + subject: str, + crontime: str, + user_timezone, + scheduling_request: str, + user: KhojUser, + calling_url: URL, +): + trigger = CronTrigger.from_crontab(crontime, user_timezone) # Generate id and metadata used by task scheduler and process locks for the task runs job_metadata = json.dumps( - {"query_to_run": query_to_run, "scheduling_request": q, "subject": subject, "crontime": crontime_string} + { + "query_to_run": query_to_run, + "scheduling_request": scheduling_request, + "subject": subject, + "crontime": crontime, + } ) - query_id = hashlib.md5(f"{query_to_run}{crontime_string}".encode("utf-8")).hexdigest() - job_id = f"automation_{user.uuid}_{crontime_string}_{query_id}" + query_id = hashlib.md5(f"{query_to_run}_{crontime}".encode("utf-8")).hexdigest() + job_id = f"automation_{user.uuid}_{query_id}" job = await sync_to_async(state.scheduler.add_job)( run_with_process_lock, trigger=trigger, @@ -939,7 +957,7 @@ async def create_automation( ), kwargs={ "query_to_run": query_to_run, - "scheduling_request": q, + "scheduling_request": scheduling_request, "subject": subject, "user": user, "calling_url": calling_url, @@ -949,7 +967,7 @@ async def create_automation( max_instances=2, # Allow second instance to kill any previous instance with stale lock jitter=30, ) - return job, crontime_string, query_to_run, subject + return job def construct_automation_created_message(automation: Job, crontime: str, query_to_run: str, subject: str, url: URL): @@ -968,5 +986,5 @@ def construct_automation_created_message(automation: Job, crontime: str, query_t - Schedule: `{schedule}` - Next Run At: {next_run_time} -Manage your tasks [here](/config#automations). +Manage your automations [here](/automations). """.strip() diff --git a/src/khoj/routers/web_client.py b/src/khoj/routers/web_client.py index 9e3a39b5..047273e9 100644 --- a/src/khoj/routers/web_client.py +++ b/src/khoj/routers/web_client.py @@ -11,6 +11,7 @@ from starlette.authentication import has_required_scope, requires from khoj.database import adapters from khoj.database.adapters import ( AgentAdapters, + AutomationAdapters, ConversationAdapters, EntryAdapters, get_user_github_config, @@ -364,3 +365,23 @@ def computer_config_page(request: Request): "khoj_version": state.khoj_version, }, ) + + +@web_client.get("/automations", response_class=HTMLResponse) +@requires(["authenticated"], redirect="login_page") +def automations_config_page(request: Request): + user = request.user.object + user_picture = request.session.get("user", {}).get("picture") + has_documents = EntryAdapters.user_has_entries(user=user) + + return templates.TemplateResponse( + "config_automation.html", + context={ + "request": request, + "username": user.username, + "user_photo": user_picture, + "is_active": has_required_scope(request, ["premium"]), + "has_documents": has_documents, + "khoj_version": state.khoj_version, + }, + )