mirror of
https://github.com/penpot/penpot.git
synced 2025-05-23 05:16:13 +02:00
🎉 Add scheduled (cron based) tasks subsystem.
This commit is contained in:
parent
9bcb91ceae
commit
b005c3905f
12 changed files with 400 additions and 139 deletions
|
@ -3,7 +3,7 @@ CREATE EXTENSION IF NOT EXISTS "pgcrypto";
|
||||||
|
|
||||||
-- Modified At
|
-- Modified At
|
||||||
|
|
||||||
CREATE OR REPLACE FUNCTION update_modified_at()
|
CREATE FUNCTION update_modified_at()
|
||||||
RETURNS TRIGGER AS $updt$
|
RETURNS TRIGGER AS $updt$
|
||||||
BEGIN
|
BEGIN
|
||||||
NEW.modified_at := clock_timestamp();
|
NEW.modified_at := clock_timestamp();
|
||||||
|
|
|
@ -29,7 +29,7 @@ CREATE INDEX users__is_demo
|
||||||
AND is_demo IS true;
|
AND is_demo IS true;
|
||||||
|
|
||||||
--- Table used for register all used emails by the user
|
--- Table used for register all used emails by the user
|
||||||
CREATE TABLE IF NOT EXISTS user_emails (
|
CREATE TABLE user_emails (
|
||||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
|
||||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||||
|
@ -46,7 +46,7 @@ CREATE INDEX user_emails__user_id__idx
|
||||||
|
|
||||||
--- Table for user key value attributes
|
--- Table for user key value attributes
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS user_attrs (
|
CREATE TABLE user_attrs (
|
||||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
|
||||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||||
|
@ -60,7 +60,7 @@ CREATE TABLE IF NOT EXISTS user_attrs (
|
||||||
|
|
||||||
--- Table for store verification tokens
|
--- Table for store verification tokens
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS tokens (
|
CREATE TABLE tokens (
|
||||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
token text NOT NULL,
|
token text NOT NULL,
|
||||||
|
|
||||||
|
@ -72,7 +72,7 @@ CREATE TABLE IF NOT EXISTS tokens (
|
||||||
|
|
||||||
--- Table for store user sessions.
|
--- Table for store user sessions.
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS sessions (
|
CREATE TABLE sessions (
|
||||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
|
|
||||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
-- Tables
|
-- Tables
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS projects (
|
CREATE TABLE projects (
|
||||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
|
||||||
|
@ -11,7 +11,10 @@ CREATE TABLE IF NOT EXISTS projects (
|
||||||
name text NOT NULL
|
name text NOT NULL
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS project_users (
|
CREATE INDEX projects__user_id__idx
|
||||||
|
ON projects(user_id);
|
||||||
|
|
||||||
|
CREATE TABLE project_users (
|
||||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
||||||
|
|
||||||
|
@ -23,7 +26,13 @@ CREATE TABLE IF NOT EXISTS project_users (
|
||||||
PRIMARY KEY (user_id, project_id)
|
PRIMARY KEY (user_id, project_id)
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS project_files (
|
CREATE INDEX project_users__user_id__idx
|
||||||
|
ON project_users(user_id);
|
||||||
|
|
||||||
|
CREATE INDEX project_users__project_id__idx
|
||||||
|
ON project_users(project_id);
|
||||||
|
|
||||||
|
CREATE TABLE project_files (
|
||||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
||||||
|
@ -35,7 +44,26 @@ CREATE TABLE IF NOT EXISTS project_files (
|
||||||
deleted_at timestamptz DEFAULT NULL
|
deleted_at timestamptz DEFAULT NULL
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS project_file_users (
|
CREATE INDEX project_files__user_id__idx
|
||||||
|
ON project_files(user_id);
|
||||||
|
|
||||||
|
CREATE INDEX project_files__project_id__idx
|
||||||
|
ON project_files(project_id);
|
||||||
|
|
||||||
|
CREATE TABLE project_file_media (
|
||||||
|
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
|
file_id uuid NOT NULL REFERENCES project_files(id) ON DELETE CASCADE,
|
||||||
|
|
||||||
|
type text NOT NULL,
|
||||||
|
path text NOT NULL,
|
||||||
|
|
||||||
|
metadata bytea NULL DEFAULT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX project_file_media__file_id__idx
|
||||||
|
ON project_file_media(file_id);
|
||||||
|
|
||||||
|
CREATE TABLE project_file_users (
|
||||||
file_id uuid NOT NULL REFERENCES project_files(id) ON DELETE CASCADE,
|
file_id uuid NOT NULL REFERENCES project_files(id) ON DELETE CASCADE,
|
||||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
|
||||||
|
@ -47,7 +75,13 @@ CREATE TABLE IF NOT EXISTS project_file_users (
|
||||||
PRIMARY KEY (user_id, file_id)
|
PRIMARY KEY (user_id, file_id)
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS project_pages (
|
CREATE INDEX project_file_users__user_id__idx
|
||||||
|
ON project_file_users(user_id);
|
||||||
|
|
||||||
|
CREATE INDEX project_file_users__file_id__idx
|
||||||
|
ON project_file_users(file_id);
|
||||||
|
|
||||||
|
CREATE TABLE project_pages (
|
||||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
|
|
||||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
@ -64,7 +98,13 @@ CREATE TABLE IF NOT EXISTS project_pages (
|
||||||
data bytea NOT NULL
|
data bytea NOT NULL
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS project_page_snapshots (
|
CREATE INDEX project_pages__user_id__idx
|
||||||
|
ON project_pages(user_id);
|
||||||
|
|
||||||
|
CREATE INDEX project_pages__file_id__idx
|
||||||
|
ON project_pages(file_id);
|
||||||
|
|
||||||
|
CREATE TABLE project_page_snapshots (
|
||||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
|
|
||||||
user_id uuid NULL REFERENCES users(id) ON DELETE SET NULL,
|
user_id uuid NULL REFERENCES users(id) ON DELETE SET NULL,
|
||||||
|
@ -81,18 +121,11 @@ CREATE TABLE IF NOT EXISTS project_page_snapshots (
|
||||||
changes bytea NULL DEFAULT NULL
|
changes bytea NULL DEFAULT NULL
|
||||||
);
|
);
|
||||||
|
|
||||||
-- Indexes
|
CREATE INDEX project_page_snapshots__user_id__idx
|
||||||
|
ON project_page_snapshots(user_id);
|
||||||
|
|
||||||
CREATE INDEX projects__user_id__idx ON projects(user_id);
|
CREATE INDEX project_page_snapshots__page_id_id__idx
|
||||||
|
ON project_page_snapshots(page_id);
|
||||||
CREATE INDEX project_files__user_id__idx ON project_files(user_id);
|
|
||||||
CREATE INDEX project_files__project_id__idx ON project_files(project_id);
|
|
||||||
|
|
||||||
CREATE INDEX project_pages__user_id__idx ON project_pages(user_id);
|
|
||||||
CREATE INDEX project_pages__file_id__idx ON project_pages(file_id);
|
|
||||||
|
|
||||||
CREATE INDEX project_page_snapshots__page_id__idx ON project_page_snapshots(page_id);
|
|
||||||
CREATE INDEX project_page_snapshots__user_id__idx ON project_page_snapshots(user_id);
|
|
||||||
|
|
||||||
-- Triggers
|
-- Triggers
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,6 @@
|
||||||
CREATE TABLE IF NOT EXISTS tasks (
|
--- Tables
|
||||||
|
|
||||||
|
CREATE TABLE tasks (
|
||||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
|
|
||||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||||
|
@ -20,3 +22,20 @@ CREATE TABLE IF NOT EXISTS tasks (
|
||||||
|
|
||||||
CREATE INDEX tasks__scheduled_at__queue__idx
|
CREATE INDEX tasks__scheduled_at__queue__idx
|
||||||
ON tasks (scheduled_at, queue);
|
ON tasks (scheduled_at, queue);
|
||||||
|
|
||||||
|
CREATE TABLE scheduled_tasks (
|
||||||
|
id text PRIMARY KEY,
|
||||||
|
|
||||||
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||||
|
modified_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||||
|
executed_at timestamptz NULL DEFAULT NULL,
|
||||||
|
|
||||||
|
cron_expr text NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
--- Triggers
|
||||||
|
|
||||||
|
CREATE TRIGGER scheduled_tasks__modified_at__tgr
|
||||||
|
BEFORE UPDATE ON scheduled_tasks
|
||||||
|
FOR EACH ROW EXECUTE PROCEDURE update_modified_at();
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,4 @@
|
||||||
-- Tables
|
CREATE TABLE image_collections (
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS image_collections (
|
|
||||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
|
||||||
|
@ -11,9 +9,13 @@ CREATE TABLE IF NOT EXISTS image_collections (
|
||||||
name text NOT NULL
|
name text NOT NULL
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS images (
|
CREATE INDEX image_collections__user_id__idx
|
||||||
|
ON image_collections(user_id);
|
||||||
|
|
||||||
|
CREATE TABLE images (
|
||||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
collection_id uuid REFERENCES image_collections(id) ON DELETE CASCADE,
|
||||||
|
|
||||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||||
modified_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
modified_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||||
|
@ -22,20 +24,16 @@ CREATE TABLE IF NOT EXISTS images (
|
||||||
width int NOT NULL,
|
width int NOT NULL,
|
||||||
height int NOT NULL,
|
height int NOT NULL,
|
||||||
mimetype text NOT NULL,
|
mimetype text NOT NULL,
|
||||||
collection_id uuid REFERENCES image_collections(id)
|
|
||||||
ON DELETE SET NULL
|
|
||||||
DEFAULT NULL,
|
|
||||||
name text NOT NULL,
|
name text NOT NULL,
|
||||||
path text NOT NULL
|
path text NOT NULL
|
||||||
);
|
);
|
||||||
|
|
||||||
-- Indexes
|
CREATE INDEX images__user_id__idx
|
||||||
|
ON images(user_id);
|
||||||
|
|
||||||
CREATE INDEX image_collections__user_id__idx ON image_collections (user_id);
|
CREATE INDEX images__collection_id__idx
|
||||||
CREATE INDEX images__collection_id__idx ON images (collection_id);
|
ON images(collection_id);
|
||||||
CREATE INDEX images__user_id__idx ON images (user_id);
|
|
||||||
|
|
||||||
-- Triggers
|
|
||||||
|
|
||||||
CREATE TRIGGER image_collections__modified_at__tgr
|
CREATE TRIGGER image_collections__modified_at__tgr
|
||||||
BEFORE UPDATE ON image_collections
|
BEFORE UPDATE ON image_collections
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
-- Tables
|
-- Tables
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS icon_collections (
|
CREATE TABLE icon_collections (
|
||||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
|
||||||
|
@ -11,7 +11,7 @@ CREATE TABLE IF NOT EXISTS icon_collections (
|
||||||
name text NOT NULL
|
name text NOT NULL
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS icons (
|
CREATE TABLE icons (
|
||||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
|
||||||
|
|
|
@ -1 +1,2 @@
|
||||||
{instant uxbox.util.time/from-string}
|
{uxbox/instant uxbox.util.time/from-string
|
||||||
|
uxbox/cron uxbox.util.time/cron}
|
||||||
|
|
|
@ -31,8 +31,6 @@
|
||||||
(log/warn (str/istr "can't parse `~{key}` env value"))
|
(log/warn (str/istr "can't parse `~{key}` env value"))
|
||||||
default)))))
|
default)))))
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
;; --- Configuration Loading & Parsing
|
;; --- Configuration Loading & Parsing
|
||||||
|
|
||||||
(defn read-config
|
(defn read-config
|
||||||
|
|
|
@ -39,10 +39,21 @@
|
||||||
;; need to perform a maintenance and delete some old tasks.
|
;; need to perform a maintenance and delete some old tasks.
|
||||||
|
|
||||||
(def ^:private tasks
|
(def ^:private tasks
|
||||||
[#'uxbox.tasks.demo-gc/handler
|
{"demo-gc" #'uxbox.tasks.demo-gc/handler
|
||||||
#'uxbox.tasks.sendmail/handler])
|
"sendmail" #'uxbox.tasks.sendmail/handler})
|
||||||
|
|
||||||
(defstate small-tasks
|
(defstate tasks-worker
|
||||||
:start (as-> (impl/verticle tasks {:queue "default"}) $$
|
:start (as-> (impl/worker-verticle {:tasks tasks}) $$
|
||||||
(vc/deploy! system $$ {:instances 1})
|
(vc/deploy! system $$ {:instances 1})
|
||||||
(deref $$)))
|
(deref $$)))
|
||||||
|
|
||||||
|
(def ^:private schedule
|
||||||
|
[{:id "every 1 hour"
|
||||||
|
:cron #uxbox/cron "1 1 */1 * * ? *"
|
||||||
|
:fn #'uxbox.tasks.demo-gc/handler
|
||||||
|
:props {:foo "bar"}}])
|
||||||
|
|
||||||
|
(defstate scheduler
|
||||||
|
:start (as-> (impl/scheduler-verticle {:schedule schedule}) $$
|
||||||
|
(vc/deploy! system $$ {:instances 1 :worker true})
|
||||||
|
(deref $$)))
|
||||||
|
|
|
@ -16,6 +16,5 @@
|
||||||
(defn handler
|
(defn handler
|
||||||
{:uxbox.tasks/name "demo-gc"}
|
{:uxbox.tasks/name "demo-gc"}
|
||||||
[{:keys [props] :as task}]
|
[{:keys [props] :as task}]
|
||||||
(ex/raise :type :foobar
|
(Thread/sleep 500)
|
||||||
:code :foobaz
|
(prn (.getName (Thread/currentThread)) "demo-gc" (:id task) (:props task)))
|
||||||
:hint "Foo bar"))
|
|
||||||
|
|
|
@ -22,7 +22,16 @@
|
||||||
[uxbox.util.time :as tm]
|
[uxbox.util.time :as tm]
|
||||||
[vertx.core :as vc]
|
[vertx.core :as vc]
|
||||||
[vertx.timers :as vt])
|
[vertx.timers :as vt])
|
||||||
(:import java.time.Duration))
|
(:import
|
||||||
|
java.time.Duration
|
||||||
|
java.time.Instant
|
||||||
|
java.util.Date))
|
||||||
|
|
||||||
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||||
|
;; Implementation
|
||||||
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||||
|
|
||||||
|
;; --- Task Execution
|
||||||
|
|
||||||
(defn- string-strack-trace
|
(defn- string-strack-trace
|
||||||
[err]
|
[err]
|
||||||
|
@ -44,7 +53,6 @@
|
||||||
(-> (db/query-one conn sqlv)
|
(-> (db/query-one conn sqlv)
|
||||||
(p/then' (constantly nil)))))
|
(p/then' (constantly nil)))))
|
||||||
|
|
||||||
|
|
||||||
(def ^:private sql:mark-as-failed
|
(def ^:private sql:mark-as-failed
|
||||||
"update tasks
|
"update tasks
|
||||||
set scheduled_at = clock_timestamp() + '5 seconds'::interval,
|
set scheduled_at = clock_timestamp() + '5 seconds'::interval,
|
||||||
|
@ -127,19 +135,139 @@
|
||||||
values ($1, $2, $3, clock_timestamp()+cast($4::text as interval))
|
values ($1, $2, $3, clock_timestamp()+cast($4::text as interval))
|
||||||
returning id")
|
returning id")
|
||||||
|
|
||||||
(s/def ::name ::us/string)
|
|
||||||
(s/def ::delay ::us/integer)
|
|
||||||
(s/def ::props map?)
|
|
||||||
(s/def ::queue ::us/string)
|
|
||||||
(s/def ::task-options
|
|
||||||
(s/keys :req-un [::name ::delay]
|
|
||||||
:opt-un [::props ::queue]))
|
|
||||||
|
|
||||||
(defn- duration->pginterval
|
(defn- duration->pginterval
|
||||||
[^Duration d]
|
[^Duration d]
|
||||||
(->> (/ (.toMillis d) 1000.0)
|
(->> (/ (.toMillis d) 1000.0)
|
||||||
(format "%s seconds")))
|
(format "%s seconds")))
|
||||||
|
|
||||||
|
(defn- on-worker-start
|
||||||
|
[ctx {:keys [tasks] :as options}]
|
||||||
|
(vt/schedule! ctx (assoc options
|
||||||
|
::vt/fn #'event-loop-handler
|
||||||
|
::vt/delay 3000
|
||||||
|
::vt/repeat true)))
|
||||||
|
|
||||||
|
|
||||||
|
;; --- Task Scheduling
|
||||||
|
|
||||||
|
(def ^:privatr sql:upsert-scheduled-task
|
||||||
|
"insert into scheduled_tasks (id, cron_expr)
|
||||||
|
values ($1, $2)
|
||||||
|
on conflict (id)
|
||||||
|
do update set cron_expr=$2")
|
||||||
|
|
||||||
|
(defn- synchronize-schedule-item
|
||||||
|
[conn {:keys [id cron]}]
|
||||||
|
(-> (db/query-one conn [sql:upsert-scheduled-task id (str cron)])
|
||||||
|
(p/then' (constantly nil))))
|
||||||
|
|
||||||
|
(defn- synchronize-schedule
|
||||||
|
[schedule]
|
||||||
|
(db/with-atomic [conn db/pool]
|
||||||
|
(p/run! (partial synchronize-schedule-item conn) schedule)))
|
||||||
|
|
||||||
|
(def ^:private sql:lock-scheduled-task
|
||||||
|
"select id from scheduled_tasks where id=$1 for update skip locked")
|
||||||
|
|
||||||
|
(declare schedule-task)
|
||||||
|
|
||||||
|
(defn thr-name
|
||||||
|
[]
|
||||||
|
(.getName (Thread/currentThread)))
|
||||||
|
|
||||||
|
(defn- execute-scheduled-task
|
||||||
|
[{:keys [id cron] :as stask}]
|
||||||
|
(db/with-atomic [conn db/pool]
|
||||||
|
(-> (db/query-one conn [sql:lock-scheduled-task id])
|
||||||
|
(p/then (fn [result]
|
||||||
|
(if result
|
||||||
|
(do
|
||||||
|
(prn (thr-name) "execute-scheduled-task" "task-locked")
|
||||||
|
(-> (p/do! ((:fn stask) stask))
|
||||||
|
(p/catch (fn [e]
|
||||||
|
(log/warn "Excepton happens on executing scheduled task" e)
|
||||||
|
nil))))
|
||||||
|
(prn (thr-name) "execute-scheduled-task" "task-already-locked"))))
|
||||||
|
(p/finally (fn [v e]
|
||||||
|
(-> (vc/current-context)
|
||||||
|
(schedule-task stask)))))))
|
||||||
|
|
||||||
|
(defn ms-until-valid
|
||||||
|
[cron]
|
||||||
|
(s/assert tm/cron? cron)
|
||||||
|
(let [^Instant now (tm/now)
|
||||||
|
^Instant next (.toInstant (.getNextValidTimeAfter cron (Date/from now)))
|
||||||
|
^Duration duration (Duration/between now next)]
|
||||||
|
(.toMillis duration)))
|
||||||
|
|
||||||
|
(defn- schedule-task
|
||||||
|
[ctx {:keys [cron] :as stask}]
|
||||||
|
(let [ms (ms-until-valid cron)]
|
||||||
|
(prn (thr-name) "schedule-task" (:id stask) ms)
|
||||||
|
(vt/schedule! ctx (assoc stask
|
||||||
|
:ctx ctx
|
||||||
|
::vt/once true
|
||||||
|
::vt/delay ms
|
||||||
|
::vt/fn execute-scheduled-task))))
|
||||||
|
|
||||||
|
(defn- on-scheduler-start
|
||||||
|
[ctx {:keys [schedule] :as options}]
|
||||||
|
(-> (synchronize-schedule schedule)
|
||||||
|
(p/then' (fn [_]
|
||||||
|
(run! #(schedule-task ctx %) schedule)))))
|
||||||
|
|
||||||
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||||
|
;; Public API
|
||||||
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||||
|
|
||||||
|
;; --- Worker Verticle
|
||||||
|
|
||||||
|
(s/def ::callable (s/or :fn fn? :var var?))
|
||||||
|
(s/def ::max-batch-size ::us/integer)
|
||||||
|
(s/def ::max-retries ::us/integer)
|
||||||
|
(s/def ::tasks (s/map-of string? ::callable))
|
||||||
|
|
||||||
|
(s/def ::worker-verticle-options
|
||||||
|
(s/keys :req-un [::tasks]
|
||||||
|
:opt-un [::queue ::max-batch-size]))
|
||||||
|
|
||||||
|
(defn worker-verticle
|
||||||
|
[options]
|
||||||
|
(s/assert ::worker-verticle-options options)
|
||||||
|
(let [on-start #(on-worker-start % options)]
|
||||||
|
(vc/verticle {:on-start on-start})))
|
||||||
|
|
||||||
|
;; --- Scheduler Verticle
|
||||||
|
|
||||||
|
(s/def ::id string?)
|
||||||
|
(s/def ::cron tm/cron?)
|
||||||
|
(s/def ::fn ::callable)
|
||||||
|
(s/def ::props (s/nilable map?))
|
||||||
|
|
||||||
|
(s/def ::scheduled-task
|
||||||
|
(s/keys :req-un [::id ::cron ::fn]
|
||||||
|
:opt-un [::props]))
|
||||||
|
|
||||||
|
(s/def ::schedule (s/coll-of ::scheduled-task))
|
||||||
|
|
||||||
|
(s/def ::scheduler-verticle-options
|
||||||
|
(s/keys :opt-un [::schedule]))
|
||||||
|
|
||||||
|
(defn scheduler-verticle
|
||||||
|
[options]
|
||||||
|
(s/assert ::scheduler-verticle-options options)
|
||||||
|
(let [on-start #(on-scheduler-start % options)]
|
||||||
|
(vc/verticle {:on-start on-start})))
|
||||||
|
|
||||||
|
;; --- Schedule API
|
||||||
|
|
||||||
|
(s/def ::name ::us/string)
|
||||||
|
(s/def ::delay ::us/integer)
|
||||||
|
(s/def ::queue ::us/string)
|
||||||
|
(s/def ::task-options
|
||||||
|
(s/keys :req-un [::name ::delay]
|
||||||
|
:opt-un [::props ::queue]))
|
||||||
|
|
||||||
(defn schedule!
|
(defn schedule!
|
||||||
[conn {:keys [name delay props queue key] :as options}]
|
[conn {:keys [name delay props queue key] :as options}]
|
||||||
(us/assert ::task-options options)
|
(us/assert ::task-options options)
|
||||||
|
@ -149,43 +277,3 @@
|
||||||
props (blob/encode props)]
|
props (blob/encode props)]
|
||||||
(-> (db/query-one conn [sql:insert-new-task name props queue duration])
|
(-> (db/query-one conn [sql:insert-new-task name props queue duration])
|
||||||
(p/then' (fn [task] (:id task))))))
|
(p/then' (fn [task] (:id task))))))
|
||||||
|
|
||||||
(defn- on-start
|
|
||||||
[ctx handlers options]
|
|
||||||
(vt/schedule! ctx (assoc options
|
|
||||||
::vt/fn #'event-loop-handler
|
|
||||||
::vt/delay 3000
|
|
||||||
::vt/repeat true
|
|
||||||
:handlers handlers)))
|
|
||||||
|
|
||||||
(defn- resolve-handlers
|
|
||||||
[tasks]
|
|
||||||
(s/assert (s/coll-of ::callable) tasks)
|
|
||||||
(reduce (fn [acc f]
|
|
||||||
(let [task-name (:uxbox.tasks/name (meta f))]
|
|
||||||
(if task-name
|
|
||||||
(assoc acc task-name f)
|
|
||||||
(do
|
|
||||||
(log/warn "skiping task, no name provided in metadata" (pr-str f))
|
|
||||||
acc))))
|
|
||||||
{}
|
|
||||||
tasks))
|
|
||||||
|
|
||||||
(s/def ::callable (s/or :fn fn? :var var?))
|
|
||||||
(s/def ::max-batch-size ::us/integer)
|
|
||||||
(s/def ::max-retries ::us/integer)
|
|
||||||
|
|
||||||
(s/def ::verticle-tasks
|
|
||||||
(s/coll-of ::callable))
|
|
||||||
|
|
||||||
(s/def ::verticle-options
|
|
||||||
(s/keys :opt-un [::queue ::max-batch-size]))
|
|
||||||
|
|
||||||
(defn verticle
|
|
||||||
[tasks options]
|
|
||||||
(s/assert ::verticle-tasks tasks)
|
|
||||||
(s/assert ::verticle-options options)
|
|
||||||
(let [handlers (resolve-handlers tasks)
|
|
||||||
on-start #(on-start % handlers options)]
|
|
||||||
(vc/verticle {:on-start on-start})))
|
|
||||||
|
|
||||||
|
|
|
@ -6,50 +6,16 @@
|
||||||
|
|
||||||
(ns uxbox.util.time
|
(ns uxbox.util.time
|
||||||
(:require
|
(:require
|
||||||
#_[suricatta.proto :as sp]
|
[uxbox.common.exceptions :as ex]
|
||||||
#_[suricatta.impl :as si]
|
|
||||||
[cognitect.transit :as t])
|
[cognitect.transit :as t])
|
||||||
(:import java.time.Instant
|
(:import
|
||||||
java.time.OffsetDateTime
|
java.time.Instant
|
||||||
java.time.Duration))
|
java.time.OffsetDateTime
|
||||||
|
java.time.Duration
|
||||||
|
org.apache.logging.log4j.core.util.CronExpression))
|
||||||
|
|
||||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||||
;; Serialization Layer conversions
|
;; Instant & Duration
|
||||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
||||||
|
|
||||||
(declare from-string)
|
|
||||||
|
|
||||||
(def ^:private instant-write-handler
|
|
||||||
(t/write-handler
|
|
||||||
(constantly "m")
|
|
||||||
(fn [v] (str (.toEpochMilli v)))))
|
|
||||||
|
|
||||||
(def ^:private offset-datetime-write-handler
|
|
||||||
(t/write-handler
|
|
||||||
(constantly "m")
|
|
||||||
(fn [v] (str (.toEpochMilli (.toInstant v))))))
|
|
||||||
|
|
||||||
(def ^:private read-handler
|
|
||||||
(t/read-handler
|
|
||||||
(fn [v] (-> (Long/parseLong v)
|
|
||||||
(Instant/ofEpochMilli)))))
|
|
||||||
|
|
||||||
(def +read-handlers+
|
|
||||||
{"m" read-handler})
|
|
||||||
|
|
||||||
(def +write-handlers+
|
|
||||||
{Instant instant-write-handler
|
|
||||||
OffsetDateTime offset-datetime-write-handler})
|
|
||||||
|
|
||||||
(defmethod print-method Instant
|
|
||||||
[mv ^java.io.Writer writer]
|
|
||||||
(.write writer (str "#instant \"" (.toString mv) "\"")))
|
|
||||||
|
|
||||||
(defmethod print-dup Instant [o w]
|
|
||||||
(print-method o w))
|
|
||||||
|
|
||||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
||||||
;; Helpers
|
|
||||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||||
|
|
||||||
(defn from-string
|
(defn from-string
|
||||||
|
@ -90,4 +56,152 @@
|
||||||
java.time.Duration
|
java.time.Duration
|
||||||
(inst-ms* [v] (.toMillis ^java.time.Duration v)))
|
(inst-ms* [v] (.toMillis ^java.time.Duration v)))
|
||||||
|
|
||||||
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||||
|
;; Cron Expression
|
||||||
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||||
|
|
||||||
|
;; Cron expressions are comprised of 6 required fields and one
|
||||||
|
;; optional field separated by white space. The fields respectively
|
||||||
|
;; are described as follows:
|
||||||
|
;;
|
||||||
|
;; Field Name Allowed Values Allowed Special Characters
|
||||||
|
;; Seconds 0-59 , - * /
|
||||||
|
;; Minutes 0-59 , - * /
|
||||||
|
;; Hours 0-23 , - * /
|
||||||
|
;; Day-of-month 1-31 , - * ? / L W
|
||||||
|
;; Month 0-11 or JAN-DEC , - * /
|
||||||
|
;; Day-of-Week 1-7 or SUN-SAT , - * ? / L #
|
||||||
|
;; Year (Optional) empty, 1970-2199 , - * /
|
||||||
|
;;
|
||||||
|
;; The '*' character is used to specify all values. For example, "*"
|
||||||
|
;; in the minute field means "every minute".
|
||||||
|
;;
|
||||||
|
;; The '?' character is allowed for the day-of-month and day-of-week
|
||||||
|
;; fields. It is used to specify 'no specific value'. This is useful
|
||||||
|
;; when you need to specify something in one of the two fields, but
|
||||||
|
;; not the other.
|
||||||
|
;;
|
||||||
|
;; The '-' character is used to specify ranges For example "10-12" in
|
||||||
|
;; the hour field means "the hours 10, 11 and 12".
|
||||||
|
;;
|
||||||
|
;; The ',' character is used to specify additional values. For
|
||||||
|
;; example "MON,WED,FRI" in the day-of-week field means "the days
|
||||||
|
;; Monday, Wednesday, and Friday".
|
||||||
|
;;
|
||||||
|
;; The '/' character is used to specify increments. For example "0/15"
|
||||||
|
;; in the seconds field means "the seconds 0, 15, 30, and
|
||||||
|
;; 45". And "5/15" in the seconds field means "the seconds 5, 20, 35,
|
||||||
|
;; and 50". Specifying '*' before the '/' is equivalent to specifying
|
||||||
|
;; 0 is the value to start with. Essentially, for each field in the
|
||||||
|
;; expression, there is a set of numbers that can be turned on or
|
||||||
|
;; off. For seconds and minutes, the numbers range from 0 to 59. For
|
||||||
|
;; hours 0 to 23, for days of the month 0 to 31, and for months 0 to
|
||||||
|
;; 11 (JAN to DEC). The "/" character simply helps you turn on
|
||||||
|
;; every "nth" value in the given set. Thus "7/6" in the month field
|
||||||
|
;; only turns on month "7", it does NOT mean every 6th month, please
|
||||||
|
;; note that subtlety.
|
||||||
|
;;
|
||||||
|
;; The 'L' character is allowed for the day-of-month and day-of-week
|
||||||
|
;; fields. This character is short-hand for "last", but it has
|
||||||
|
;; different meaning in each of the two fields. For example, the
|
||||||
|
;; value "L" in the day-of-month field means "the last day of the
|
||||||
|
;; month" - day 31 for January, day 28 for February on non-leap
|
||||||
|
;; years. If used in the day-of-week field by itself, it simply
|
||||||
|
;; means "7" or "SAT". But if used in the day-of-week field after
|
||||||
|
;; another value, it means "the last xxx day of the month" - for
|
||||||
|
;; example "6L" means "the last friday of the month". You can also
|
||||||
|
;; specify an offset from the last day of the month, such as "L-3"
|
||||||
|
;; which would mean the third-to-last day of the calendar month. When
|
||||||
|
;; using the 'L' option, it is important not to specify lists, or
|
||||||
|
;; ranges of values, as you'll get confusing/unexpected results.
|
||||||
|
;;
|
||||||
|
;; The 'W' character is allowed for the day-of-month field. This
|
||||||
|
;; character is used to specify the weekday (Monday-Friday) nearest
|
||||||
|
;; the given day. As an example, if you were to specify "15W" as the
|
||||||
|
;; value for the day-of-month field, the meaning is: "the nearest
|
||||||
|
;; weekday to the 15th of the month". So if the 15th is a Saturday,
|
||||||
|
;; the trigger will fire on Friday the 14th. If the 15th is a Sunday,
|
||||||
|
;; the trigger will fire on Monday the 16th. If the 15th is a Tuesday,
|
||||||
|
;; then it will fire on Tuesday the 15th. However if you specify "1W"
|
||||||
|
;; as the value for day-of-month, and the 1st is a Saturday, the
|
||||||
|
;; trigger will fire on Monday the 3rd, as it will not 'jump' over the
|
||||||
|
;; boundary of a month's days. The 'W' character can only be specified
|
||||||
|
;; when the day-of-month is a single day, not a range or list of days.
|
||||||
|
;;
|
||||||
|
;; The 'L' and 'W' characters can also be combined for the
|
||||||
|
;; day-of-month expression to yield 'LW', which translates to "last
|
||||||
|
;; weekday of the month".
|
||||||
|
;;
|
||||||
|
;; The '#' character is allowed for the day-of-week field. This
|
||||||
|
;; character is used to specify "the nth" XXX day of the month. For
|
||||||
|
;; example, the value of "6#3" in the day-of-week field means the
|
||||||
|
;; third Friday of the month (day 6 = Friday and "#3" = the 3rd one in
|
||||||
|
;; the month). Other examples: "2#1" = the first Monday of the month
|
||||||
|
;; and "4#5" = the fifth Wednesday of the month. Note that if you
|
||||||
|
;; specify "#5" and there is not 5 of the given day-of-week in the
|
||||||
|
;; month, then no firing will occur that month. If the '#' character
|
||||||
|
;; is used, there can only be one expression in the day-of-week
|
||||||
|
;; field ("3#1,6#3" is not valid, since there are two expressions).
|
||||||
|
;;
|
||||||
|
;; The legal characters and the names of months and days of the week
|
||||||
|
;; are not case sensitive.
|
||||||
|
|
||||||
|
(defn cron
|
||||||
|
"Creates an instance of CronExpression from string."
|
||||||
|
[s]
|
||||||
|
(try
|
||||||
|
(CronExpression. s)
|
||||||
|
(catch java.text.ParseException e
|
||||||
|
(ex/raise :type :parse
|
||||||
|
:code :invalid-cron-expression
|
||||||
|
:cause e
|
||||||
|
:context {:expr s}))))
|
||||||
|
|
||||||
|
(defn cron?
|
||||||
|
[v]
|
||||||
|
(instance? CronExpression v))
|
||||||
|
|
||||||
|
(defmethod print-method CronExpression
|
||||||
|
[mv ^java.io.Writer writer]
|
||||||
|
(.write writer (str "#uxbox/cron \"" (.toString mv) "\"")))
|
||||||
|
|
||||||
|
(defmethod print-dup CronExpression
|
||||||
|
[o w]
|
||||||
|
(print-ctor o (fn [o w] (print-dup (.toString o) w)) w))
|
||||||
|
|
||||||
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||||
|
;; Serialization
|
||||||
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||||
|
|
||||||
|
(declare from-string)
|
||||||
|
|
||||||
|
(def ^:private instant-write-handler
|
||||||
|
(t/write-handler
|
||||||
|
(constantly "m")
|
||||||
|
(fn [v] (str (.toEpochMilli v)))))
|
||||||
|
|
||||||
|
(def ^:private offset-datetime-write-handler
|
||||||
|
(t/write-handler
|
||||||
|
(constantly "m")
|
||||||
|
(fn [v] (str (.toEpochMilli (.toInstant v))))))
|
||||||
|
|
||||||
|
(def ^:private read-handler
|
||||||
|
(t/read-handler
|
||||||
|
(fn [v] (-> (Long/parseLong v)
|
||||||
|
(Instant/ofEpochMilli)))))
|
||||||
|
|
||||||
|
(def +read-handlers+
|
||||||
|
{"m" read-handler})
|
||||||
|
|
||||||
|
(def +write-handlers+
|
||||||
|
{Instant instant-write-handler
|
||||||
|
OffsetDateTime offset-datetime-write-handler})
|
||||||
|
|
||||||
|
(defmethod print-method Instant
|
||||||
|
[mv ^java.io.Writer writer]
|
||||||
|
(.write writer (str "#instant \"" (.toString mv) "\"")))
|
||||||
|
|
||||||
|
(defmethod print-dup Instant [o w]
|
||||||
|
(print-method o w))
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue