3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-05-12 15:46:38 +02:00
Code Issues Projects Releases Packages Wiki Activity

♻️ Make the worker abstraction more scalable

Browse source 
Start using redis for dispatcher to worker communication
and add the ability to start multiple threads to worker
for increase the concurrency.
This commit is contained in:
Andrey Antukh 2022-11-22 18:06:24 +01:00
parent 13a092b192
commit 0600b2abe4
16 changed files with 827 additions and 625 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.clj-kondo/config.edn
View file

@ -7,6 +7,7 @@
app.common.data/export clojure.core/def app.common.data/export clojure.core/def
app.db/with-atomic clojure.core/with-open app.db/with-atomic clojure.core/with-open
app.common.data.macros/get-in clojure.core/get-in app.common.data.macros/get-in clojure.core/get-in
app.common.data.macros/with-open clojure.core/with-open
app.common.data.macros/select-keys clojure.core/select-keys app.common.data.macros/select-keys clojure.core/select-keys
app.common.logging/with-context clojure.core/do} app.common.logging/with-context clojure.core/do}

6
backend/src/app/config.clj
View file

@ -106,7 +106,8 @@
(s/def ::file-change-snapshot-timeout ::dt/duration) (s/def ::file-change-snapshot-timeout ::dt/duration)
(s/def ::default-executor-parallelism ::us/integer) (s/def ::default-executor-parallelism ::us/integer)
(s/def ::worker-executor-parallelism ::us/integer) (s/def ::scheduled-executor-parallelism ::us/integer)
(s/def ::worker-parallelism ::us/integer)
(s/def ::authenticated-cookie-domain ::us/string) (s/def ::authenticated-cookie-domain ::us/string)
(s/def ::authenticated-cookie-name ::us/string) (s/def ::authenticated-cookie-name ::us/string)
@ -218,7 +219,8 @@
::default-rpc-rlimit ::default-rpc-rlimit
::error-report-webhook ::error-report-webhook
::default-executor-parallelism ::default-executor-parallelism
::worker-executor-parallelism ::scheduled-executor-parallelism
::worker-parallelism
::file-change-snapshot-every ::file-change-snapshot-every
::file-change-snapshot-timeout ::file-change-snapshot-timeout
::user-feedback-destination ::user-feedback-destination

15
backend/src/app/db.clj
View file

@ -493,3 +493,18 @@
(let [n (xact-check-param n) (let [n (xact-check-param n)
row (exec-one! conn ["select pg_try_advisory_xact_lock(?::bigint) as lock" n])] row (exec-one! conn ["select pg_try_advisory_xact_lock(?::bigint) as lock" n])]
(:lock row))) (:lock row)))
(defn sql-exception?
[cause]
(instance? java.sql.SQLException cause))
(defn connection-error?
[cause]
(and (sql-exception? cause)
(contains? #{"08003" "08006" "08001" "08004"}
(.getSQLState ^java.sql.SQLException cause))))
(defn serialization-error?
[cause]
(and (sql-exception? cause)
(= "40001" (.getSQLState ^java.sql.SQLException cause))))

202
backend/src/app/main.clj
View file

@ -9,8 +9,13 @@
[app.auth.oidc] [app.auth.oidc]
[app.common.logging :as l] [app.common.logging :as l]
[app.config :as cf] [app.config :as cf]
[app.db :as-alias db]
[app.metrics :as-alias mtx]
[app.metrics.definition :as-alias mdef] [app.metrics.definition :as-alias mdef]
[app.redis :as-alias rds]
[app.storage :as-alias sto]
[app.util.time :as dt] [app.util.time :as dt]
[app.worker :as-alias wrk]
[cuerdas.core :as str] [cuerdas.core :as str]
[integrant.core :as ig]) [integrant.core :as ig])
(:gen-class)) (:gen-class))
@ -120,90 +125,84 @@
::mdef/type :gauge}}) ::mdef/type :gauge}})
(def system-config (def system-config
{:app.db/pool {::db/pool
{:uri (cf/get :database-uri) {:uri (cf/get :database-uri)
:username (cf/get :database-username) :username (cf/get :database-username)
:password (cf/get :database-password) :password (cf/get :database-password)
:read-only (cf/get :database-readonly false) :read-only (cf/get :database-readonly false)
:metrics (ig/ref :app.metrics/metrics) :metrics (ig/ref ::mtx/metrics)
:migrations (ig/ref :app.migrations/all) :migrations (ig/ref :app.migrations/all)
:name :main :name :main
:min-size (cf/get :database-min-pool-size 0) :min-size (cf/get :database-min-pool-size 0)
:max-size (cf/get :database-max-pool-size 60)} :max-size (cf/get :database-max-pool-size 60)}
;; Default thread pool for IO operations ;; Default thread pool for IO operations
[::default :app.worker/executor] ::wrk/executor
{:parallelism (cf/get :default-executor-parallelism 70)} {::wrk/parallelism (cf/get :default-executor-parallelism 100)}
;; Dedicated thread pool for background tasks execution. ::wrk/scheduled-executor
[::worker :app.worker/executor] {::wrk/parallelism (cf/get :scheduled-executor-parallelism 20)}
{:parallelism (cf/get :worker-executor-parallelism 20)}
:app.worker/scheduled-executor ::wrk/monitor
{:parallelism 1} {::mtx/metrics (ig/ref ::mtx/metrics)
::wrk/name "default"
:app.worker/executors ::wrk/executor (ig/ref ::wrk/executor)}
{:default (ig/ref [::default :app.worker/executor])
:worker (ig/ref [::worker :app.worker/executor])}
:app.worker/executor-monitor
{:metrics (ig/ref :app.metrics/metrics)
:executors (ig/ref :app.worker/executors)}
:app.migrations/migrations :app.migrations/migrations
{} {}
:app.metrics/metrics ::mtx/metrics
{:default default-metrics} {:default default-metrics}
:app.migrations/all :app.migrations/all
{:main (ig/ref :app.migrations/migrations)} {:main (ig/ref :app.migrations/migrations)}
:app.redis/redis ::rds/redis
{:uri (cf/get :redis-uri) {::rds/uri (cf/get :redis-uri)
:metrics (ig/ref :app.metrics/metrics)} ::mtx/metrics (ig/ref ::mtx/metrics)}
:app.msgbus/msgbus :app.msgbus/msgbus
{:backend (cf/get :msgbus-backend :redis) {:backend (cf/get :msgbus-backend :redis)
:executor (ig/ref [::default :app.worker/executor]) :executor (ig/ref ::wrk/executor)
:redis (ig/ref :app.redis/redis)} :redis (ig/ref ::rds/redis)}
;; TODO: refactor execution model
:app.storage.tmp/cleaner :app.storage.tmp/cleaner
{:executor (ig/ref [::worker :app.worker/executor]) {:executor (ig/ref ::wrk/executor)
:scheduled-executor (ig/ref :app.worker/scheduled-executor)} :scheduled-executor (ig/ref ::wrk/scheduled-executor)}
:app.storage/gc-deleted-task ::sto/gc-deleted-task
{:pool (ig/ref :app.db/pool) {:pool (ig/ref ::db/pool)
:storage (ig/ref :app.storage/storage) :storage (ig/ref ::sto/storage)
:executor (ig/ref [::worker :app.worker/executor])} :executor (ig/ref ::wrk/executor)}
:app.storage/gc-touched-task ::sto/gc-touched-task
{:pool (ig/ref :app.db/pool)} {:pool (ig/ref ::db/pool)}
:app.http/client :app.http/client
{:executor (ig/ref [::default :app.worker/executor])} {:executor (ig/ref ::wrk/executor)}
:app.http.session/manager :app.http.session/manager
{:pool (ig/ref :app.db/pool) {:pool (ig/ref ::db/pool)
:sprops (ig/ref :app.setup/props) :sprops (ig/ref :app.setup/props)
:executor (ig/ref [::default :app.worker/executor])} :executor (ig/ref ::wrk/executor)}
:app.http.session/gc-task :app.http.session/gc-task
{:pool (ig/ref :app.db/pool) {:pool (ig/ref ::db/pool)
:max-age (cf/get :auth-token-cookie-max-age)} :max-age (cf/get :auth-token-cookie-max-age)}
:app.http.awsns/handler :app.http.awsns/handler
{:sprops (ig/ref :app.setup/props) {:sprops (ig/ref :app.setup/props)
:pool (ig/ref :app.db/pool) :pool (ig/ref ::db/pool)
:http-client (ig/ref :app.http/client) :http-client (ig/ref :app.http/client)
:executor (ig/ref [::worker :app.worker/executor])} :executor (ig/ref ::wrk/executor)}
:app.http/server :app.http/server
{:port (cf/get :http-server-port) {:port (cf/get :http-server-port)
:host (cf/get :http-server-host) :host (cf/get :http-server-host)
:router (ig/ref :app.http/router) :router (ig/ref :app.http/router)
:metrics (ig/ref :app.metrics/metrics) :metrics (ig/ref ::mtx/metrics)
:executor (ig/ref [::default :app.worker/executor]) :executor (ig/ref ::wrk/executor)
:io-threads (cf/get :http-server-io-threads) :io-threads (cf/get :http-server-io-threads)
:max-body-size (cf/get :http-server-max-body-size) :max-body-size (cf/get :http-server-max-body-size)
:max-multipart-body-size (cf/get :http-server-max-multipart-body-size)} :max-multipart-body-size (cf/get :http-server-max-multipart-body-size)}
@ -263,10 +262,10 @@
:oidc (ig/ref :app.auth.oidc/generic-provider)} :oidc (ig/ref :app.auth.oidc/generic-provider)}
:sprops (ig/ref :app.setup/props) :sprops (ig/ref :app.setup/props)
:http-client (ig/ref :app.http/client) :http-client (ig/ref :app.http/client)
:pool (ig/ref :app.db/pool) :pool (ig/ref ::db/pool)
:session (ig/ref :app.http.session/manager) :session (ig/ref :app.http.session/manager)
:public-uri (cf/get :public-uri) :public-uri (cf/get :public-uri)
:executor (ig/ref [::default :app.worker/executor])} :executor (ig/ref ::wrk/executor)}
;; TODO: revisit the dependencies of this service, looks they are too much unused of them ;; TODO: revisit the dependencies of this service, looks they are too much unused of them
:app.http/router :app.http/router
@ -277,61 +276,60 @@
:debug-routes (ig/ref :app.http.debug/routes) :debug-routes (ig/ref :app.http.debug/routes)
:oidc-routes (ig/ref :app.auth.oidc/routes) :oidc-routes (ig/ref :app.auth.oidc/routes)
:ws (ig/ref :app.http.websocket/handler) :ws (ig/ref :app.http.websocket/handler)
:metrics (ig/ref :app.metrics/metrics) :metrics (ig/ref ::mtx/metrics)
:public-uri (cf/get :public-uri) :public-uri (cf/get :public-uri)
:storage (ig/ref :app.storage/storage) :storage (ig/ref ::sto/storage)
:audit-handler (ig/ref :app.loggers.audit/http-handler) :audit-handler (ig/ref :app.loggers.audit/http-handler)
:rpc-routes (ig/ref :app.rpc/routes) :rpc-routes (ig/ref :app.rpc/routes)
:doc-routes (ig/ref :app.rpc.doc/routes) :doc-routes (ig/ref :app.rpc.doc/routes)
:executor (ig/ref [::default :app.worker/executor])} :executor (ig/ref ::wrk/executor)}
:app.http.debug/routes :app.http.debug/routes
{:pool (ig/ref :app.db/pool) {:pool (ig/ref ::db/pool)
:executor (ig/ref [::worker :app.worker/executor]) :executor (ig/ref ::wrk/executor)
:storage (ig/ref :app.storage/storage) :storage (ig/ref ::sto/storage)
:session (ig/ref :app.http.session/manager)} :session (ig/ref :app.http.session/manager)}
:app.http.websocket/handler :app.http.websocket/handler
{:pool (ig/ref :app.db/pool) {:pool (ig/ref ::db/pool)
:metrics (ig/ref :app.metrics/metrics) :metrics (ig/ref ::mtx/metrics)
:msgbus (ig/ref :app.msgbus/msgbus)} :msgbus (ig/ref :app.msgbus/msgbus)}
:app.http.assets/handlers :app.http.assets/handlers
{:metrics (ig/ref :app.metrics/metrics) {:metrics (ig/ref ::mtx/metrics)
:assets-path (cf/get :assets-path) :assets-path (cf/get :assets-path)
:storage (ig/ref :app.storage/storage) :storage (ig/ref ::sto/storage)
:executor (ig/ref [::default :app.worker/executor]) :executor (ig/ref ::wrk/executor)
:cache-max-age (dt/duration {:hours 24}) :cache-max-age (dt/duration {:hours 24})
:signature-max-age (dt/duration {:hours 24 :minutes 5})} :signature-max-age (dt/duration {:hours 24 :minutes 5})}
:app.http.feedback/handler :app.http.feedback/handler
{:pool (ig/ref :app.db/pool) {:pool (ig/ref ::db/pool)
:executor (ig/ref [::default :app.worker/executor])} :executor (ig/ref ::wrk/executor)}
:app.rpc/climit :app.rpc/climit
{:metrics (ig/ref :app.metrics/metrics) {:metrics (ig/ref ::mtx/metrics)
:executor (ig/ref [::default :app.worker/executor])} :executor (ig/ref ::wrk/executor)}
:app.rpc/rlimit :app.rpc/rlimit
{:executor (ig/ref [::worker :app.worker/executor]) {:executor (ig/ref ::wrk/executor)
:scheduled-executor (ig/ref :app.worker/scheduled-executor)} :scheduled-executor (ig/ref ::wrk/scheduled-executor)}
:app.rpc/methods :app.rpc/methods
{:pool (ig/ref :app.db/pool) {:pool (ig/ref ::db/pool)
:session (ig/ref :app.http.session/manager) :session (ig/ref :app.http.session/manager)
:sprops (ig/ref :app.setup/props) :sprops (ig/ref :app.setup/props)
:metrics (ig/ref :app.metrics/metrics) :metrics (ig/ref ::mtx/metrics)
:storage (ig/ref :app.storage/storage) :storage (ig/ref ::sto/storage)
:msgbus (ig/ref :app.msgbus/msgbus) :msgbus (ig/ref :app.msgbus/msgbus)
:public-uri (cf/get :public-uri) :public-uri (cf/get :public-uri)
:redis (ig/ref :app.redis/redis) :redis (ig/ref ::rds/redis)
:audit (ig/ref :app.loggers.audit/collector) :audit (ig/ref :app.loggers.audit/collector)
:ldap (ig/ref :app.auth.ldap/provider) :ldap (ig/ref :app.auth.ldap/provider)
:http-client (ig/ref :app.http/client) :http-client (ig/ref :app.http/client)
:climit (ig/ref :app.rpc/climit) :climit (ig/ref :app.rpc/climit)
:rlimit (ig/ref :app.rpc/rlimit) :rlimit (ig/ref :app.rpc/rlimit)
:executors (ig/ref :app.worker/executors) :executor (ig/ref ::wrk/executor)
:executor (ig/ref [::default :app.worker/executor])
:templates (ig/ref :app.setup/builtin-templates) :templates (ig/ref :app.setup/builtin-templates)
} }
@ -341,15 +339,15 @@
:app.rpc/routes :app.rpc/routes
{:methods (ig/ref :app.rpc/methods)} {:methods (ig/ref :app.rpc/methods)}
:app.worker/registry ::wrk/registry
{:metrics (ig/ref :app.metrics/metrics) {:metrics (ig/ref ::mtx/metrics)
:tasks :tasks
{:sendmail (ig/ref :app.emails/handler) {:sendmail (ig/ref :app.emails/handler)
:objects-gc (ig/ref :app.tasks.objects-gc/handler) :objects-gc (ig/ref :app.tasks.objects-gc/handler)
:file-gc (ig/ref :app.tasks.file-gc/handler) :file-gc (ig/ref :app.tasks.file-gc/handler)
:file-xlog-gc (ig/ref :app.tasks.file-xlog-gc/handler) :file-xlog-gc (ig/ref :app.tasks.file-xlog-gc/handler)
:storage-gc-deleted (ig/ref :app.storage/gc-deleted-task) :storage-gc-deleted (ig/ref ::sto/gc-deleted-task)
:storage-gc-touched (ig/ref :app.storage/gc-touched-task) :storage-gc-touched (ig/ref ::sto/gc-touched-task)
:tasks-gc (ig/ref :app.tasks.tasks-gc/handler) :tasks-gc (ig/ref :app.tasks.tasks-gc/handler)
:telemetry (ig/ref :app.tasks.telemetry/handler) :telemetry (ig/ref :app.tasks.telemetry/handler)
:session-gc (ig/ref :app.http.session/gc-task) :session-gc (ig/ref :app.http.session/gc-task)
@ -369,24 +367,24 @@
:app.emails/handler :app.emails/handler
{:sendmail (ig/ref :app.emails/sendmail) {:sendmail (ig/ref :app.emails/sendmail)
:metrics (ig/ref :app.metrics/metrics)} :metrics (ig/ref ::mtx/metrics)}
:app.tasks.tasks-gc/handler :app.tasks.tasks-gc/handler
{:pool (ig/ref :app.db/pool) {:pool (ig/ref ::db/pool)
:max-age cf/deletion-delay} :max-age cf/deletion-delay}
:app.tasks.objects-gc/handler :app.tasks.objects-gc/handler
{:pool (ig/ref :app.db/pool) {:pool (ig/ref ::db/pool)
:storage (ig/ref :app.storage/storage)} :storage (ig/ref ::sto/storage)}
:app.tasks.file-gc/handler :app.tasks.file-gc/handler
{:pool (ig/ref :app.db/pool)} {:pool (ig/ref ::db/pool)}
:app.tasks.file-xlog-gc/handler :app.tasks.file-xlog-gc/handler
{:pool (ig/ref :app.db/pool)} {:pool (ig/ref ::db/pool)}
:app.tasks.telemetry/handler :app.tasks.telemetry/handler
{:pool (ig/ref :app.db/pool) {:pool (ig/ref ::db/pool)
:version (:full cf/version) :version (:full cf/version)
:uri (cf/get :telemetry-uri) :uri (cf/get :telemetry-uri)
:sprops (ig/ref :app.setup/props) :sprops (ig/ref :app.setup/props)
@ -400,28 +398,28 @@
{:http-client (ig/ref :app.http/client)} {:http-client (ig/ref :app.http/client)}
:app.setup/props :app.setup/props
{:pool (ig/ref :app.db/pool) {:pool (ig/ref ::db/pool)
:key (cf/get :secret-key)} :key (cf/get :secret-key)}
:app.loggers.zmq/receiver :app.loggers.zmq/receiver
{:endpoint (cf/get :loggers-zmq-uri)} {:endpoint (cf/get :loggers-zmq-uri)}
:app.loggers.audit/http-handler :app.loggers.audit/http-handler
{:pool (ig/ref :app.db/pool) {:pool (ig/ref ::db/pool)
:executor (ig/ref [::default :app.worker/executor])} :executor (ig/ref ::wrk/executor)}
:app.loggers.audit/collector :app.loggers.audit/collector
{:pool (ig/ref :app.db/pool) {:pool (ig/ref ::db/pool)
:executor (ig/ref [::worker :app.worker/executor])} :executor (ig/ref ::wrk/executor)}
:app.loggers.audit/archive-task :app.loggers.audit/archive-task
{:uri (cf/get :audit-log-archive-uri) {:uri (cf/get :audit-log-archive-uri)
:sprops (ig/ref :app.setup/props) :sprops (ig/ref :app.setup/props)
:pool (ig/ref :app.db/pool) :pool (ig/ref ::db/pool)
:http-client (ig/ref :app.http/client)} :http-client (ig/ref :app.http/client)}
:app.loggers.audit/gc-task :app.loggers.audit/gc-task
{:pool (ig/ref :app.db/pool)} {:pool (ig/ref ::db/pool)}
:app.loggers.loki/reporter :app.loggers.loki/reporter
{:uri (cf/get :loggers-loki-uri) {:uri (cf/get :loggers-loki-uri)
@ -435,12 +433,12 @@
:app.loggers.database/reporter :app.loggers.database/reporter
{:receiver (ig/ref :app.loggers.zmq/receiver) {:receiver (ig/ref :app.loggers.zmq/receiver)
:pool (ig/ref :app.db/pool) :pool (ig/ref ::db/pool)
:executor (ig/ref [::worker :app.worker/executor])} :executor (ig/ref ::wrk/executor)}
:app.storage/storage ::sto/storage
{:pool (ig/ref :app.db/pool) {:pool (ig/ref ::db/pool)
:executor (ig/ref [::default :app.worker/executor]) :executor (ig/ref ::wrk/executor)
:backends :backends
{:assets-s3 (ig/ref [::assets :app.storage.s3/backend]) {:assets-s3 (ig/ref [::assets :app.storage.s3/backend])
@ -454,7 +452,7 @@
{:region (cf/get :storage-assets-s3-region) {:region (cf/get :storage-assets-s3-region)
:endpoint (cf/get :storage-assets-s3-endpoint) :endpoint (cf/get :storage-assets-s3-endpoint)
:bucket (cf/get :storage-assets-s3-bucket) :bucket (cf/get :storage-assets-s3-bucket)
:executor (ig/ref [::default :app.worker/executor])} :executor (ig/ref ::wrk/executor)}
[::assets :app.storage.fs/backend] [::assets :app.storage.fs/backend]
{:directory (cf/get :storage-assets-fs-directory)} {:directory (cf/get :storage-assets-fs-directory)}
@ -462,12 +460,11 @@
(def worker-config (def worker-config
{:app.worker/cron {::wrk/cron
{:executor (ig/ref [::worker :app.worker/executor]) {::wrk/scheduled-executor (ig/ref ::wrk/scheduled-executor)
:scheduled-executor (ig/ref :app.worker/scheduled-executor) ::wrk/registry (ig/ref ::wrk/registry)
:tasks (ig/ref :app.worker/registry) ::db/pool (ig/ref ::db/pool)
:pool (ig/ref :app.db/pool) ::wrk/entries
:entries
[{:cron #app/cron "0 0 * * * ?" ;; hourly [{:cron #app/cron "0 0 * * * ?" ;; hourly
:task :file-xlog-gc} :task :file-xlog-gc}
@ -500,11 +497,18 @@
{:cron #app/cron "30 */5 * * * ?" ;; every 5m {:cron #app/cron "30 */5 * * * ?" ;; every 5m
:task :audit-log-gc})]} :task :audit-log-gc})]}
:app.worker/worker ::wrk/scheduler
{:executor (ig/ref [::worker :app.worker/executor]) {::rds/redis (ig/ref ::rds/redis)
:tasks (ig/ref :app.worker/registry) ::mtx/metrics (ig/ref ::mtx/metrics)
:metrics (ig/ref :app.metrics/metrics) ::db/pool (ig/ref ::db/pool)}
:pool (ig/ref :app.db/pool)}})
::wrk/worker
{::wrk/parallelism (cf/get ::worker-parallelism 3)
::wrk/queue "default"
::rds/redis (ig/ref ::rds/redis)
::wrk/registry (ig/ref ::wrk/registry)
::mtx/metrics (ig/ref ::mtx/metrics)
::db/pool (ig/ref ::db/pool)}})
(def system nil) (def system nil)

4
backend/src/app/msgbus.clj
View file

@ -123,8 +123,8 @@
(defn- redis-disconnect (defn- redis-disconnect
[{:keys [::pconn ::sconn] :as cfg}] [{:keys [::pconn ::sconn] :as cfg}]
(redis/close! pconn) (d/close! pconn)
(redis/close! sconn)) (d/close! sconn))
(defn- conj-subscription (defn- conj-subscription
"A low level function that is responsible to create on-demand "A low level function that is responsible to create on-demand

176
backend/src/app/redis.clj
View file

@ -21,13 +21,19 @@
[promesa.core :as p]) [promesa.core :as p])
(:import (:import
clojure.lang.IDeref clojure.lang.IDeref
clojure.lang.MapEntry
io.lettuce.core.KeyValue
io.lettuce.core.RedisClient io.lettuce.core.RedisClient
io.lettuce.core.RedisCommandInterruptedException
io.lettuce.core.RedisCommandTimeoutException
io.lettuce.core.RedisException
io.lettuce.core.RedisURI io.lettuce.core.RedisURI
io.lettuce.core.ScriptOutputType io.lettuce.core.ScriptOutputType
io.lettuce.core.api.StatefulConnection io.lettuce.core.api.StatefulConnection
io.lettuce.core.api.StatefulRedisConnection io.lettuce.core.api.StatefulRedisConnection
io.lettuce.core.api.async.RedisAsyncCommands io.lettuce.core.api.async.RedisAsyncCommands
io.lettuce.core.api.async.RedisScriptingAsyncCommands io.lettuce.core.api.async.RedisScriptingAsyncCommands
io.lettuce.core.api.sync.RedisCommands
io.lettuce.core.codec.ByteArrayCodec io.lettuce.core.codec.ByteArrayCodec
io.lettuce.core.codec.RedisCodec io.lettuce.core.codec.RedisCodec
io.lettuce.core.codec.StringCodec io.lettuce.core.codec.StringCodec
@ -45,8 +51,7 @@
(declare initialize-resources) (declare initialize-resources)
(declare shutdown-resources) (declare shutdown-resources)
(declare connect) (declare connect*)
(declare close!)
(s/def ::timer (s/def ::timer
#(instance? Timer %)) #(instance? Timer %))
@ -82,32 +87,37 @@
(s/def ::connect? ::us/boolean) (s/def ::connect? ::us/boolean)
(s/def ::io-threads ::us/integer) (s/def ::io-threads ::us/integer)
(s/def ::worker-threads ::us/integer) (s/def ::worker-threads ::us/integer)
(s/def ::cache #(instance? clojure.lang.Atom %))
(s/def ::redis (s/def ::redis
(s/keys :req [::resources ::redis-uri ::timer ::mtx/metrics] (s/keys :req [::resources
:opt [::connection])) ::redis-uri
::timer
(defmethod ig/pre-init-spec ::redis [_] ::mtx/metrics]
(s/keys :req-un [::uri ::mtx/metrics] :opt [::connection
:opt-un [::timeout ::cache]))
::connect?
::io-threads
::worker-threads]))
(defmethod ig/prep-key ::redis (defmethod ig/prep-key ::redis
[_ cfg] [_ cfg]
(let [runtime (Runtime/getRuntime) (let [runtime (Runtime/getRuntime)
cpus (.availableProcessors ^Runtime runtime)] cpus (.availableProcessors ^Runtime runtime)]
(merge {:timeout (dt/duration 5000) (merge {::timeout (dt/duration "10s")
:io-threads (max 3 cpus) ::io-threads (max 3 cpus)
:worker-threads (max 3 cpus)} ::worker-threads (max 3 cpus)}
(d/without-nils cfg)))) (d/without-nils cfg))))
(defmethod ig/pre-init-spec ::redis [_]
(s/keys :req [::uri ::mtx/metrics]
:opt [::timeout
::connect?
::io-threads
::worker-threads]))
(defmethod ig/init-key ::redis (defmethod ig/init-key ::redis
[_ {:keys [connect?] :as cfg}] [_ {:keys [::connect?] :as cfg}]
(let [cfg (initialize-resources cfg)] (let [state (initialize-resources cfg)]
(cond-> cfg (cond-> state
connect? (assoc ::connection (connect cfg))))) connect? (assoc ::connection (connect* cfg {})))))
(defmethod ig/halt-key! ::redis (defmethod ig/halt-key! ::redis
[_ state] [_ state]
@ -121,7 +131,7 @@
(defn- initialize-resources (defn- initialize-resources
"Initialize redis connection resources" "Initialize redis connection resources"
[{:keys [uri io-threads worker-threads connect? metrics] :as cfg}] [{:keys [::uri ::io-threads ::worker-threads ::connect?] :as cfg}]
(l/info :hint "initialize redis resources" (l/info :hint "initialize redis resources"
:uri uri :uri uri
:io-threads io-threads :io-threads io-threads
@ -138,53 +148,60 @@
redis-uri (RedisURI/create ^String uri)] redis-uri (RedisURI/create ^String uri)]
(-> cfg (-> cfg
(assoc ::mtx/metrics metrics) (assoc ::resources resources)
(assoc ::cache (atom {}))
(assoc ::timer timer) (assoc ::timer timer)
(assoc ::redis-uri redis-uri) (assoc ::cache (atom {}))
(assoc ::resources resources)))) (assoc ::redis-uri redis-uri))))
(defn- shutdown-resources (defn- shutdown-resources
[{:keys [::resources ::cache ::timer]}] [{:keys [::resources ::cache ::timer]}]
(run! close! (vals @cache)) (run! d/close! (vals @cache))
(when resources (when resources
(.shutdown ^ClientResources resources)) (.shutdown ^ClientResources resources))
(when timer (when timer
(.stop ^Timer timer))) (.stop ^Timer timer)))
(defn connect (defn connect*
[{:keys [::resources ::redis-uri] :as state} [{:keys [::resources ::redis-uri] :as state}
& {:keys [timeout codec type] {:keys [timeout codec type]
:or {codec default-codec type :default}}] :or {codec default-codec type :default}}]
(us/assert! ::resources resources) (us/assert! ::resources resources)
(let [client (RedisClient/create ^ClientResources resources ^RedisURI redis-uri) (let [client (RedisClient/create ^ClientResources resources ^RedisURI redis-uri)
timeout (or timeout (:timeout state)) timeout (or timeout (::timeout state))
conn (case type conn (case type
:default (.connect ^RedisClient client ^RedisCodec codec) :default (.connect ^RedisClient client ^RedisCodec codec)
:pubsub (.connectPubSub ^RedisClient client ^RedisCodec codec))] :pubsub (.connectPubSub ^RedisClient client ^RedisCodec codec))]
(.setTimeout ^StatefulConnection conn ^Duration timeout) (.setTimeout ^StatefulConnection conn ^Duration timeout)
(assoc state ::connection (reify
(reify IDeref
IDeref (deref [_] conn)
(deref [_] conn)
AutoCloseable AutoCloseable
(close [_] (close [_]
(.close ^StatefulConnection conn) (.close ^StatefulConnection conn)
(.shutdown ^RedisClient client)))))) (.shutdown ^RedisClient client)))))
(defn connect
[state & {:as opts}]
(let [connection (connect* state opts)]
(-> state
(assoc ::connection connection)
(dissoc ::cache)
(vary-meta assoc `d/close! (fn [_] (d/close! connection))))))
(defn get-or-connect (defn get-or-connect
[{:keys [::cache] :as state} key options] [{:keys [::cache] :as state} key options]
(assoc state ::connection (-> state
(or (get @cache key) (assoc ::connection
(-> (swap! cache (fn [cache] (or (get @cache key)
(when-let [prev (get cache key)] (-> (swap! cache (fn [cache]
(close! prev)) (when-let [prev (get cache key)]
(assoc cache key (connect state options)))) (d/close! prev))
(get key))))) (assoc cache key (connect* state options))))
(get key))))
(dissoc ::cache)))
(defn add-listener! (defn add-listener!
[{:keys [::connection] :as conn} listener] [{:keys [::connection] :as conn} listener]
@ -210,18 +227,63 @@
[{:keys [::connection] :as conn} & topics] [{:keys [::connection] :as conn} & topics]
(us/assert! ::connection-holder conn) (us/assert! ::connection-holder conn)
(us/assert! ::pubsub-connection connection) (us/assert! ::pubsub-connection connection)
(let [topics (into-array String (map str topics)) (try
cmd (.sync ^StatefulRedisPubSubConnection @connection)] (let [topics (into-array String (map str topics))
(.subscribe ^RedisPubSubCommands cmd topics))) cmd (.sync ^StatefulRedisPubSubConnection @connection)]
(.subscribe ^RedisPubSubCommands cmd topics))
(catch RedisCommandInterruptedException cause
(throw (InterruptedException. (ex-message cause))))))
(defn unsubscribe! (defn unsubscribe!
"Blocking operation, intended to be used on a thread/agent thread." "Blocking operation, intended to be used on a thread/agent thread."
[{:keys [::connection] :as conn} & topics] [{:keys [::connection] :as conn} & topics]
(us/assert! ::connection-holder conn) (us/assert! ::connection-holder conn)
(us/assert! ::pubsub-connection connection) (us/assert! ::pubsub-connection connection)
(let [topics (into-array String (map str topics)) (try
cmd (.sync ^StatefulRedisPubSubConnection @connection)] (let [topics (into-array String (map str topics))
(.unsubscribe ^RedisPubSubCommands cmd topics))) cmd (.sync ^StatefulRedisPubSubConnection @connection)]
(.unsubscribe ^RedisPubSubCommands cmd topics))
(catch RedisCommandInterruptedException cause
(throw (InterruptedException. (ex-message cause))))))
(defn rpush!
[{:keys [::connection] :as conn} key payload]
(us/assert! ::connection-holder conn)
(us/assert! (or (and (vector? payload)
(every? bytes? payload))
(bytes? payload)))
(try
(let [cmd (.sync ^StatefulRedisConnection @connection)
data (if (vector? payload) payload [payload])
vals (make-array (. Class (forName "[B")) (count data))]
(loop [i 0 xs (seq data)]
(when xs
(aset ^"[[B" vals i ^bytes (first xs))
(recur (inc i) (next xs))))
(.rpush ^RedisCommands cmd
^String key
^"[[B" vals))
(catch RedisCommandInterruptedException cause
(throw (InterruptedException. (ex-message cause))))))
(defn blpop!
[{:keys [::connection] :as conn} timeout & keys]
(us/assert! ::connection-holder conn)
(try
(let [keys (into-array Object (map str keys))
cmd (.sync ^StatefulRedisConnection @connection)
timeout (/ (double (inst-ms timeout)) 1000.0)]
(when-let [res (.blpop ^RedisCommands cmd
^double timeout
^"[Ljava.lang.String;" keys)]
(MapEntry/create
(.getKey ^KeyValue res)
(.getValue ^KeyValue res))))
(catch RedisCommandInterruptedException cause
(throw (InterruptedException. (ex-message cause))))))
(defn open? (defn open?
[{:keys [::connection] :as conn}] [{:keys [::connection] :as conn}]
@ -256,12 +318,6 @@
(when on-unsubscribe (when on-unsubscribe
(on-unsubscribe nil topic count))))) (on-unsubscribe nil topic count)))))
(defn close!
[{:keys [::connection] :as conn}]
(us/assert! ::connection-holder conn)
(us/assert! ::connection connection)
(.close ^AutoCloseable connection))
(def ^:private scripts-cache (atom {})) (def ^:private scripts-cache (atom {}))
(def noop-fn (constantly nil)) (def noop-fn (constantly nil))
@ -332,3 +388,11 @@
(eval-script sha) (eval-script sha)
(->> (load-script) (->> (load-script)
(p/mapcat eval-script)))))) (p/mapcat eval-script))))))
(defn timeout-exception?
[cause]
(instance? RedisCommandTimeoutException cause))
(defn exception?
[cause]
(instance? RedisException cause))

2
backend/src/app/rpc.clj
View file

@ -23,6 +23,7 @@
[app.storage :as-alias sto] [app.storage :as-alias sto]
[app.util.services :as sv] [app.util.services :as sv]
[app.util.time :as ts] [app.util.time :as ts]
[app.worker :as-alias wrk]
[clojure.spec.alpha :as s] [clojure.spec.alpha :as s]
[integrant.core :as ig] [integrant.core :as ig]
[promesa.core :as p] [promesa.core :as p]
@ -270,6 +271,7 @@
::http-client ::http-client
::rlimit ::rlimit
::climit ::climit
::wrk/executor
::mtx/metrics ::mtx/metrics
::db/pool ::db/pool
::ldap])) ::ldap]))

11
backend/src/app/srepl/main.clj
View file

@ -20,6 +20,7 @@
[app.util.objects-map :as omap] [app.util.objects-map :as omap]
[app.util.pointer-map :as pmap] [app.util.pointer-map :as pmap]
[app.util.time :as dt] [app.util.time :as dt]
[app.worker :as wrk]
[clojure.pprint :refer [pprint]] [clojure.pprint :refer [pprint]]
[cuerdas.core :as str])) [cuerdas.core :as str]))
@ -37,6 +38,16 @@
(task-fn params) (task-fn params)
(println (format "no task '%s' found" name)))))) (println (format "no task '%s' found" name))))))
(defn schedule-task!
([system name]
(schedule-task! system name {}))
([system name props]
(let [pool (:app.db/pool system)]
(wrk/submit!
::wrk/conn pool
::wrk/task name
::wrk/props props))))
(defn send-test-email! (defn send-test-email!
[system destination] [system destination]
(us/verify! (us/verify!

31
backend/src/app/util/closeable.clj
View file

@ -1,31 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC
(ns app.util.closeable
"A closeable abstraction. A drop in replacement for
clojure builtin `with-open` syntax abstraction."
(:refer-clojure :exclude [with-open]))
(defprotocol ICloseable
(-close [_] "Close the resource."))
(defmacro with-open
[bindings & body]
{:pre [(vector? bindings)
(even? (count bindings))
(pos? (count bindings))]}
(reduce (fn [acc bindings]
`(let ~(vec bindings)
(try
~acc
(finally
(-close ~(first bindings))))))
`(do ~@body)
(reverse (partition 2 bindings))))
(extend-protocol ICloseable
java.lang.AutoCloseable
(-close [this] (.close this)))

951
backend/src/app/worker.clj
View file

File diff suppressed because it is too large Load diff

2
backend/test/backend_tests/helpers.clj
View file

@ -324,7 +324,7 @@
(run-task! name {})) (run-task! name {}))
([name params] ([name params]
(let [tasks (:app.worker/registry *system*)] (let [tasks (:app.worker/registry *system*)]
(let [task-fn (get tasks name)] (let [task-fn (get tasks (d/name name))]
(task-fn params))))) (task-fn params)))))
;; --- UTILS ;; --- UTILS

8
backend/test/backend_tests/tasks_telemetry_test.clj
View file

@ -20,12 +20,10 @@
(t/deftest test-base-report-data-structure (t/deftest test-base-report-data-structure
(with-mocks [mock {:target 'app.tasks.telemetry/send! (with-mocks [mock {:target 'app.tasks.telemetry/send!
:return nil}] :return nil}]
(let [task-fn (-> th/*system* :app.worker/registry :telemetry) (let [prof (th/create-profile* 1 {:is-active true
prof (th/create-profile* 1 {:is-active true :props {:newsletter-news true}})]
:props {:newsletter-news true}})]
;; run the task (th/run-task! :telemetry {:send? true :enabled? true})
(task-fn {:send? true :enabled? true})
(t/is (:called? @mock)) (t/is (:called? @mock))
(let [[_ data] (-> @mock :call-args)] (let [[_ data] (-> @mock :call-args)]

2
common/deps.edn
View file

@ -23,7 +23,7 @@
com.cognitect/transit-cljs {:mvn/version "0.8.280"} com.cognitect/transit-cljs {:mvn/version "0.8.280"}
java-http-clj/java-http-clj {:mvn/version "0.4.3"} java-http-clj/java-http-clj {:mvn/version "0.4.3"}
funcool/promesa {:mvn/version "9.1.540"} funcool/promesa {:mvn/version "9.2.541"}
funcool/cuerdas {:mvn/version "2022.06.16-403"} funcool/cuerdas {:mvn/version "2022.06.16-403"}
lambdaisland/uri {:mvn/version "1.13.95" lambdaisland/uri {:mvn/version "1.13.95"

20
common/src/app/common/data.cljc
View file

@ -5,7 +5,8 @@
;; Copyright (c) KALEIDOS INC ;; Copyright (c) KALEIDOS INC
(ns app.common.data (ns app.common.data
"Data manipulation and query helper functions." "A collection if helpers for working with data structures and other
data resources."
(:refer-clojure :exclude [read-string hash-map merge name update-vals (:refer-clojure :exclude [read-string hash-map merge name update-vals
parse-double group-by iteration concat mapcat]) parse-double group-by iteration concat mapcat])
#?(:cljs #?(:cljs
@ -22,7 +23,9 @@
[linked.set :as lks]) [linked.set :as lks])
#?(:clj #?(:clj
(:import linked.set.LinkedSet))) (:import
linked.set.LinkedSet
java.lang.AutoCloseable)))
(def boolean-or-nil? (def boolean-or-nil?
(some-fn nil? boolean?)) (some-fn nil? boolean?))
@ -697,3 +700,16 @@
(map (fn [key] (map (fn [key]
[key (delay (generator-fn key))])) [key (delay (generator-fn key))]))
keys)) keys))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Util protocols
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defprotocol ICloseable
:extend-via-metadata true
(close! [_] "Close the resource."))
#?(:clj
(extend-protocol ICloseable
AutoCloseable
(close! [this] (.close this))))

17
common/src/app/common/data/macros.cljc
View file

@ -7,7 +7,7 @@
#_:clj-kondo/ignore #_:clj-kondo/ignore
(ns app.common.data.macros (ns app.common.data.macros
"Data retrieval & manipulation specific macros." "Data retrieval & manipulation specific macros."
(:refer-clojure :exclude [get-in select-keys str]) (:refer-clojure :exclude [get-in select-keys str with-open])
#?(:cljs (:require-macros [app.common.data.macros])) #?(:cljs (:require-macros [app.common.data.macros]))
(:require (:require
#?(:clj [clojure.core :as c] #?(:clj [clojure.core :as c]
@ -94,5 +94,16 @@
[s & params] [s & params]
`(str/ffmt ~s ~@params)) `(str/ffmt ~s ~@params))
(defmacro with-open
[bindings & body]
{:pre [(vector? bindings)
(even? (count bindings))
(pos? (count bindings))]}
(reduce (fn [acc bindings]
`(let ~(vec bindings)
(try
~acc
(finally
(d/close! ~(first bindings))))))
`(do ~@body)
(reverse (partition 2 bindings))))

4
common/src/app/common/exceptions.cljc
View file

@ -50,6 +50,10 @@
[& exprs] [& exprs]
`(try* (^:once fn* [] ~@exprs) identity)) `(try* (^:once fn* [] ~@exprs) identity))
(defmacro try!
[& exprs]
`(try* (^:once fn* [] ~@exprs) identity))
(defn with-always (defn with-always
"A helper that evaluates an exptession independently if the body "A helper that evaluates an exptession independently if the body
raises exception or not." raises exception or not."