mirror of
https://github.com/penpot/penpot.git
synced 2025-06-07 23:51:38 +02:00
♻️ Refactor general resource and concurrency model on backend
This commit is contained in:
Andrey Antukh
Alonso Torres
parent
d24f16563f
commit
7cf27ac86d
32 changed files with 917 additions and 797 deletions
|
|
@ -13,79 +13,164 @@
|
|||
[app.db :as db]
|
||||
[app.loggers.audit :as audit]
|
||||
[app.metrics :as mtx]
|
||||
[app.util.retry :as retry]
|
||||
[app.util.rlimit :as rlimit]
|
||||
[app.rpc.retry :as retry]
|
||||
[app.rpc.rlimit :as rlimit]
|
||||
[app.util.async :as async]
|
||||
[app.util.services :as sv]
|
||||
[app.worker :as wrk]
|
||||
[clojure.spec.alpha :as s]
|
||||
[integrant.core :as ig]))
|
||||
[integrant.core :as ig]
|
||||
[promesa.core :as p]
|
||||
[promesa.exec :as px]))
|
||||
|
||||
(defn- default-handler
|
||||
[_]
|
||||
(ex/raise :type :not-found))
|
||||
(p/rejected (ex/error :type :not-found)))
|
||||
|
||||
(defn- run-hook
|
||||
[hook-fn response]
|
||||
(ex/ignoring (hook-fn))
|
||||
(defn- handle-response-transformation
|
||||
[response request mdata]
|
||||
(if-let [transform-fn (:transform-response mdata)]
|
||||
(transform-fn request response)
|
||||
response))
|
||||
|
||||
(defn- handle-before-comple-hook
|
||||
[response mdata]
|
||||
(when-let [hook-fn (:before-complete mdata)]
|
||||
(ex/ignoring (hook-fn)))
|
||||
response)
|
||||
|
||||
(defn- rpc-query-handler
|
||||
[methods {:keys [profile-id session-id] :as request}]
|
||||
(let [type (keyword (get-in request [:path-params :type]))
|
||||
"Ring handler that dispatches query requests and convert between
|
||||
internal async flow into ring async flow."
|
||||
[methods {:keys [profile-id session-id] :as request} respond raise]
|
||||
(letfn [(handle-response [result]
|
||||
(let [mdata (meta result)]
|
||||
(-> {:status 200 :body result}
|
||||
(handle-response-transformation request mdata))))]
|
||||
|
||||
data (merge (:params request)
|
||||
(:body-params request)
|
||||
(:uploads request)
|
||||
{::request request})
|
||||
(let [type (keyword (get-in request [:path-params :type]))
|
||||
data (merge (:params request)
|
||||
(:body-params request)
|
||||
(:uploads request)
|
||||
{::request request})
|
||||
|
||||
data (if profile-id
|
||||
(assoc data :profile-id profile-id ::session-id session-id)
|
||||
(dissoc data :profile-id))
|
||||
data (if profile-id
|
||||
(assoc data :profile-id profile-id ::session-id session-id)
|
||||
(dissoc data :profile-id))
|
||||
|
||||
result ((get methods type default-handler) data)
|
||||
mdata (meta result)]
|
||||
;; Get the method from methods registry and if method does
|
||||
;; not exists asigns it to the default handler.
|
||||
method (get methods type default-handler)]
|
||||
|
||||
(cond->> {:status 200 :body result}
|
||||
(fn? (:transform-response mdata))
|
||||
((:transform-response mdata) request))))
|
||||
(-> (method data)
|
||||
(p/then #(respond (handle-response %)))
|
||||
(p/catch raise)))))
|
||||
|
||||
(defn- rpc-mutation-handler
|
||||
[methods {:keys [profile-id session-id] :as request}]
|
||||
(let [type (keyword (get-in request [:path-params :type]))
|
||||
data (merge (:params request)
|
||||
(:body-params request)
|
||||
(:uploads request)
|
||||
{::request request})
|
||||
"Ring handler that dispatches mutation requests and convert between
|
||||
internal async flow into ring async flow."
|
||||
[methods {:keys [profile-id session-id] :as request} respond raise]
|
||||
(letfn [(handle-response [result]
|
||||
(let [mdata (meta result)]
|
||||
(-> {:status 200 :body result}
|
||||
(handle-response-transformation request mdata)
|
||||
(handle-before-comple-hook mdata))))]
|
||||
|
||||
data (if profile-id
|
||||
(assoc data :profile-id profile-id ::session-id session-id)
|
||||
(dissoc data :profile-id))
|
||||
(let [type (keyword (get-in request [:path-params :type]))
|
||||
data (merge (:params request)
|
||||
(:body-params request)
|
||||
(:uploads request)
|
||||
{::request request})
|
||||
|
||||
result ((get methods type default-handler) data)
|
||||
mdata (meta result)]
|
||||
(cond->> {:status 200 :body result}
|
||||
(fn? (:transform-response mdata))
|
||||
((:transform-response mdata) request)
|
||||
data (if profile-id
|
||||
(assoc data :profile-id profile-id ::session-id session-id)
|
||||
(dissoc data :profile-id))
|
||||
|
||||
(fn? (:before-complete mdata))
|
||||
(run-hook (:before-complete mdata)))))
|
||||
method (get methods type default-handler)]
|
||||
|
||||
(defn- wrap-with-metrics
|
||||
[cfg f mdata]
|
||||
(mtx/wrap-summary f (::mobj cfg) [(::sv/name mdata)]))
|
||||
(-> (method data)
|
||||
(p/then #(respond (handle-response %)))
|
||||
(p/catch raise)))))
|
||||
|
||||
(defn- wrap-impl
|
||||
(defn- wrap-metrics
|
||||
"Wrap service method with metrics measurement."
|
||||
[{:keys [metrics ::metrics-id]} f mdata]
|
||||
(let [labels (into-array String [(::sv/name mdata)])]
|
||||
(fn [cfg params]
|
||||
(let [start (System/nanoTime)]
|
||||
(p/finally
|
||||
(f cfg params)
|
||||
(fn [_ _]
|
||||
(mtx/run! metrics
|
||||
{:id metrics-id
|
||||
:val (/ (- (System/nanoTime) start) 1000000)
|
||||
:labels labels})))))))
|
||||
|
||||
(defn- wrap-dispatch
|
||||
"Wraps service method into async flow, with the ability to dispatching
|
||||
it to a preconfigured executor service."
|
||||
[{:keys [executors] :as cfg} f mdata]
|
||||
(let [dname (::async/dispatch mdata :none)]
|
||||
(if (= :none dname)
|
||||
(with-meta
|
||||
(fn [cfg params]
|
||||
(try
|
||||
(p/wrap (f cfg params))
|
||||
(catch Throwable cause
|
||||
(p/rejected cause))))
|
||||
mdata)
|
||||
|
||||
(let [executor (get executors dname)]
|
||||
(when-not executor
|
||||
(ex/raise :type :internal
|
||||
:code :executor-not-configured
|
||||
:hint (format "executor %s not configured" dname)))
|
||||
(with-meta
|
||||
(fn [cfg params]
|
||||
(-> (px/submit! executor #(f cfg params))
|
||||
(p/bind p/wrap)))
|
||||
mdata)))))
|
||||
|
||||
(defn- wrap-audit
|
||||
[{:keys [audit] :as cfg} f mdata]
|
||||
(if audit
|
||||
(with-meta
|
||||
(fn [cfg {:keys [::request] :as params}]
|
||||
(p/finally (f cfg params)
|
||||
(fn [result _]
|
||||
(when result
|
||||
(let [resultm (meta result)
|
||||
profile-id (or (:profile-id params)
|
||||
(:profile-id result)
|
||||
(::audit/profile-id resultm))
|
||||
props (d/merge params (::audit/props resultm))]
|
||||
(audit :cmd :submit
|
||||
:type (or (::audit/type resultm)
|
||||
(::type cfg))
|
||||
:name (or (::audit/name resultm)
|
||||
(::sv/name mdata))
|
||||
:profile-id profile-id
|
||||
:ip-addr (audit/parse-client-ip request)
|
||||
:props (dissoc props ::request)))))))
|
||||
mdata)
|
||||
f))
|
||||
|
||||
(defn- wrap
|
||||
[cfg f mdata]
|
||||
(let [f (as-> f $
|
||||
(wrap-dispatch cfg $ mdata)
|
||||
(rlimit/wrap-rlimit cfg $ mdata)
|
||||
(retry/wrap-retry cfg $ mdata)
|
||||
(wrap-with-metrics cfg $ mdata))
|
||||
(wrap-audit cfg $ mdata)
|
||||
(wrap-metrics cfg $ mdata)
|
||||
)
|
||||
|
||||
spec (or (::sv/spec mdata) (s/spec any?))
|
||||
auth? (:auth mdata true)]
|
||||
|
||||
(l/trace :action "register" :name (::sv/name mdata))
|
||||
(with-meta
|
||||
(fn [params]
|
||||
(fn [{:keys [::request] :as params}]
|
||||
;; Raise authentication error when rpc method requires auth but
|
||||
;; no profile-id is found in the request.
|
||||
(when (and auth? (not (uuid? (:profile-id params))))
|
||||
|
|
@ -93,44 +178,19 @@
|
|||
:code :authentication-required
|
||||
:hint "authentication required for this endpoint"))
|
||||
|
||||
(let [params' (dissoc params ::request)
|
||||
params' (us/conform spec params')
|
||||
result (f cfg params')]
|
||||
|
||||
;; When audit log is enabled (default false).
|
||||
(when (fn? audit)
|
||||
(let [resultm (meta result)
|
||||
request (::request params)
|
||||
profile-id (or (:profile-id params')
|
||||
(:profile-id result)
|
||||
(::audit/profile-id resultm))
|
||||
props (d/merge params' (::audit/props resultm))]
|
||||
(audit :cmd :submit
|
||||
:type (or (::audit/type resultm)
|
||||
(::type cfg))
|
||||
:name (or (::audit/name resultm)
|
||||
(::sv/name mdata))
|
||||
:profile-id profile-id
|
||||
:ip-addr (audit/parse-client-ip request)
|
||||
:props props)))
|
||||
result))
|
||||
(let [params (us/conform spec (dissoc params ::request))]
|
||||
(f cfg (assoc params ::request request))))
|
||||
mdata)))
|
||||
|
||||
(defn- process-method
|
||||
[cfg vfn]
|
||||
(let [mdata (meta vfn)]
|
||||
[(keyword (::sv/name mdata))
|
||||
(wrap-impl cfg (deref vfn) mdata)]))
|
||||
(wrap cfg (deref vfn) mdata)]))
|
||||
|
||||
(defn- resolve-query-methods
|
||||
[cfg]
|
||||
(let [mobj (mtx/create
|
||||
{:name "rpc_query_timing"
|
||||
:labels ["name"]
|
||||
:registry (get-in cfg [:metrics :registry])
|
||||
:type :histogram
|
||||
:help "Timing of query services."})
|
||||
cfg (assoc cfg ::mobj mobj ::type "query")]
|
||||
(let [cfg (assoc cfg ::type "query" ::metrics-id :rpc-query-timing)]
|
||||
(->> (sv/scan-ns 'app.rpc.queries.projects
|
||||
'app.rpc.queries.files
|
||||
'app.rpc.queries.teams
|
||||
|
|
@ -143,13 +203,7 @@
|
|||
|
||||
(defn- resolve-mutation-methods
|
||||
[cfg]
|
||||
(let [mobj (mtx/create
|
||||
{:name "rpc_mutation_timing"
|
||||
:labels ["name"]
|
||||
:registry (get-in cfg [:metrics :registry])
|
||||
:type :histogram
|
||||
:help "Timing of mutation services."})
|
||||
cfg (assoc cfg ::mobj mobj ::type "mutation")]
|
||||
(let [cfg (assoc cfg ::type "mutation" ::metrics-id :rpc-mutation-timing)]
|
||||
(->> (sv/scan-ns 'app.rpc.mutations.demo
|
||||
'app.rpc.mutations.media
|
||||
'app.rpc.mutations.profile
|
||||
|
|
@ -169,15 +223,16 @@
|
|||
(s/def ::session map?)
|
||||
(s/def ::tokens fn?)
|
||||
(s/def ::audit (s/nilable fn?))
|
||||
(s/def ::executors (s/map-of keyword? ::wrk/executor))
|
||||
|
||||
(defmethod ig/pre-init-spec ::rpc [_]
|
||||
(s/keys :req-un [::storage ::session ::tokens ::audit
|
||||
::mtx/metrics ::db/pool]))
|
||||
::executors ::mtx/metrics ::db/pool]))
|
||||
|
||||
(defmethod ig/init-key ::rpc
|
||||
[_ cfg]
|
||||
(let [mq (resolve-query-methods cfg)
|
||||
mm (resolve-mutation-methods cfg)]
|
||||
{:methods {:query mq :mutation mm}
|
||||
:query-handler #(rpc-query-handler mq %)
|
||||
:mutation-handler #(rpc-mutation-handler mm %)}))
|
||||
:query-handler (partial rpc-query-handler mq)
|
||||
:mutation-handler (partial rpc-mutation-handler mm)}))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue