mirror of
https://github.com/penpot/penpot.git
synced 2025-05-18 20:46:11 +02:00
♻️ Refactor general resource and concurrency model on backend
This commit is contained in:
Andrey Antukh
Alonso Torres
parent
d24f16563f
commit
7cf27ac86d
32 changed files with 917 additions and 797 deletions
|
|
@ -5,46 +5,40 @@
|
|||
;; Copyright (c) UXBOX Labs SL
|
||||
|
||||
(ns app.metrics
|
||||
(:refer-clojure :exclude [run!])
|
||||
(:require
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[clojure.spec.alpha :as s]
|
||||
[integrant.core :as ig])
|
||||
(:import
|
||||
io.prometheus.client.CollectorRegistry
|
||||
io.prometheus.client.Counter
|
||||
io.prometheus.client.Counter$Child
|
||||
io.prometheus.client.Gauge
|
||||
io.prometheus.client.Gauge$Child
|
||||
io.prometheus.client.Summary
|
||||
io.prometheus.client.Summary$Child
|
||||
io.prometheus.client.Summary$Builder
|
||||
io.prometheus.client.Histogram
|
||||
io.prometheus.client.Histogram$Child
|
||||
io.prometheus.client.exporter.common.TextFormat
|
||||
io.prometheus.client.hotspot.DefaultExports
|
||||
io.prometheus.client.jetty.JettyStatisticsCollector
|
||||
org.eclipse.jetty.server.handler.StatisticsHandler
|
||||
java.io.StringWriter))
|
||||
|
||||
(declare instrument-vars!)
|
||||
(declare instrument)
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(declare create-registry)
|
||||
(declare create)
|
||||
(declare handler)
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; Defaults
|
||||
;; METRICS SERVICE PROVIDER
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
|
||||
(def default-metrics
|
||||
{:profile-register
|
||||
{:name "actions_profile_register_count"
|
||||
:help "A global counter of user registrations."
|
||||
:type :counter}
|
||||
|
||||
:profile-activation
|
||||
{:name "actions_profile_activation_count"
|
||||
:help "A global counter of profile activations"
|
||||
:type :counter}
|
||||
|
||||
:update-file-changes
|
||||
{:update-file-changes
|
||||
{:name "rpc_update_file_changes_total"
|
||||
:help "A total number of changes submitted to update-file."
|
||||
:type :counter}
|
||||
|
|
@ -54,6 +48,18 @@
|
|||
:help "A total number of bytes processed by update-file."
|
||||
:type :counter}
|
||||
|
||||
:rpc-mutation-timing
|
||||
{:name "rpc_mutation_timing"
|
||||
:help "RPC mutation method call timming."
|
||||
:labels ["name"]
|
||||
:type :histogram}
|
||||
|
||||
:rpc-query-timing
|
||||
{:name "rpc_query_timing"
|
||||
:help "RPC query method call timing."
|
||||
:labels ["name"]
|
||||
:type :histogram}
|
||||
|
||||
:websocket-active-connections
|
||||
{:name "websocket_active_connections"
|
||||
:help "Active websocket connections gauge"
|
||||
|
|
@ -68,12 +74,54 @@
|
|||
:websocket-session-timing
|
||||
{:name "websocket_session_timing"
|
||||
:help "Websocket session timing (seconds)."
|
||||
:quantiles []
|
||||
:type :summary}})
|
||||
:type :summary}
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; Entry Point
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
:session-update-total
|
||||
{:name "http_session_update_total"
|
||||
:help "A counter of session update batch events."
|
||||
:type :counter}
|
||||
|
||||
:tasks-timing
|
||||
{:name "penpot_tasks_timing"
|
||||
:help "Background tasks timing (milliseconds)."
|
||||
:labels ["name"]
|
||||
:type :summary}
|
||||
|
||||
:rlimit-queued-submissions
|
||||
{:name "penpot_rlimit_queued_submissions"
|
||||
:help "Current number of queued submissions on RLIMIT."
|
||||
:labels ["name"]
|
||||
:type :gauge}
|
||||
|
||||
:rlimit-used-permits
|
||||
{:name "penpot_rlimit_used_permits"
|
||||
:help "Current number of used permits on RLIMIT."
|
||||
:labels ["name"]
|
||||
:type :gauge}
|
||||
|
||||
:rlimit-acquires-total
|
||||
{:name "penpot_rlimit_acquires_total"
|
||||
:help "Total number of acquire operations on RLIMIT."
|
||||
:labels ["name"]
|
||||
:type :counter}
|
||||
|
||||
:executors-active-threads
|
||||
{:name "penpot_executors_active_threads"
|
||||
:help "Current number of threads available in the executor service."
|
||||
:labels ["name"]
|
||||
:type :gauge}
|
||||
|
||||
:executors-running-threads
|
||||
{:name "penpot_executors_running_threads"
|
||||
:help "Current number of threads with state RUNNING."
|
||||
:labels ["name"]
|
||||
:type :gauge}
|
||||
|
||||
:executors-queued-submissions
|
||||
{:name "penpot_executors_queued_submissions"
|
||||
:help "Current number of queued submissions."
|
||||
:labels ["name"]
|
||||
:type :gauge}})
|
||||
|
||||
(defmethod ig/init-key ::metrics
|
||||
[_ _]
|
||||
|
|
@ -95,31 +143,44 @@
|
|||
(s/keys :req-un [::registry ::handler]))
|
||||
|
||||
(defn- handler
|
||||
[registry _request]
|
||||
[registry _ respond _]
|
||||
(let [samples (.metricFamilySamples ^CollectorRegistry registry)
|
||||
writer (StringWriter.)]
|
||||
(TextFormat/write004 writer samples)
|
||||
{:headers {"content-type" TextFormat/CONTENT_TYPE_004}
|
||||
:body (.toString writer)}))
|
||||
(respond {:headers {"content-type" TextFormat/CONTENT_TYPE_004}
|
||||
:body (.toString writer)})))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; Implementation
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(def default-empty-labels (into-array String []))
|
||||
|
||||
(def default-quantiles
|
||||
[[0.5 0.01]
|
||||
[0.90 0.01]
|
||||
[0.99 0.001]])
|
||||
|
||||
(def default-histogram-buckets
|
||||
[1 5 10 25 50 75 100 250 500 750 1000 2500 5000 7500])
|
||||
|
||||
(defn run!
|
||||
[{:keys [definitions]} {:keys [id] :as params}]
|
||||
(when-let [mobj (get definitions id)]
|
||||
((::fn mobj) params)
|
||||
true))
|
||||
|
||||
(defn create-registry
|
||||
[]
|
||||
(let [registry (CollectorRegistry.)]
|
||||
(DefaultExports/register registry)
|
||||
registry))
|
||||
|
||||
(defmacro with-measure
|
||||
[& {:keys [expr cb]}]
|
||||
`(let [start# (System/nanoTime)
|
||||
tdown# ~cb]
|
||||
(try
|
||||
~expr
|
||||
(finally
|
||||
(tdown# (/ (- (System/nanoTime) start#) 1000000))))))
|
||||
(defn- is-array?
|
||||
[o]
|
||||
(let [oc (class o)]
|
||||
(and (.isArray ^Class oc)
|
||||
(= (.getComponentType oc) String))))
|
||||
|
||||
(defn make-counter
|
||||
[{:keys [name help registry reg labels] :as props}]
|
||||
|
|
@ -132,12 +193,9 @@
|
|||
instance (.register instance registry)]
|
||||
|
||||
{::instance instance
|
||||
::fn (fn [{:keys [by labels] :or {by 1}}]
|
||||
(if labels
|
||||
(.. ^Counter instance
|
||||
(labels (into-array String labels))
|
||||
(inc by))
|
||||
(.inc ^Counter instance by)))}))
|
||||
::fn (fn [{:keys [inc labels] :or {inc 1 labels default-empty-labels}}]
|
||||
(let [instance (.labels instance (if (is-array? labels) labels (into-array String labels)))]
|
||||
(.inc ^Counter$Child instance (double inc))))}))
|
||||
|
||||
(defn make-gauge
|
||||
[{:keys [name help registry reg labels] :as props}]
|
||||
|
|
@ -148,48 +206,33 @@
|
|||
_ (when (seq labels)
|
||||
(.labelNames instance (into-array String labels)))
|
||||
instance (.register instance registry)]
|
||||
|
||||
{::instance instance
|
||||
::fn (fn [{:keys [cmd by labels] :or {by 1}}]
|
||||
(if labels
|
||||
(let [labels (into-array String [labels])]
|
||||
(case cmd
|
||||
:inc (.. ^Gauge instance (labels labels) (inc by))
|
||||
:dec (.. ^Gauge instance (labels labels) (dec by))))
|
||||
(case cmd
|
||||
:inc (.inc ^Gauge instance by)
|
||||
:dec (.dec ^Gauge instance by))))}))
|
||||
|
||||
(def default-quantiles
|
||||
[[0.75 0.02]
|
||||
[0.99 0.001]])
|
||||
::fn (fn [{:keys [inc dec labels val] :or {labels default-empty-labels}}]
|
||||
(let [instance (.labels ^Gauge instance (if (is-array? labels) labels (into-array String labels)))]
|
||||
(cond (number? inc) (.inc ^Gauge$Child instance (double inc))
|
||||
(number? dec) (.dec ^Gauge$Child instance (double dec))
|
||||
(number? val) (.set ^Gauge$Child instance (double val)))))}))
|
||||
|
||||
(defn make-summary
|
||||
[{:keys [name help registry reg labels max-age quantiles buckets]
|
||||
:or {max-age 3600 buckets 6 quantiles default-quantiles} :as props}]
|
||||
:or {max-age 3600 buckets 12 quantiles default-quantiles} :as props}]
|
||||
(let [registry (or registry reg)
|
||||
instance (doto (Summary/build)
|
||||
builder (doto (Summary/build)
|
||||
(.name name)
|
||||
(.help help))
|
||||
_ (when (seq quantiles)
|
||||
(.maxAgeSeconds ^Summary instance max-age)
|
||||
(.ageBuckets ^Summary instance buckets))
|
||||
(.maxAgeSeconds ^Summary$Builder builder ^long max-age)
|
||||
(.ageBuckets ^Summary$Builder builder buckets))
|
||||
_ (doseq [[q e] quantiles]
|
||||
(.quantile ^Summary instance q e))
|
||||
(.quantile ^Summary$Builder builder q e))
|
||||
_ (when (seq labels)
|
||||
(.labelNames instance (into-array String labels)))
|
||||
instance (.register instance registry)]
|
||||
(.labelNames ^Summary$Builder builder (into-array String labels)))
|
||||
instance (.register ^Summary$Builder builder registry)]
|
||||
|
||||
{::instance instance
|
||||
::fn (fn [{:keys [val labels]}]
|
||||
(if labels
|
||||
(.. ^Summary instance
|
||||
(labels (into-array String labels))
|
||||
(observe val))
|
||||
(.observe ^Summary instance val)))}))
|
||||
|
||||
(def default-histogram-buckets
|
||||
[1 5 10 25 50 75 100 250 500 750 1000 2500 5000 7500])
|
||||
::fn (fn [{:keys [val labels] :or {labels default-empty-labels}}]
|
||||
(let [instance (.labels ^Summary instance (if (is-array? labels) labels (into-array String labels)))]
|
||||
(.observe ^Summary$Child instance val)))}))
|
||||
|
||||
(defn make-histogram
|
||||
[{:keys [name help registry reg labels buckets]
|
||||
|
|
@ -204,12 +247,9 @@
|
|||
instance (.register instance registry)]
|
||||
|
||||
{::instance instance
|
||||
::fn (fn [{:keys [val labels]}]
|
||||
(if labels
|
||||
(.. ^Histogram instance
|
||||
(labels (into-array String labels))
|
||||
(observe val))
|
||||
(.observe ^Histogram instance val)))}))
|
||||
::fn (fn [{:keys [val labels] :or {labels default-empty-labels}}]
|
||||
(let [instance (.labels ^Histogram instance (if (is-array? labels) labels (into-array String labels)))]
|
||||
(.observe ^Histogram$Child instance val)))}))
|
||||
|
||||
(defn create
|
||||
[{:keys [type] :as props}]
|
||||
|
|
@ -219,114 +259,6 @@
|
|||
:summary (make-summary props)
|
||||
:histogram (make-histogram props)))
|
||||
|
||||
(defn wrap-counter
|
||||
([rootf mobj]
|
||||
(let [mdata (meta rootf)
|
||||
origf (::original mdata rootf)]
|
||||
(with-meta
|
||||
(fn
|
||||
([a]
|
||||
((::fn mobj) nil)
|
||||
(origf a))
|
||||
([a b]
|
||||
((::fn mobj) nil)
|
||||
(origf a b))
|
||||
([a b c]
|
||||
((::fn mobj) nil)
|
||||
(origf a b c))
|
||||
([a b c d]
|
||||
((::fn mobj) nil)
|
||||
(origf a b c d))
|
||||
([a b c d & more]
|
||||
((::fn mobj) nil)
|
||||
(apply origf a b c d more)))
|
||||
(assoc mdata ::original origf))))
|
||||
([rootf mobj labels]
|
||||
(let [mdata (meta rootf)
|
||||
origf (::original mdata rootf)]
|
||||
(with-meta
|
||||
(fn
|
||||
([a]
|
||||
((::fn mobj) {:labels labels})
|
||||
(origf a))
|
||||
([a b]
|
||||
((::fn mobj) {:labels labels})
|
||||
(origf a b))
|
||||
([a b & more]
|
||||
((::fn mobj) {:labels labels})
|
||||
(apply origf a b more)))
|
||||
(assoc mdata ::original origf)))))
|
||||
|
||||
(defn wrap-summary
|
||||
([rootf mobj]
|
||||
(let [mdata (meta rootf)
|
||||
origf (::original mdata rootf)]
|
||||
(with-meta
|
||||
(fn
|
||||
([a]
|
||||
(with-measure
|
||||
:expr (origf a)
|
||||
:cb #((::fn mobj) {:val %})))
|
||||
([a b]
|
||||
(with-measure
|
||||
:expr (origf a b)
|
||||
:cb #((::fn mobj) {:val %})))
|
||||
([a b & more]
|
||||
(with-measure
|
||||
:expr (apply origf a b more)
|
||||
:cb #((::fn mobj) {:val %}))))
|
||||
(assoc mdata ::original origf))))
|
||||
|
||||
([rootf mobj labels]
|
||||
(let [mdata (meta rootf)
|
||||
origf (::original mdata rootf)]
|
||||
(with-meta
|
||||
(fn
|
||||
([a]
|
||||
(with-measure
|
||||
:expr (origf a)
|
||||
:cb #((::fn mobj) {:val % :labels labels})))
|
||||
([a b]
|
||||
(with-measure
|
||||
:expr (origf a b)
|
||||
:cb #((::fn mobj) {:val % :labels labels})))
|
||||
([a b & more]
|
||||
(with-measure
|
||||
:expr (apply origf a b more)
|
||||
:cb #((::fn mobj) {:val % :labels labels}))))
|
||||
(assoc mdata ::original origf)))))
|
||||
|
||||
(defn instrument-vars!
|
||||
[vars {:keys [wrap] :as props}]
|
||||
(let [obj (create props)]
|
||||
(cond
|
||||
(instance? Counter (::instance obj))
|
||||
(doseq [var vars]
|
||||
(alter-var-root var (or wrap wrap-counter) obj))
|
||||
|
||||
(instance? Summary (::instance obj))
|
||||
(doseq [var vars]
|
||||
(alter-var-root var (or wrap wrap-summary) obj))
|
||||
|
||||
:else
|
||||
(ex/raise :type :not-implemented))))
|
||||
|
||||
(defn instrument
|
||||
[f {:keys [wrap] :as props}]
|
||||
(let [obj (create props)]
|
||||
(cond
|
||||
(instance? Counter (::instance obj))
|
||||
((or wrap wrap-counter) f obj)
|
||||
|
||||
(instance? Summary (::instance obj))
|
||||
((or wrap wrap-summary) f obj)
|
||||
|
||||
(instance? Histogram (::instance obj))
|
||||
((or wrap wrap-summary) f obj)
|
||||
|
||||
:else
|
||||
(ex/raise :type :not-implemented))))
|
||||
|
||||
(defn instrument-jetty!
|
||||
[^CollectorRegistry registry ^StatisticsHandler handler]
|
||||
(doto (JettyStatisticsCollector. handler)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue