mirror of
https://github.com/penpot/penpot.git
synced 2025-07-27 04:07:26 +02:00
♻️ Refactor websockets subsystem (on backend)
- Refactor msgbus subsystem, simplifying many parts. - Enable persistent websocket connection for the all session duration.
This commit is contained in:
Andrey Antukh
Alonso Torres
parent
4a9e38a221
commit
f60d8c6c96
12 changed files with 482 additions and 362 deletions
|
|
@ -7,12 +7,15 @@
|
|||
(ns app.msgbus
|
||||
"The msgbus abstraction implemented using redis as underlying backend."
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.common.spec :as us]
|
||||
[app.common.transit :as t]
|
||||
[app.config :as cfg]
|
||||
[app.util.blob :as blob]
|
||||
[app.util.async :as aa]
|
||||
[app.util.time :as dt]
|
||||
[app.worker :as wrk]
|
||||
[clojure.core.async :as a]
|
||||
[clojure.spec.alpha :as s]
|
||||
[integrant.core :as ig]
|
||||
|
|
@ -28,120 +31,83 @@
|
|||
io.lettuce.core.codec.StringCodec
|
||||
io.lettuce.core.pubsub.RedisPubSubListener
|
||||
io.lettuce.core.pubsub.StatefulRedisPubSubConnection
|
||||
io.lettuce.core.pubsub.api.async.RedisPubSubAsyncCommands
|
||||
io.lettuce.core.pubsub.api.sync.RedisPubSubCommands
|
||||
io.lettuce.core.resource.ClientResources
|
||||
io.lettuce.core.resource.DefaultClientResources
|
||||
java.time.Duration))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(def ^:private prefix (cfg/get :tenant))
|
||||
|
||||
(defn- prefix-topic
|
||||
[topic]
|
||||
(str prefix "." topic))
|
||||
|
||||
(def xform-prefix (map prefix-topic))
|
||||
(def xform-topics (map (fn [m] (update m :topics #(into #{} xform-prefix %)))))
|
||||
(def xform-topic (map (fn [m] (update m :topic prefix-topic))))
|
||||
(def ^:private xform-prefix-topic
|
||||
(map (fn [obj] (update obj :topic prefix-topic))))
|
||||
|
||||
(s/def ::redis-uri ::us/string)
|
||||
(s/def ::buffer-size ::us/integer)
|
||||
|
||||
(defmulti init-backend :backend)
|
||||
(defmulti stop-backend :backend)
|
||||
(defmulti init-pub-loop :backend)
|
||||
(defmulti init-sub-loop :backend)
|
||||
|
||||
(defmethod ig/pre-init-spec ::msgbus [_]
|
||||
(s/keys :opt-un [::buffer-size ::redis-uri]))
|
||||
(declare ^:private redis-connect)
|
||||
(declare ^:private redis-disconnect)
|
||||
(declare ^:private start-io-loop)
|
||||
(declare ^:private subscribe)
|
||||
(declare ^:private purge)
|
||||
(declare ^:private redis-pub)
|
||||
(declare ^:private redis-sub)
|
||||
(declare ^:private redis-unsub)
|
||||
|
||||
(defmethod ig/prep-key ::msgbus
|
||||
[_ cfg]
|
||||
(merge {:buffer-size 128} cfg))
|
||||
(merge {:buffer-size 128
|
||||
:timeout (dt/duration {:seconds 30})}
|
||||
(d/without-nils cfg)))
|
||||
|
||||
(s/def ::timeout ::dt/duration)
|
||||
(s/def ::redis-uri ::us/string)
|
||||
(s/def ::buffer-size ::us/integer)
|
||||
|
||||
(defmethod ig/pre-init-spec ::msgbus [_]
|
||||
(s/keys :req-un [::buffer-size ::redis-uri ::timeout ::wrk/executor]))
|
||||
|
||||
(defmethod ig/init-key ::msgbus
|
||||
[_ {:keys [backend buffer-size] :as cfg}]
|
||||
(l/debug :action "initialize msgbus"
|
||||
:backend (name backend))
|
||||
(let [cfg (init-backend cfg)
|
||||
[_ {:keys [buffer-size redis-uri] :as cfg}]
|
||||
(l/info :hint "initialize msgbus"
|
||||
:buffer-size buffer-size
|
||||
:redis-uri redis-uri)
|
||||
(let [cmd-ch (a/chan buffer-size)
|
||||
rcv-ch (a/chan (a/dropping-buffer buffer-size))
|
||||
pub-ch (a/chan (a/dropping-buffer buffer-size) xform-prefix-topic)
|
||||
state (agent {} :error-handler #(l/error :cause % :hint "unexpected error on agent" ::l/async false))
|
||||
cfg (-> (redis-connect cfg)
|
||||
(assoc ::cmd-ch cmd-ch)
|
||||
(assoc ::rcv-ch rcv-ch)
|
||||
(assoc ::pub-ch pub-ch)
|
||||
(assoc ::state state))]
|
||||
|
||||
;; Channel used for receive publications from the application.
|
||||
pub-ch (-> (a/dropping-buffer buffer-size)
|
||||
(a/chan xform-topic))
|
||||
|
||||
;; Channel used for receive subscription requests.
|
||||
sub-ch (a/chan 1 xform-topics)
|
||||
|
||||
cfg (-> cfg
|
||||
(assoc ::pub-ch pub-ch)
|
||||
(assoc ::sub-ch sub-ch))]
|
||||
|
||||
(init-pub-loop cfg)
|
||||
(init-sub-loop cfg)
|
||||
(start-io-loop cfg)
|
||||
|
||||
(with-meta
|
||||
(fn run
|
||||
([command] (run command nil))
|
||||
([command params]
|
||||
(a/go
|
||||
(case command
|
||||
:pub (a/>! pub-ch params)
|
||||
:sub (a/>! sub-ch params)))))
|
||||
(fn [& {:keys [cmd] :as params}]
|
||||
(a/go
|
||||
(case cmd
|
||||
:pub (a/>! pub-ch params)
|
||||
:sub (a/<! (subscribe cfg params))
|
||||
:purge (a/<! (purge cfg params))
|
||||
(l/error :hint "unexpeced error on msgbus command processing" :params params))))
|
||||
cfg)))
|
||||
|
||||
(defmethod ig/halt-key! ::msgbus
|
||||
[_ f]
|
||||
(let [mdata (meta f)]
|
||||
(stop-backend mdata)
|
||||
(a/close! (::pub-ch mdata))
|
||||
(a/close! (::sub-ch mdata))))
|
||||
(redis-disconnect mdata)
|
||||
(a/close! (::cmd-ch mdata))
|
||||
(a/close! (::rcv-ch mdata))))
|
||||
|
||||
;; --- IN-MEMORY BACKEND IMPL
|
||||
;; --- IMPL
|
||||
|
||||
(defmethod init-backend :memory [cfg] cfg)
|
||||
(defmethod stop-backend :memory [_])
|
||||
(defmethod init-pub-loop :memory [_])
|
||||
|
||||
(defmethod init-sub-loop :memory
|
||||
[{:keys [::sub-ch ::pub-ch]}]
|
||||
(a/go-loop [state {}]
|
||||
(let [[val port] (a/alts! [pub-ch sub-ch])]
|
||||
(cond
|
||||
(and (= port sub-ch) (some? val))
|
||||
(let [{:keys [topics chan]} val]
|
||||
(recur (reduce #(update %1 %2 (fnil conj #{}) chan) state topics)))
|
||||
|
||||
(and (= port pub-ch) (some? val))
|
||||
(let [topic (:topic val)
|
||||
message (:message val)
|
||||
state (loop [state state
|
||||
chans (get state topic)]
|
||||
(if-let [c (first chans)]
|
||||
(if (a/>! c message)
|
||||
(recur state (rest chans))
|
||||
(recur (update state topic disj c)
|
||||
(rest chans)))
|
||||
state))]
|
||||
(recur state))
|
||||
|
||||
:else
|
||||
(->> (vals state)
|
||||
(mapcat identity)
|
||||
(run! a/close!))))))
|
||||
|
||||
|
||||
;; Add a unique listener to connection
|
||||
|
||||
;; --- REDIS BACKEND IMPL
|
||||
|
||||
(declare impl-redis-open?)
|
||||
(declare impl-redis-pub)
|
||||
(declare impl-redis-sub)
|
||||
(declare impl-redis-unsub)
|
||||
|
||||
|
||||
(defmethod init-backend :redis
|
||||
[{:keys [redis-uri] :as cfg}]
|
||||
(let [codec (RedisCodec/of StringCodec/UTF8 ByteArrayCodec/INSTANCE)
|
||||
(defn- redis-connect
|
||||
[{:keys [redis-uri timeout] :as cfg}]
|
||||
(let [codec (RedisCodec/of StringCodec/UTF8 ByteArrayCodec/INSTANCE)
|
||||
|
||||
resources (.. (DefaultClientResources/builder)
|
||||
(ioThreadPoolSize 4)
|
||||
|
|
@ -151,162 +117,181 @@
|
|||
uri (RedisURI/create redis-uri)
|
||||
rclient (RedisClient/create ^ClientResources resources ^RedisURI uri)
|
||||
|
||||
pub-conn (.connect ^RedisClient rclient ^RedisCodec codec)
|
||||
sub-conn (.connectPubSub ^RedisClient rclient ^RedisCodec codec)]
|
||||
pconn (.connect ^RedisClient rclient ^RedisCodec codec)
|
||||
sconn (.connectPubSub ^RedisClient rclient ^RedisCodec codec)]
|
||||
|
||||
(.setTimeout ^StatefulRedisConnection pub-conn ^Duration (dt/duration {:seconds 10}))
|
||||
(.setTimeout ^StatefulRedisPubSubConnection sub-conn ^Duration (dt/duration {:seconds 10}))
|
||||
(.setTimeout ^StatefulRedisConnection pconn ^Duration timeout)
|
||||
(.setTimeout ^StatefulRedisPubSubConnection sconn ^Duration timeout)
|
||||
|
||||
(-> cfg
|
||||
(assoc ::resources resources)
|
||||
(assoc ::pub-conn pub-conn)
|
||||
(assoc ::sub-conn sub-conn))))
|
||||
(assoc ::pconn pconn)
|
||||
(assoc ::sconn sconn))))
|
||||
|
||||
(defmethod stop-backend :redis
|
||||
[{:keys [::pub-conn ::sub-conn ::resources] :as cfg}]
|
||||
(.close ^StatefulRedisConnection pub-conn)
|
||||
(.close ^StatefulRedisPubSubConnection sub-conn)
|
||||
(defn- redis-disconnect
|
||||
[{:keys [::pconn ::sconn ::resources] :as cfg}]
|
||||
(.. ^StatefulConnection pconn close)
|
||||
(.. ^StatefulConnection sconn close)
|
||||
(.shutdown ^ClientResources resources))
|
||||
|
||||
(defmethod init-pub-loop :redis
|
||||
[{:keys [::pub-conn ::pub-ch]}]
|
||||
(let [rac (.async ^StatefulRedisConnection pub-conn)]
|
||||
(a/go-loop []
|
||||
(when-let [val (a/<! pub-ch)]
|
||||
(let [result (a/<! (impl-redis-pub rac val))]
|
||||
(when (and (impl-redis-open? pub-conn)
|
||||
(ex/exception? result))
|
||||
(l/error :cause result
|
||||
:hint "unexpected error on publish message to redis")))
|
||||
(recur)))))
|
||||
(defn- conj-subscription
|
||||
"A low level function that is responsible to create on-demand
|
||||
subscriptions on redis. It reuses the same subscription if it is
|
||||
already established. Intended to be executed in agent."
|
||||
[nsubs cfg topic chan]
|
||||
(let [nsubs (if (nil? nsubs) #{chan} (conj nsubs chan))]
|
||||
(when (= 1 (count nsubs))
|
||||
(l/trace :hint "open subscription" :topic topic ::l/async false)
|
||||
(redis-sub cfg topic))
|
||||
nsubs))
|
||||
|
||||
(defmethod init-sub-loop :redis
|
||||
[{:keys [::sub-conn ::sub-ch buffer-size]}]
|
||||
(let [rcv-ch (a/chan (a/dropping-buffer buffer-size))
|
||||
chans (agent {} :error-handler #(l/error :cause % :hint "unexpected error on agent"))
|
||||
rac (.async ^StatefulRedisPubSubConnection sub-conn)]
|
||||
(defn- disj-subscription
|
||||
"A low level function responsible on removing subscriptions. The
|
||||
subscription is trully removed from redis once no single local
|
||||
subscription is look for it. Intended to be executed in agent."
|
||||
[nsubs cfg topic chan]
|
||||
(let [nsubs (disj nsubs chan)]
|
||||
(when (empty? nsubs)
|
||||
(l/trace :hint "close subscription" :topic topic ::l/async false)
|
||||
(redis-unsub cfg topic))
|
||||
nsubs))
|
||||
|
||||
;; Add a unique listener to connection
|
||||
(.addListener sub-conn
|
||||
(reify RedisPubSubListener
|
||||
(message [_ _pattern _topic _message])
|
||||
(message [_ topic message]
|
||||
;; There are no back pressure, so we use a slidding
|
||||
;; buffer for cases when the pubsub broker sends
|
||||
;; more messages that we can process.
|
||||
(let [val {:topic topic :message (blob/decode message)}]
|
||||
(when-not (a/offer! rcv-ch val)
|
||||
(l/warn :msg "dropping message on subscription loop"))))
|
||||
(psubscribed [_ _pattern _count])
|
||||
(punsubscribed [_ _pattern _count])
|
||||
(subscribed [_ _topic _count])
|
||||
(unsubscribed [_ _topic _count])))
|
||||
(defn- subscribe-to-topics
|
||||
"Function responsible to attach local subscription to the
|
||||
state. Intended to be used in agent."
|
||||
[state cfg topics chan done-ch]
|
||||
(l/trace :hint "subscribe-to-topics" :topics topics ::l/async false)
|
||||
(aa/with-closing done-ch
|
||||
(let [state (update state :chans assoc chan topics)]
|
||||
(reduce (fn [state topic]
|
||||
(update-in state [:topics topic] conj-subscription cfg topic chan))
|
||||
state
|
||||
topics))))
|
||||
|
||||
(letfn [(subscribe-to-single-topic [nsubs topic chan]
|
||||
(let [nsubs (if (nil? nsubs) #{chan} (conj nsubs chan))]
|
||||
(when (= 1 (count nsubs))
|
||||
(let [result (a/<!! (impl-redis-sub rac topic))]
|
||||
(l/trace :action "open subscription"
|
||||
:topic topic)
|
||||
(when (ex/exception? result)
|
||||
(l/error :cause result
|
||||
:hint "unexpected exception on subscribing"
|
||||
:topic topic))))
|
||||
nsubs))
|
||||
(defn- unsubscribe-single-channel
|
||||
"Auxiliar function responsible on removing a single local
|
||||
subscription from the state."
|
||||
[state cfg chan]
|
||||
(let [topics (get-in state [:chans chan])
|
||||
state (update state :chans dissoc chan)]
|
||||
(reduce (fn [state topic]
|
||||
(update-in state [:topics topic] disj-subscription cfg topic chan))
|
||||
state
|
||||
topics)))
|
||||
|
||||
(subscribe-to-topics [state topics chan]
|
||||
(let [state (update state :chans assoc chan topics)]
|
||||
(reduce (fn [state topic]
|
||||
(update-in state [:topics topic] subscribe-to-single-topic topic chan))
|
||||
state
|
||||
topics)))
|
||||
(defn- unsubscribe-channels
|
||||
"Function responsible from detach from state a seq of channels,
|
||||
useful when client disconnects or in-bulk unsubscribe
|
||||
operations. Intended to be executed in agent."
|
||||
[state cfg channels done-ch]
|
||||
(l/trace :hint "unsubscribe-channels" :chans (count channels) ::l/async false)
|
||||
(aa/with-closing done-ch
|
||||
(reduce #(unsubscribe-single-channel %1 cfg %2) state channels)))
|
||||
|
||||
(unsubscribe-from-single-topic [nsubs topic chan]
|
||||
(let [nsubs (disj nsubs chan)]
|
||||
(when (empty? nsubs)
|
||||
(let [result (a/<!! (impl-redis-unsub rac topic))]
|
||||
(l/trace :action "close subscription"
|
||||
:topic topic)
|
||||
(when (and (impl-redis-open? sub-conn)
|
||||
(ex/exception? result))
|
||||
(l/error :cause result
|
||||
:hint "unexpected exception on unsubscribing"
|
||||
:topic topic))))
|
||||
nsubs))
|
||||
(defn- subscribe
|
||||
[{:keys [::state executor] :as cfg} {:keys [topic topics chan]}]
|
||||
(let [done-ch (a/chan)
|
||||
topics (into [] (map prefix-topic) (if topic [topic] topics))]
|
||||
(l/trace :hint "subscribe" :topics topics)
|
||||
(send-via executor state subscribe-to-topics cfg topics chan done-ch)
|
||||
done-ch))
|
||||
|
||||
(unsubscribe-channels [state pending]
|
||||
(reduce (fn [state ch]
|
||||
(let [topics (get-in state [:chans ch])
|
||||
state (update state :chans dissoc ch)]
|
||||
(reduce (fn [state topic]
|
||||
(update-in state [:topics topic] unsubscribe-from-single-topic topic ch))
|
||||
state
|
||||
topics)))
|
||||
state
|
||||
pending))]
|
||||
(defn- purge
|
||||
[{:keys [::state executor] :as cfg} {:keys [chans]}]
|
||||
(l/trace :hint "purge" :chans (count chans))
|
||||
(let [done-ch (a/chan)]
|
||||
(send-via executor state unsubscribe-channels cfg chans done-ch)
|
||||
done-ch))
|
||||
|
||||
;; Asynchronous subscription loop;
|
||||
(a/go-loop []
|
||||
(if-let [{:keys [topics chan]} (a/<! sub-ch)]
|
||||
(do
|
||||
(send-off chans subscribe-to-topics topics chan)
|
||||
(recur))
|
||||
(a/close! rcv-ch)))
|
||||
(defn- create-listener
|
||||
[rcv-ch]
|
||||
(reify RedisPubSubListener
|
||||
(message [_ _pattern _topic _message])
|
||||
(message [_ topic message]
|
||||
;; There are no back pressure, so we use a slidding
|
||||
;; buffer for cases when the pubsub broker sends
|
||||
;; more messages that we can process.
|
||||
(let [val {:topic topic :message (t/decode message)}]
|
||||
(when-not (a/offer! rcv-ch val)
|
||||
(l/warn :msg "dropping message on subscription loop"))))
|
||||
(psubscribed [_ _pattern _count])
|
||||
(punsubscribed [_ _pattern _count])
|
||||
(subscribed [_ _topic _count])
|
||||
(unsubscribed [_ _topic _count])))
|
||||
|
||||
;; Asynchronous message processing loop;x
|
||||
(a/go-loop []
|
||||
(if-let [{:keys [topic message]} (a/<! rcv-ch)]
|
||||
;; This means we receive data from redis and we need to
|
||||
;; forward it to the underlying subscriptions.
|
||||
(let [pending (loop [chans (seq (get-in @chans [:topics topic]))
|
||||
pending #{}]
|
||||
(if-let [ch (first chans)]
|
||||
(if (a/>! ch message)
|
||||
(recur (rest chans) pending)
|
||||
(recur (rest chans) (conj pending ch)))
|
||||
pending))]
|
||||
(some->> (seq pending)
|
||||
(send-off chans unsubscribe-channels))
|
||||
(defn start-io-loop
|
||||
[{:keys [::sconn ::rcv-ch ::pub-ch ::state executor] :as cfg}]
|
||||
|
||||
(recur))
|
||||
;; Add a single listener to the pubsub connection
|
||||
(.addListener ^StatefulRedisPubSubConnection sconn
|
||||
^RedisPubSubListener (create-listener rcv-ch))
|
||||
|
||||
;; Stop condition; close all underlying subscriptions and
|
||||
;; exit. The close operation is performed asynchronously.
|
||||
(send-off chans (fn [state]
|
||||
(->> (vals state)
|
||||
(mapcat identity)
|
||||
(filter some?)
|
||||
(run! a/close!)))))))))
|
||||
(letfn [(send-to-topic [topic message]
|
||||
(a/go-loop [chans (seq (get-in @state [:topics topic]))
|
||||
closed #{}]
|
||||
(if-let [ch (first chans)]
|
||||
(if (a/>! ch message)
|
||||
(recur (rest chans) closed)
|
||||
(recur (rest chans) (conj closed ch)))
|
||||
(seq closed))))
|
||||
|
||||
(process-incoming [{:keys [topic message]}]
|
||||
(a/go
|
||||
(when-let [closed (a/<! (send-to-topic topic message))]
|
||||
(send-via executor state unsubscribe-channels cfg closed nil))))
|
||||
]
|
||||
|
||||
(defn- impl-redis-open?
|
||||
[^StatefulConnection conn]
|
||||
(.isOpen conn))
|
||||
(a/go-loop []
|
||||
(let [[val port] (a/alts! [pub-ch rcv-ch])]
|
||||
(cond
|
||||
(nil? val)
|
||||
(do
|
||||
(l/trace :hint "stoping io-loop, nil received")
|
||||
(send-via executor state (fn [state]
|
||||
(->> (vals state)
|
||||
(mapcat identity)
|
||||
(filter some?)
|
||||
(run! a/close!))
|
||||
nil)))
|
||||
|
||||
(defn- impl-redis-pub
|
||||
[^RedisAsyncCommands rac {:keys [topic message]}]
|
||||
(let [message (blob/encode message)
|
||||
res (a/chan 1)]
|
||||
(-> (.publish rac ^String topic ^bytes message)
|
||||
(p/finally (fn [_ e]
|
||||
(when e (a/>!! res e))
|
||||
(= port rcv-ch)
|
||||
(do
|
||||
(a/<! (process-incoming val))
|
||||
(recur))
|
||||
|
||||
(= port pub-ch)
|
||||
(let [result (a/<! (redis-pub cfg val))]
|
||||
(when (ex/exception? result)
|
||||
(l/error :hint "unexpected error on publishing" :message val
|
||||
:cause result))
|
||||
(recur)))))))
|
||||
|
||||
(defn- redis-pub
|
||||
"Publish a message to the redis server. Asynchronous operation,
|
||||
intended to be used in core.async go blocks."
|
||||
[{:keys [::pconn] :as cfg} {:keys [topic message]}]
|
||||
(let [message (t/encode message)
|
||||
res (a/chan 1)
|
||||
pcomm (.async ^StatefulRedisConnection pconn)]
|
||||
(-> (.publish ^RedisAsyncCommands pcomm ^String topic ^bytes message)
|
||||
(p/finally (fn [_ cause]
|
||||
(when (and cause (.isOpen ^StatefulConnection pconn))
|
||||
(a/offer! res cause))
|
||||
(a/close! res))))
|
||||
res))
|
||||
|
||||
(defn impl-redis-sub
|
||||
[^RedisPubSubAsyncCommands rac topic]
|
||||
(let [res (a/chan 1)]
|
||||
(-> (.subscribe rac (into-array String [topic]))
|
||||
(p/finally (fn [_ e]
|
||||
(when e (a/>!! res e))
|
||||
(a/close! res))))
|
||||
res))
|
||||
(defn redis-sub
|
||||
"Create redis subscription. Blocking operation, intended to be used
|
||||
inside an agent."
|
||||
[{:keys [::sconn] :as cfg} topic]
|
||||
(let [topic (into-array String [topic])
|
||||
scomm (.sync ^StatefulRedisPubSubConnection sconn)]
|
||||
(.subscribe ^RedisPubSubCommands scomm topic)))
|
||||
|
||||
(defn impl-redis-unsub
|
||||
[rac topic]
|
||||
(let [res (a/chan 1)]
|
||||
(-> (.unsubscribe rac (into-array String [topic]))
|
||||
(p/finally (fn [_ e]
|
||||
(when e (a/>!! res e))
|
||||
(a/close! res))))
|
||||
res))
|
||||
(defn redis-unsub
|
||||
"Removes redis subscription. Blocking operation, intended to be used
|
||||
inside an agent."
|
||||
[{:keys [::sconn] :as cfg} topic]
|
||||
(let [topic (into-array String [topic])
|
||||
scomm (.sync ^StatefulRedisPubSubConnection sconn)]
|
||||
(.unsubscribe ^RedisPubSubCommands scomm topic)))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue