mirror of
https://github.com/penpot/penpot.git
synced 2025-06-28 14:06:59 +02:00
♻️ Refactor logging subsystem and error reporting
This commit is contained in:
Andrey Antukh
parent
50ee0ad3fd
commit
bb055a3c84
30 changed files with 759 additions and 892 deletions
|
|
@ -51,7 +51,6 @@
|
|||
:database-password "penpot"
|
||||
|
||||
:default-blob-version 5
|
||||
:loggers-zmq-uri "tcp://localhost:45556"
|
||||
|
||||
:rpc-rlimit-config (fs/path "resources/rlimit.edn")
|
||||
:rpc-climit-config (fs/path "resources/climit.edn")
|
||||
|
|
@ -175,8 +174,6 @@
|
|||
(s/def ::ldap-ssl ::us/boolean)
|
||||
(s/def ::ldap-starttls ::us/boolean)
|
||||
(s/def ::ldap-user-query ::us/string)
|
||||
(s/def ::loggers-loki-uri ::us/string)
|
||||
(s/def ::loggers-zmq-uri ::us/string)
|
||||
(s/def ::media-directory ::us/string)
|
||||
(s/def ::media-uri ::us/string)
|
||||
(s/def ::profile-bounce-max-age ::dt/duration)
|
||||
|
|
@ -272,8 +269,6 @@
|
|||
::ldap-starttls
|
||||
::ldap-user-query
|
||||
::local-assets-uri
|
||||
::loggers-loki-uri
|
||||
::loggers-zmq-uri
|
||||
::media-max-file-size
|
||||
::profile-bounce-max-age
|
||||
::profile-bounce-threshold
|
||||
|
|
@ -357,7 +352,7 @@
|
|||
(merge defaults)
|
||||
(us/conform ::config))
|
||||
(catch Throwable e
|
||||
(when (ex/ex-info? e)
|
||||
(when (ex/error? e)
|
||||
(println ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;")
|
||||
(println "Error on validating configuration:")
|
||||
(println (some-> e ex-data ex/explain))
|
||||
|
|
|
|||
|
|
@ -205,45 +205,47 @@
|
|||
|
||||
(defn error-handler
|
||||
[{:keys [::db/pool]} request]
|
||||
(letfn [(parse-id [request]
|
||||
(let [id (get-in request [:path-params :id])
|
||||
id (parse-uuid id)]
|
||||
(when (uuid? id)
|
||||
id)))
|
||||
|
||||
(retrieve-report [id]
|
||||
(letfn [(get-report [{:keys [path-params]}]
|
||||
(ex/ignoring
|
||||
(some-> (db/get-by-id pool :server-error-report id) :content db/decode-transit-pgobject)))
|
||||
(let [report-id (some-> path-params :id parse-uuid)]
|
||||
(some-> (db/get-by-id pool :server-error-report report-id)
|
||||
(update :content db/decode-transit-pgobject)))))
|
||||
|
||||
(render-template [report]
|
||||
(let [context (dissoc report
|
||||
(render-template-v1 [{:keys [content]}]
|
||||
(let [context (dissoc content
|
||||
:trace :cause :params :data :spec-problems :message
|
||||
:spec-explain :spec-value :error :explain :hint)
|
||||
params {:context (pp/pprint-str context :width 200)
|
||||
:hint (:hint report)
|
||||
:spec-explain (:spec-explain report)
|
||||
:spec-problems (:spec-problems report)
|
||||
:spec-value (:spec-value report)
|
||||
:data (:data report)
|
||||
:trace (or (:trace report)
|
||||
(some-> report :error :trace))
|
||||
:params (:params report)}]
|
||||
:hint (:hint content)
|
||||
:spec-explain (:spec-explain content)
|
||||
:spec-problems (:spec-problems content)
|
||||
:spec-value (:spec-value content)
|
||||
:data (:data content)
|
||||
:trace (or (:trace content)
|
||||
(some-> content :error :trace))
|
||||
:params (:params content)}]
|
||||
(-> (io/resource "app/templates/error-report.tmpl")
|
||||
(tmpl/render params))))]
|
||||
(tmpl/render params))))
|
||||
|
||||
(render-template-v2 [{report :content}]
|
||||
(-> (io/resource "app/templates/error-report.v2.tmpl")
|
||||
(tmpl/render report)))
|
||||
|
||||
]
|
||||
|
||||
(when-not (authorized? pool request)
|
||||
(ex/raise :type :authentication
|
||||
:code :only-admins-allowed))
|
||||
|
||||
(let [result (some-> (parse-id request)
|
||||
(retrieve-report)
|
||||
(render-template))]
|
||||
(if result
|
||||
(if-let [report (get-report request)]
|
||||
(let [result (if (= 1 (:version report))
|
||||
(render-template-v1 report)
|
||||
(render-template-v2 report))]
|
||||
(yrs/response :status 200
|
||||
:body result
|
||||
:headers {"content-type" "text/html; charset=utf-8"
|
||||
"x-robots-tag" "noindex"})
|
||||
(yrs/response 404 "not found")))))
|
||||
"x-robots-tag" "noindex"}))
|
||||
(yrs/response 404 "not found"))))
|
||||
|
||||
(def sql:error-reports
|
||||
"SELECT id, created_at,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@
|
|||
(ns app.http.errors
|
||||
"A errors handling for the http server."
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.http :as-alias http]
|
||||
|
|
@ -18,30 +17,26 @@
|
|||
[yetti.request :as yrq]
|
||||
[yetti.response :as yrs]))
|
||||
|
||||
(def ^:dynamic *context* {})
|
||||
|
||||
(defn- parse-client-ip
|
||||
[request]
|
||||
(or (some-> (yrq/get-header request "x-forwarded-for") (str/split ",") first)
|
||||
(yrq/get-header request "x-real-ip")
|
||||
(yrq/remote-addr request)))
|
||||
|
||||
(defn get-context
|
||||
(defn request->context
|
||||
"Extracts error report relevant context data from request."
|
||||
[request]
|
||||
(let [claims (-> {}
|
||||
(into (::session/token-claims request))
|
||||
(into (::actoken/token-claims request)))]
|
||||
(merge
|
||||
*context*
|
||||
{:path (:path request)
|
||||
:method (:method request)
|
||||
:params (:params request)
|
||||
:ip-addr (parse-client-ip request)}
|
||||
(d/without-nils
|
||||
{:user-agent (yrq/get-header request "user-agent")
|
||||
:frontend-version (or (yrq/get-header request "x-frontend-version")
|
||||
"unknown")
|
||||
:profile-id (:uid claims)}))))
|
||||
{:path (:path request)
|
||||
:method (:method request)
|
||||
:params (:params request)
|
||||
:ip-addr (parse-client-ip request)
|
||||
:user-agent (yrq/get-header request "user-agent")
|
||||
:profile-id (:uid claims)
|
||||
:version (or (yrq/get-header request "x-frontend-version")
|
||||
"unknown")}))
|
||||
|
||||
(defmulti handle-exception
|
||||
(fn [err & _rest]
|
||||
|
|
@ -87,15 +82,14 @@
|
|||
[error request]
|
||||
(let [edata (ex-data error)
|
||||
explain (ex/explain edata)]
|
||||
(l/error :hint (ex-message error)
|
||||
:cause error
|
||||
::l/context (get-context request))
|
||||
(yrs/response :status 500
|
||||
:body {:type :server-error
|
||||
:code :assertion
|
||||
:data (-> edata
|
||||
(dissoc ::s/problems ::s/value ::s/spec)
|
||||
(cond-> explain (assoc :explain explain)))})))
|
||||
(binding [l/*context* (request->context request)]
|
||||
(l/error :hint "Assertion error" :message (ex-message error) :cause error)
|
||||
(yrs/response :status 500
|
||||
:body {:type :server-error
|
||||
:code :assertion
|
||||
:data (-> edata
|
||||
(dissoc ::s/problems ::s/value ::s/spec)
|
||||
(cond-> explain (assoc :explain explain)))}))))
|
||||
|
||||
(defmethod handle-exception :not-found
|
||||
[err _]
|
||||
|
|
@ -109,10 +103,8 @@
|
|||
(yrs/response 429)
|
||||
|
||||
:else
|
||||
(do
|
||||
(l/error :hint (ex-message error)
|
||||
:cause error
|
||||
::l/context (get-context request))
|
||||
(binding [l/*context* (request->context request)]
|
||||
(l/error :hint "Internal error" :message (ex-message error) :cause error)
|
||||
(yrs/response 500 {:type :server-error
|
||||
:code :unhandled
|
||||
:hint (ex-message error)
|
||||
|
|
@ -121,25 +113,24 @@
|
|||
(defmethod handle-exception org.postgresql.util.PSQLException
|
||||
[error request]
|
||||
(let [state (.getSQLState ^java.sql.SQLException error)]
|
||||
(l/error :hint (ex-message error)
|
||||
:cause error
|
||||
::l/context (get-context request))
|
||||
(cond
|
||||
(= state "57014")
|
||||
(yrs/response 504 {:type :server-error
|
||||
:code :statement-timeout
|
||||
:hint (ex-message error)})
|
||||
(binding [l/*context* (request->context request)]
|
||||
(l/error :hint "PSQL error" :message (ex-message error) :cause error)
|
||||
(cond
|
||||
(= state "57014")
|
||||
(yrs/response 504 {:type :server-error
|
||||
:code :statement-timeout
|
||||
:hint (ex-message error)})
|
||||
|
||||
(= state "25P03")
|
||||
(yrs/response 504 {:type :server-error
|
||||
:code :idle-in-transaction-timeout
|
||||
:hint (ex-message error)})
|
||||
(= state "25P03")
|
||||
(yrs/response 504 {:type :server-error
|
||||
:code :idle-in-transaction-timeout
|
||||
:hint (ex-message error)})
|
||||
|
||||
:else
|
||||
(yrs/response 500 {:type :server-error
|
||||
:code :unexpected
|
||||
:hint (ex-message error)
|
||||
:state state}))))
|
||||
:else
|
||||
(yrs/response 500 {:type :server-error
|
||||
:code :unexpected
|
||||
:hint (ex-message error)
|
||||
:state state})))))
|
||||
|
||||
(defmethod handle-exception :default
|
||||
[error request]
|
||||
|
|
@ -147,10 +138,8 @@
|
|||
(cond
|
||||
;; This means that exception is not a controlled exception.
|
||||
(nil? edata)
|
||||
(do
|
||||
(l/error :hint (ex-message error)
|
||||
:cause error
|
||||
::l/context (get-context request))
|
||||
(binding [l/*context* (request->context request)]
|
||||
(l/error :hint "Unexpected error" :message (ex-message error) :cause error)
|
||||
(yrs/response 500 {:type :server-error
|
||||
:code :unexpected
|
||||
:hint (ex-message error)}))
|
||||
|
|
@ -165,10 +154,8 @@
|
|||
(handle-exception (:handling edata) request)
|
||||
|
||||
:else
|
||||
(do
|
||||
(l/error :hint (ex-message error)
|
||||
:cause error
|
||||
::l/context (get-context request))
|
||||
(binding [l/*context* (request->context request)]
|
||||
(l/error :hint "Unhandled error" :message (ex-message error) :cause error)
|
||||
(yrs/response 500 {:type :server-error
|
||||
:code :unhandled
|
||||
:hint (ex-message error)
|
||||
|
|
@ -176,16 +163,7 @@
|
|||
|
||||
(defn handle
|
||||
[cause request]
|
||||
(cond
|
||||
(or (instance? java.util.concurrent.CompletionException cause)
|
||||
(instance? java.util.concurrent.ExecutionException cause))
|
||||
(handle-exception (.getCause ^Throwable cause) request)
|
||||
|
||||
(ex/wrapped? cause)
|
||||
(let [context (meta cause)
|
||||
cause (deref cause)]
|
||||
(binding [*context* context]
|
||||
(handle-exception cause request)))
|
||||
|
||||
:else
|
||||
(if (or (instance? java.util.concurrent.CompletionException cause)
|
||||
(instance? java.util.concurrent.ExecutionException cause))
|
||||
(handle-exception (ex-cause cause) request)
|
||||
(handle-exception cause request)))
|
||||
|
|
|
|||
|
|
@ -80,8 +80,8 @@
|
|||
(fn [request respond raise]
|
||||
(let [request (ex/try! (process-request request))]
|
||||
(if (ex/exception? request)
|
||||
(if (instance? RuntimeException request)
|
||||
(handle-error raise (or (ex/cause request) request))
|
||||
(if (ex/runtime-exception? request)
|
||||
(handle-error raise (or (ex-cause request) request))
|
||||
(handle-error raise request))
|
||||
(handler request respond raise))))))
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
(:refer-clojure :exclude [read])
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.common.spec :as us]
|
||||
[app.config :as cf]
|
||||
|
|
@ -230,17 +231,18 @@
|
|||
|
||||
(let [{:keys [::wrk/executor ::main/props]} (meta manager)]
|
||||
(fn [request respond raise]
|
||||
(let [token (get-token request)]
|
||||
(->> (px/submit! executor (partial decode-token props token))
|
||||
(p/fnly (fn [claims cause]
|
||||
(when cause
|
||||
(l/trace :hint "exception on decoding malformed token" :cause cause))
|
||||
|
||||
(let [request (cond-> request
|
||||
(map? claims)
|
||||
(-> (assoc ::token-claims claims)
|
||||
(assoc ::token token)))]
|
||||
(handler request respond raise)))))))))
|
||||
(let [token (ex/try! (get-token request))]
|
||||
(if (ex/exception? token)
|
||||
(raise token)
|
||||
(->> (px/submit! executor (partial decode-token props token))
|
||||
(p/fnly (fn [claims cause]
|
||||
(when cause
|
||||
(l/trace :hint "exception on decoding malformed token" :cause cause))
|
||||
(let [request (cond-> request
|
||||
(map? claims)
|
||||
(-> (assoc ::token-claims claims)
|
||||
(assoc ::token token)))]
|
||||
(handler request respond raise))))))))))
|
||||
|
||||
(defn- wrap-authz
|
||||
[handler {:keys [::manager]}]
|
||||
|
|
|
|||
|
|
@ -7,16 +7,17 @@
|
|||
(ns app.loggers.database
|
||||
"A specific logger impl that persists errors on the database."
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.common.pprint :as pp]
|
||||
[app.common.spec :as us]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.loggers.zmq :as lzmq]
|
||||
[clojure.core.async :as a]
|
||||
[clojure.spec.alpha :as s]
|
||||
[cuerdas.core :as str]
|
||||
[integrant.core :as ig]
|
||||
[promesa.exec :as px]))
|
||||
[promesa.exec :as px]
|
||||
[promesa.exec.csp :as sp]))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; Error Listener
|
||||
|
|
@ -27,73 +28,79 @@
|
|||
(defonce enabled (atom true))
|
||||
|
||||
(defn- persist-on-database!
|
||||
[{:keys [::db/pool] :as cfg} {:keys [id] :as event}]
|
||||
[pool id report]
|
||||
(when-not (db/read-only? pool)
|
||||
(db/insert! pool :server-error-report {:id id :content (db/tjson event)})))
|
||||
(db/insert! pool :server-error-report
|
||||
{:id id
|
||||
:version 2
|
||||
:content (db/tjson report)})))
|
||||
|
||||
(defn- parse-event-data
|
||||
[event]
|
||||
(reduce-kv
|
||||
(fn [acc k v]
|
||||
(cond
|
||||
(= k :id) (assoc acc k (uuid/uuid v))
|
||||
(= k :profile-id) (assoc acc k (uuid/uuid v))
|
||||
(str/blank? v) acc
|
||||
:else (assoc acc k v)))
|
||||
{}
|
||||
event))
|
||||
(defn record->report
|
||||
[{:keys [::l/context ::l/message ::l/props ::l/logger ::l/level ::l/cause] :as record}]
|
||||
(us/assert! ::l/record record)
|
||||
|
||||
(defn parse-event
|
||||
[event]
|
||||
(-> (parse-event-data event)
|
||||
(assoc :hint (or (:hint event) (:message event)))
|
||||
(assoc :tenant (cf/get :tenant))
|
||||
(assoc :host (cf/get :host))
|
||||
(assoc :public-uri (cf/get :public-uri))
|
||||
(assoc :version (:full cf/version))
|
||||
(update :id #(or % (uuid/next)))))
|
||||
(merge
|
||||
{:context (-> context
|
||||
(assoc :tenant (cf/get :tenant))
|
||||
(assoc :host (cf/get :host))
|
||||
(assoc :public-uri (cf/get :public-uri))
|
||||
(assoc :version (:full cf/version))
|
||||
(assoc :logger-name logger)
|
||||
(assoc :logger-level level)
|
||||
(dissoc :params)
|
||||
(pp/pprint-str :width 200))
|
||||
:params (some-> (:params context)
|
||||
(pp/pprint-str :width 200))
|
||||
:props (pp/pprint-str props :width 200)
|
||||
:hint (or (ex-message cause) @message)
|
||||
:trace (ex/format-throwable cause :data? false :explain? false :header? false :summary? false)}
|
||||
|
||||
(when-let [data (ex-data cause)]
|
||||
{:spec-value (some-> (::s/value data) (pp/pprint-str :width 200))
|
||||
:spec-explain (ex/explain data)
|
||||
:data (-> data
|
||||
(dissoc ::s/problems ::s/value ::s/spec :hint)
|
||||
(pp/pprint-str :width 200))})))
|
||||
|
||||
(defn- handle-event
|
||||
[cfg event]
|
||||
[{:keys [::db/pool]} {:keys [::l/id] :as record}]
|
||||
(try
|
||||
(let [event (parse-event event)
|
||||
uri (cf/get :public-uri)]
|
||||
(let [uri (cf/get :public-uri)
|
||||
report (-> record record->report d/without-nils)]
|
||||
(l/debug :hint "registering error on database" :id id
|
||||
:uri (str uri "/dbg/error/" id))
|
||||
|
||||
(l/debug :hint "registering error on database" :id (:id event)
|
||||
:uri (str uri "/dbg/error/" (:id event)))
|
||||
|
||||
(persist-on-database! cfg event))
|
||||
(persist-on-database! pool id report))
|
||||
(catch Throwable cause
|
||||
(l/warn :hint "unexpected exception on database error logger" :cause cause))))
|
||||
|
||||
(defn- error-event?
|
||||
[event]
|
||||
(= "error" (:logger/level event)))
|
||||
(defn error-record?
|
||||
[{:keys [::l/level ::l/cause]}]
|
||||
(and (= :error level)
|
||||
(ex/exception? cause)))
|
||||
|
||||
(defmethod ig/pre-init-spec ::reporter [_]
|
||||
(s/keys :req [::db/pool ::lzmq/receiver]))
|
||||
(s/keys :req [::db/pool]))
|
||||
|
||||
(defmethod ig/init-key ::reporter
|
||||
[_ {:keys [::lzmq/receiver] :as cfg}]
|
||||
(px/thread
|
||||
{:name "penpot/database-reporter"}
|
||||
(l/info :hint "initializing database error persistence")
|
||||
|
||||
(let [input (a/chan (a/sliding-buffer 5)
|
||||
(filter error-event?))]
|
||||
[_ cfg]
|
||||
(let [input (sp/chan (sp/sliding-buffer 32) (filter error-record?))]
|
||||
(add-watch l/log-record ::reporter #(sp/put! input %4))
|
||||
(px/thread
|
||||
{:name "penpot/database-reporter" :virtual true}
|
||||
(l/info :hint "initializing database error persistence")
|
||||
(try
|
||||
(lzmq/sub! receiver input)
|
||||
(loop []
|
||||
(when-let [msg (a/<!! input)]
|
||||
(handle-event cfg msg))
|
||||
(recur))
|
||||
|
||||
(when-let [record (sp/take! input)]
|
||||
(handle-event cfg record)
|
||||
(recur)))
|
||||
(catch InterruptedException _
|
||||
(l/debug :hint "reporter interrupted"))
|
||||
(catch Throwable cause
|
||||
(l/error :hint "unexpected error" :cause cause))
|
||||
(finally
|
||||
(a/close! input)
|
||||
(sp/close! input)
|
||||
(remove-watch l/log-record ::reporter)
|
||||
(l/info :hint "reporter terminated"))))))
|
||||
|
||||
(defmethod ig/halt-key! ::reporter
|
||||
|
|
|
|||
|
|
@ -1,89 +0,0 @@
|
|||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC
|
||||
|
||||
(ns app.loggers.loki
|
||||
"A Loki integration."
|
||||
(:require
|
||||
[app.common.logging :as l]
|
||||
[app.config :as cf]
|
||||
[app.http.client :as http]
|
||||
[app.loggers.zmq :as lzmq]
|
||||
[app.util.json :as json]
|
||||
[clojure.core.async :as a]
|
||||
[clojure.spec.alpha :as s]
|
||||
[integrant.core :as ig]
|
||||
[promesa.exec :as px]))
|
||||
|
||||
(declare ^:private handle-event)
|
||||
|
||||
(defmethod ig/pre-init-spec ::reporter [_]
|
||||
(s/keys :req [::http/client
|
||||
::lzmq/receiver]))
|
||||
|
||||
(defmethod ig/init-key ::reporter
|
||||
[_ cfg]
|
||||
(when-let [uri (cf/get :loggers-loki-uri)]
|
||||
(px/thread
|
||||
{:name "penpot/loki-reporter"}
|
||||
(l/info :hint "reporter started" :uri uri)
|
||||
(let [input (a/chan (a/dropping-buffer 2048))
|
||||
cfg (assoc cfg ::uri uri)]
|
||||
|
||||
(try
|
||||
(lzmq/sub! (::lzmq/receiver cfg) input)
|
||||
(loop []
|
||||
(when-let [msg (a/<!! input)]
|
||||
(handle-event cfg msg)
|
||||
(recur)))
|
||||
|
||||
(catch InterruptedException _
|
||||
(l/debug :hint "reporter interrupted"))
|
||||
(catch Throwable cause
|
||||
(l/error :hint "unexpected exception"
|
||||
:cause cause))
|
||||
(finally
|
||||
(a/close! input)
|
||||
(l/info :hint "reporter terminated")))))))
|
||||
|
||||
(defmethod ig/halt-key! ::reporter
|
||||
[_ thread]
|
||||
(some-> thread px/interrupt!))
|
||||
|
||||
(defn- prepare-payload
|
||||
[event]
|
||||
(let [labels {:host (cf/get :host)
|
||||
:tenant (cf/get :tenant)
|
||||
:version (:full cf/version)
|
||||
:logger (:logger/name event)
|
||||
:level (:logger/level event)}]
|
||||
{:streams
|
||||
[{:stream labels
|
||||
:values [[(str (* (inst-ms (:created-at event)) 1000000))
|
||||
(str (:message event)
|
||||
(when-let [error (:trace event)]
|
||||
(str "\n" error)))]]}]}))
|
||||
|
||||
(defn- make-request
|
||||
[{:keys [::uri] :as cfg} payload]
|
||||
(http/req! cfg
|
||||
{:uri uri
|
||||
:timeout 3000
|
||||
:method :post
|
||||
:headers {"content-type" "application/json"}
|
||||
:body (json/encode payload)}
|
||||
{:sync? true}))
|
||||
|
||||
(defn- handle-event
|
||||
[cfg event]
|
||||
(try
|
||||
(let [payload (prepare-payload event)
|
||||
response (make-request cfg payload)]
|
||||
(when-not (= 204 (:status response))
|
||||
(l/error :hint "error on sending log to loki (unexpected response)"
|
||||
:response (pr-str response))))
|
||||
(catch Throwable cause
|
||||
(l/error :hint "error on sending log to loki (unexpected exception)"
|
||||
:cause cause))))
|
||||
|
|
@ -7,24 +7,35 @@
|
|||
(ns app.loggers.mattermost
|
||||
"A mattermost integration for error reporting."
|
||||
(:require
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.common.spec :as us]
|
||||
[app.config :as cf]
|
||||
[app.http.client :as http]
|
||||
[app.loggers.database :as ldb]
|
||||
[app.loggers.zmq :as lzmq]
|
||||
[app.util.json :as json]
|
||||
[clojure.core.async :as a]
|
||||
[clojure.spec.alpha :as s]
|
||||
[integrant.core :as ig]
|
||||
[promesa.exec :as px]))
|
||||
[promesa.exec :as px]
|
||||
[promesa.exec.csp :as sp]))
|
||||
|
||||
(defonce enabled (atom true))
|
||||
(defonce enabled (atom false))
|
||||
|
||||
(defn- send-mattermost-notification!
|
||||
[cfg {:keys [host id public-uri] :as event}]
|
||||
(let [text (str "Exception on (host: " host ", url: " public-uri "/dbg/error/" id ")\n"
|
||||
(when-let [pid (:profile-id event)]
|
||||
(str "- profile-id: #uuid-" pid "\n")))
|
||||
[cfg {:keys [id public-uri] :as report}]
|
||||
(let [text (str "Exception: " public-uri "/dbg/error/" id " "
|
||||
(when-let [pid (:profile-id report)]
|
||||
(str "(pid: #uuid-" pid ")"))
|
||||
"\n"
|
||||
"```\n"
|
||||
"- host: `" (:host report) "`\n"
|
||||
"- tenant: `" (:tenant report) "`\n"
|
||||
"- version: `" (:version report) "`\n"
|
||||
"\n"
|
||||
"Trace:\n"
|
||||
(:trace report)
|
||||
"```")
|
||||
|
||||
resp (http/req! cfg
|
||||
{:uri (cf/get :error-report-webhook)
|
||||
:method :post
|
||||
|
|
@ -36,32 +47,41 @@
|
|||
(l/warn :hint "error on sending data"
|
||||
:response (pr-str resp)))))
|
||||
|
||||
(defn record->report
|
||||
[{:keys [::l/context ::l/id ::l/cause] :as record}]
|
||||
(us/assert! ::l/record record)
|
||||
{:id id
|
||||
:tenant (cf/get :tenant)
|
||||
:host (cf/get :host)
|
||||
:public-uri (cf/get :public-uri)
|
||||
:version (:full cf/version)
|
||||
:profile-id (:profile-id context)
|
||||
:trace (ex/format-throwable cause :detail? false :header? false)})
|
||||
|
||||
(defn handle-event
|
||||
[cfg event]
|
||||
[cfg record]
|
||||
(when @enabled
|
||||
(try
|
||||
(let [event (ldb/parse-event event)]
|
||||
(send-mattermost-notification! cfg event))
|
||||
(let [report (record->report record)]
|
||||
(send-mattermost-notification! cfg report))
|
||||
(catch Throwable cause
|
||||
(l/warn :hint "unhandled error"
|
||||
:cause cause)))))
|
||||
(l/warn :hint "unhandled error" :cause cause)))))
|
||||
|
||||
(defmethod ig/pre-init-spec ::reporter [_]
|
||||
(s/keys :req [::http/client
|
||||
::lzmq/receiver]))
|
||||
(s/keys :req [::http/client]))
|
||||
|
||||
(defmethod ig/init-key ::reporter
|
||||
[_ cfg]
|
||||
(when-let [uri (cf/get :error-report-webhook)]
|
||||
(px/thread
|
||||
{:name "penpot/mattermost-reporter"}
|
||||
(l/info :msg "initializing error reporter" :uri uri)
|
||||
(let [input (a/chan (a/sliding-buffer 128)
|
||||
(filter #(= (:logger/level %) "error")))]
|
||||
{:name "penpot/mattermost-reporter"
|
||||
:virtual true}
|
||||
(l/info :hint "initializing error reporter" :uri uri)
|
||||
(let [input (sp/chan (sp/sliding-buffer 128) (filter ldb/error-record?))]
|
||||
(add-watch l/log-record ::reporter #(sp/put! input %4))
|
||||
(try
|
||||
(lzmq/sub! (::lzmq/receiver cfg) input)
|
||||
(loop []
|
||||
(when-let [msg (a/<!! input)]
|
||||
(when-let [msg (sp/take! input)]
|
||||
(handle-event cfg msg)
|
||||
(recur)))
|
||||
(catch InterruptedException _
|
||||
|
|
@ -69,7 +89,8 @@
|
|||
(catch Throwable cause
|
||||
(l/error :hint "unexpected error" :cause cause))
|
||||
(finally
|
||||
(a/close! input)
|
||||
(sp/close! input)
|
||||
(remove-watch l/log-record ::reporter)
|
||||
(l/info :hint "reporter terminated")))))))
|
||||
|
||||
(defmethod ig/halt-key! ::reporter
|
||||
|
|
|
|||
|
|
@ -1,130 +0,0 @@
|
|||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC
|
||||
|
||||
(ns app.loggers.zmq
|
||||
"A generic ZMQ listener."
|
||||
(:require
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.config :as cf]
|
||||
[app.loggers.zmq.receiver :as-alias receiver]
|
||||
[app.util.json :as json]
|
||||
[app.util.time :as dt]
|
||||
[clojure.core.async :as a]
|
||||
[clojure.spec.alpha :as s]
|
||||
[cuerdas.core :as str]
|
||||
[integrant.core :as ig]
|
||||
[promesa.exec :as px])
|
||||
(:import
|
||||
org.zeromq.SocketType
|
||||
org.zeromq.ZMQ$Socket
|
||||
org.zeromq.ZContext))
|
||||
|
||||
(declare prepare)
|
||||
(declare start-rcv-loop)
|
||||
|
||||
(defmethod ig/init-key ::receiver
|
||||
[_ cfg]
|
||||
(let [uri (cf/get :loggers-zmq-uri)
|
||||
buffer (a/chan 1)
|
||||
output (a/chan 1 (comp (filter map?)
|
||||
(keep prepare)))
|
||||
mult (a/mult output)
|
||||
thread (when uri
|
||||
(px/thread
|
||||
{:name "penpot/zmq-receiver"
|
||||
:daemon false}
|
||||
(l/info :hint "receiver started")
|
||||
(try
|
||||
(start-rcv-loop buffer uri)
|
||||
(catch InterruptedException _
|
||||
(l/debug :hint "receiver interrupted"))
|
||||
(catch java.lang.IllegalStateException cause
|
||||
(if (= "errno 4" (ex-message cause))
|
||||
(l/debug :hint "receiver interrupted")
|
||||
(l/error :hint "unhandled error" :cause cause)))
|
||||
(catch Throwable cause
|
||||
(l/error :hint "unhandled error" :cause cause))
|
||||
(finally
|
||||
(l/info :hint "receiver terminated")))))]
|
||||
|
||||
(a/pipe buffer output)
|
||||
(-> cfg
|
||||
(assoc ::receiver/mult mult)
|
||||
(assoc ::receiver/thread thread)
|
||||
(assoc ::receiver/output output)
|
||||
(assoc ::receiver/buffer buffer))))
|
||||
|
||||
(s/def ::receiver/mult some?)
|
||||
(s/def ::receiver/thread #(instance? Thread %))
|
||||
(s/def ::receiver/output some?)
|
||||
(s/def ::receiver/buffer some?)
|
||||
(s/def ::receiver
|
||||
(s/keys :req [::receiver/mult
|
||||
::receiver/thread
|
||||
::receiver/output
|
||||
::receiver/buffer]))
|
||||
|
||||
(defn sub!
|
||||
[{:keys [::receiver/mult]} ch]
|
||||
(a/tap mult ch))
|
||||
|
||||
(defmethod ig/halt-key! ::receiver
|
||||
[_ {:keys [::receiver/buffer ::receiver/thread]}]
|
||||
(some-> thread px/interrupt!)
|
||||
(some-> buffer a/close!))
|
||||
|
||||
(def ^:private json-mapper
|
||||
(json/mapper
|
||||
{:encode-key-fn str/camel
|
||||
:decode-key-fn (comp keyword str/kebab)}))
|
||||
|
||||
(defn- start-rcv-loop
|
||||
[output endpoint]
|
||||
(let [zctx (ZContext. 1)
|
||||
socket (.. zctx (createSocket SocketType/SUB))]
|
||||
(try
|
||||
(.. socket (connect ^String endpoint))
|
||||
(.. socket (subscribe ""))
|
||||
(.. socket (setReceiveTimeOut 5000))
|
||||
(loop []
|
||||
(let [msg (.recv ^ZMQ$Socket socket)
|
||||
msg (ex/ignoring (json/decode msg json-mapper))
|
||||
msg (if (nil? msg) :empty msg)]
|
||||
(when (a/>!! output msg)
|
||||
(recur))))
|
||||
|
||||
(finally
|
||||
(.close ^java.lang.AutoCloseable socket)
|
||||
(.destroy ^ZContext zctx)))))
|
||||
|
||||
(s/def ::logger-name string?)
|
||||
(s/def ::level string?)
|
||||
(s/def ::thread string?)
|
||||
(s/def ::time-millis integer?)
|
||||
(s/def ::message string?)
|
||||
(s/def ::context-map map?)
|
||||
(s/def ::thrown map?)
|
||||
|
||||
(s/def ::log4j-event
|
||||
(s/keys :req-un [::logger-name ::level ::thread ::time-millis ::message]
|
||||
:opt-un [::context-map ::thrown]))
|
||||
|
||||
(defn- prepare
|
||||
[event]
|
||||
(if (s/valid? ::log4j-event event)
|
||||
(merge {:message (:message event)
|
||||
:created-at (dt/instant (:time-millis event))
|
||||
:logger/name (:logger-name event)
|
||||
:logger/level (str/lower (:level event))}
|
||||
|
||||
(when-let [trace (-> event :thrown :extended-stack-trace)]
|
||||
{:trace trace})
|
||||
|
||||
(:context-map event))
|
||||
(do
|
||||
(l/warn :hint "invalid event" :event event)
|
||||
nil)))
|
||||
|
|
@ -23,7 +23,6 @@
|
|||
[app.loggers.audit :as-alias audit]
|
||||
[app.loggers.audit.tasks :as-alias audit.tasks]
|
||||
[app.loggers.webhooks :as-alias webhooks]
|
||||
[app.loggers.zmq :as-alias lzmq]
|
||||
[app.metrics :as-alias mtx]
|
||||
[app.metrics.definition :as-alias mdef]
|
||||
[app.msgbus :as-alias mbus]
|
||||
|
|
@ -430,9 +429,6 @@
|
|||
{:pool (ig/ref ::db/pool)
|
||||
:key (cf/get :secret-key)}
|
||||
|
||||
::lzmq/receiver
|
||||
{}
|
||||
|
||||
::audit/collector
|
||||
{::db/pool (ig/ref ::db/pool)
|
||||
::wrk/executor (ig/ref ::wrk/executor)
|
||||
|
|
@ -454,17 +450,11 @@
|
|||
{::db/pool (ig/ref ::db/pool)
|
||||
::http.client/client (ig/ref ::http.client/client)}
|
||||
|
||||
:app.loggers.loki/reporter
|
||||
{::lzmq/receiver (ig/ref ::lzmq/receiver)
|
||||
::http.client/client (ig/ref ::http.client/client)}
|
||||
|
||||
:app.loggers.mattermost/reporter
|
||||
{::lzmq/receiver (ig/ref ::lzmq/receiver)
|
||||
::http.client/client (ig/ref ::http.client/client)}
|
||||
{::http.client/client (ig/ref ::http.client/client)}
|
||||
|
||||
:app.loggers.database/reporter
|
||||
{::lzmq/receiver (ig/ref :app.loggers.zmq/receiver)
|
||||
::db/pool (ig/ref ::db/pool)}
|
||||
{::db/pool (ig/ref ::db/pool)}
|
||||
|
||||
::sto/storage
|
||||
{:pool (ig/ref ::db/pool)
|
||||
|
|
|
|||
|
|
@ -308,6 +308,9 @@
|
|||
{:name "0100-mod-profile-indexes"
|
||||
:fn (mg/resource "app/migrations/sql/0100-mod-profile-indexes.sql")}
|
||||
|
||||
{:name "0101-mod-server-error-report-table"
|
||||
:fn (mg/resource "app/migrations/sql/0101-mod-server-error-report-table.sql")}
|
||||
|
||||
])
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
ALTER TABLE server_error_report
|
||||
ADD COLUMN version integer DEFAULT 1;
|
||||
|
|
@ -79,7 +79,7 @@
|
|||
|
||||
(us/verify! ::msgbus msgbus)
|
||||
|
||||
(set-error-handler! state #(l/error :cause % :hint "unexpected error on agent" ::l/async false))
|
||||
(set-error-handler! state #(l/error :cause % :hint "unexpected error on agent" ::l/sync? true))
|
||||
(set-error-mode! state :continue)
|
||||
(start-io-loop! msgbus)
|
||||
|
||||
|
|
@ -133,7 +133,7 @@
|
|||
[nsubs cfg topic chan]
|
||||
(let [nsubs (if (nil? nsubs) #{chan} (conj nsubs chan))]
|
||||
(when (= 1 (count nsubs))
|
||||
(l/trace :hint "open subscription" :topic topic ::l/async false)
|
||||
(l/trace :hint "open subscription" :topic topic ::l/sync? true)
|
||||
(redis-sub cfg topic))
|
||||
nsubs))
|
||||
|
||||
|
|
@ -144,7 +144,7 @@
|
|||
[nsubs cfg topic chan]
|
||||
(let [nsubs (disj nsubs chan)]
|
||||
(when (empty? nsubs)
|
||||
(l/trace :hint "close subscription" :topic topic ::l/async false)
|
||||
(l/trace :hint "close subscription" :topic topic ::l/sync? true)
|
||||
(redis-unsub cfg topic))
|
||||
nsubs))
|
||||
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@
|
|||
(p/mcat (partial handle-response request))
|
||||
(p/fnly (fn [response cause]
|
||||
(if cause
|
||||
(raise (ex/wrap-with-context cause {:profile-id profile-id}))
|
||||
(raise cause)
|
||||
(respond response)))))))
|
||||
|
||||
(defn- rpc-mutation-handler
|
||||
|
|
@ -117,7 +117,7 @@
|
|||
(p/mcat (partial handle-response request))
|
||||
(p/fnly (fn [response cause]
|
||||
(if cause
|
||||
(raise (ex/wrap-with-context cause {:profile-id profile-id}))
|
||||
(raise cause)
|
||||
(respond response)))))))
|
||||
|
||||
(defn- rpc-command-handler
|
||||
|
|
@ -144,7 +144,7 @@
|
|||
(p/mcat (partial handle-response request))
|
||||
(p/fnly (fn [response cause]
|
||||
(if cause
|
||||
(raise (ex/wrap-with-context cause {:profile-id profile-id}))
|
||||
(raise cause)
|
||||
(respond response))))))))
|
||||
|
||||
(defn- wrap-metrics
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@
|
|||
|
||||
(defn- capacity-exception?
|
||||
[o]
|
||||
(and (ex/ex-info? o)
|
||||
(and (ex/error? o)
|
||||
(let [data (ex-data o)]
|
||||
(and (= :bulkhead-error (:type data))
|
||||
(= :capacity-limit-reached (:code data))))))
|
||||
|
|
|
|||
|
|
@ -109,20 +109,20 @@
|
|||
|
||||
(defn write-byte!
|
||||
[^DataOutputStream output data]
|
||||
(l/trace :fn "write-byte!" :data data :position @*position* ::l/async false)
|
||||
(l/trace :fn "write-byte!" :data data :position @*position* ::l/sync? true)
|
||||
(.writeByte output (byte data))
|
||||
(swap! *position* inc))
|
||||
|
||||
(defn read-byte!
|
||||
[^DataInputStream input]
|
||||
(let [v (.readByte input)]
|
||||
(l/trace :fn "read-byte!" :val v :position @*position* ::l/async false)
|
||||
(l/trace :fn "read-byte!" :val v :position @*position* ::l/sync? true)
|
||||
(swap! *position* inc)
|
||||
v))
|
||||
|
||||
(defn write-long!
|
||||
[^DataOutputStream output data]
|
||||
(l/trace :fn "write-long!" :data data :position @*position* ::l/async false)
|
||||
(l/trace :fn "write-long!" :data data :position @*position* ::l/sync? true)
|
||||
(.writeLong output (long data))
|
||||
(swap! *position* + 8))
|
||||
|
||||
|
|
@ -130,14 +130,14 @@
|
|||
(defn read-long!
|
||||
[^DataInputStream input]
|
||||
(let [v (.readLong input)]
|
||||
(l/trace :fn "read-long!" :val v :position @*position* ::l/async false)
|
||||
(l/trace :fn "read-long!" :val v :position @*position* ::l/sync? true)
|
||||
(swap! *position* + 8)
|
||||
v))
|
||||
|
||||
(defn write-bytes!
|
||||
[^DataOutputStream output ^bytes data]
|
||||
(let [size (alength data)]
|
||||
(l/trace :fn "write-bytes!" :size size :position @*position* ::l/async false)
|
||||
(l/trace :fn "write-bytes!" :size size :position @*position* ::l/sync? true)
|
||||
(.write output data 0 size)
|
||||
(swap! *position* + size)))
|
||||
|
||||
|
|
@ -145,7 +145,7 @@
|
|||
[^InputStream input ^bytes buff]
|
||||
(let [size (alength buff)
|
||||
readed (.readNBytes input buff 0 size)]
|
||||
(l/trace :fn "read-bytes!" :expected (alength buff) :readed readed :position @*position* ::l/async false)
|
||||
(l/trace :fn "read-bytes!" :expected (alength buff) :readed readed :position @*position* ::l/sync? true)
|
||||
(swap! *position* + readed)
|
||||
readed))
|
||||
|
||||
|
|
@ -153,7 +153,7 @@
|
|||
|
||||
(defn write-uuid!
|
||||
[^DataOutputStream output id]
|
||||
(l/trace :fn "write-uuid!" :position @*position* :WRITTEN? (.size output) ::l/async false)
|
||||
(l/trace :fn "write-uuid!" :position @*position* :WRITTEN? (.size output) ::l/sync? true)
|
||||
|
||||
(doto output
|
||||
(write-byte! (get-mark :uuid))
|
||||
|
|
@ -162,7 +162,7 @@
|
|||
|
||||
(defn read-uuid!
|
||||
[^DataInputStream input]
|
||||
(l/trace :fn "read-uuid!" :position @*position* ::l/async false)
|
||||
(l/trace :fn "read-uuid!" :position @*position* ::l/sync? true)
|
||||
(let [m (read-byte! input)]
|
||||
(assert-mark m :uuid)
|
||||
(let [a (read-long! input)
|
||||
|
|
@ -171,7 +171,7 @@
|
|||
|
||||
(defn write-obj!
|
||||
[^DataOutputStream output data]
|
||||
(l/trace :fn "write-obj!" :position @*position* ::l/async false)
|
||||
(l/trace :fn "write-obj!" :position @*position* ::l/sync? true)
|
||||
(let [^bytes data (fres/encode data)]
|
||||
(doto output
|
||||
(write-byte! (get-mark :obj))
|
||||
|
|
@ -180,7 +180,7 @@
|
|||
|
||||
(defn read-obj!
|
||||
[^DataInputStream input]
|
||||
(l/trace :fn "read-obj!" :position @*position* ::l/async false)
|
||||
(l/trace :fn "read-obj!" :position @*position* ::l/sync? true)
|
||||
(let [m (read-byte! input)]
|
||||
(assert-mark m :obj)
|
||||
(let [size (read-long! input)]
|
||||
|
|
@ -191,14 +191,14 @@
|
|||
|
||||
(defn write-label!
|
||||
[^DataOutputStream output label]
|
||||
(l/trace :fn "write-label!" :label label :position @*position* ::l/async false)
|
||||
(l/trace :fn "write-label!" :label label :position @*position* ::l/sync? true)
|
||||
(doto output
|
||||
(write-byte! (get-mark :label))
|
||||
(write-obj! label)))
|
||||
|
||||
(defn read-label!
|
||||
[^DataInputStream input]
|
||||
(l/trace :fn "read-label!" :position @*position* ::l/async false)
|
||||
(l/trace :fn "read-label!" :position @*position* ::l/sync? true)
|
||||
(let [m (read-byte! input)]
|
||||
(assert-mark m :label)
|
||||
(read-obj! input)))
|
||||
|
|
@ -208,7 +208,7 @@
|
|||
(l/trace :fn "write-header!"
|
||||
:version version
|
||||
:position @*position*
|
||||
::l/async false)
|
||||
::l/sync? true)
|
||||
(let [vers (-> version name (subs 1) parse-long)
|
||||
output (io/data-output-stream output)]
|
||||
(doto output
|
||||
|
|
@ -218,7 +218,7 @@
|
|||
|
||||
(defn read-header!
|
||||
[^InputStream input]
|
||||
(l/trace :fn "read-header!" :position @*position* ::l/async false)
|
||||
(l/trace :fn "read-header!" :position @*position* ::l/sync? true)
|
||||
(let [input (io/data-input-stream input)
|
||||
mark (read-byte! input)
|
||||
mnum (read-long! input)
|
||||
|
|
@ -235,13 +235,13 @@
|
|||
(defn copy-stream!
|
||||
[^OutputStream output ^InputStream input ^long size]
|
||||
(let [written (io/copy! input output :size size)]
|
||||
(l/trace :fn "copy-stream!" :position @*position* :size size :written written ::l/async false)
|
||||
(l/trace :fn "copy-stream!" :position @*position* :size size :written written ::l/sync? true)
|
||||
(swap! *position* + written)
|
||||
written))
|
||||
|
||||
(defn write-stream!
|
||||
[^DataOutputStream output stream size]
|
||||
(l/trace :fn "write-stream!" :position @*position* ::l/async false :size size)
|
||||
(l/trace :fn "write-stream!" :position @*position* ::l/sync? true :size size)
|
||||
(doto output
|
||||
(write-byte! (get-mark :stream))
|
||||
(write-long! size))
|
||||
|
|
@ -250,7 +250,7 @@
|
|||
|
||||
(defn read-stream!
|
||||
[^DataInputStream input]
|
||||
(l/trace :fn "read-stream!" :position @*position* ::l/async false)
|
||||
(l/trace :fn "read-stream!" :position @*position* ::l/sync? true)
|
||||
(let [m (read-byte! input)
|
||||
s (read-long! input)
|
||||
p (tmp/tempfile :prefix "penpot.binfile.")]
|
||||
|
|
@ -264,7 +264,7 @@
|
|||
(if (> s temp-file-threshold)
|
||||
(with-open [^OutputStream output (io/output-stream p)]
|
||||
(let [readed (io/copy! input output :offset 0 :size s)]
|
||||
(l/trace :fn "read-stream*!" :expected s :readed readed :position @*position* ::l/async false)
|
||||
(l/trace :fn "read-stream*!" :expected s :readed readed :position @*position* ::l/sync? true)
|
||||
(swap! *position* + readed)
|
||||
[s p]))
|
||||
[s (io/read-as-bytes input :size s)])))
|
||||
|
|
@ -465,7 +465,7 @@
|
|||
(with-open [output (io/data-output-stream output)]
|
||||
(binding [*state* (volatile! {})]
|
||||
(run! (fn [section]
|
||||
(l/debug :hint "write section" :section section ::l/async false)
|
||||
(l/debug :hint "write section" :section section ::l/sync? true)
|
||||
(write-label! output section)
|
||||
(let [options (-> options
|
||||
(assoc ::output output)
|
||||
|
|
@ -499,7 +499,7 @@
|
|||
(l/debug :hint "write penpot file"
|
||||
:id file-id
|
||||
:media (count media)
|
||||
::l/async false)
|
||||
::l/sync? true)
|
||||
|
||||
(doto output
|
||||
(write-obj! file)
|
||||
|
|
@ -511,7 +511,7 @@
|
|||
[{:keys [::db/pool ::output ::include-libraries?]}]
|
||||
(let [rels (when include-libraries?
|
||||
(retrieve-library-relations pool (-> *state* deref :files)))]
|
||||
(l/debug :hint "found rels" :total (count rels) ::l/async false)
|
||||
(l/debug :hint "found rels" :total (count rels) ::l/sync? true)
|
||||
(write-obj! output rels)))
|
||||
|
||||
(defmethod write-section :v1/sobjects
|
||||
|
|
@ -520,14 +520,14 @@
|
|||
storage (media/configure-assets-storage storage)]
|
||||
(l/debug :hint "found sobjects"
|
||||
:items (count sids)
|
||||
::l/async false)
|
||||
::l/sync? true)
|
||||
|
||||
;; Write all collected storage objects
|
||||
(write-obj! output sids)
|
||||
|
||||
(doseq [id sids]
|
||||
(let [{:keys [size] :as obj} @(sto/get-object storage id)]
|
||||
(l/debug :hint "write sobject" :id id ::l/async false)
|
||||
(l/debug :hint "write sobject" :id id ::l/sync? true)
|
||||
(doto output
|
||||
(write-uuid! id)
|
||||
(write-obj! (meta obj)))
|
||||
|
|
@ -587,7 +587,7 @@
|
|||
(db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED;"])
|
||||
(binding [*state* (volatile! {:media [] :index {}})]
|
||||
(run! (fn [section]
|
||||
(l/debug :hint "reading section" :section section ::l/async false)
|
||||
(l/debug :hint "reading section" :section section ::l/sync? true)
|
||||
(assert-read-label! input section)
|
||||
(let [options (-> options
|
||||
(assoc ::section section)
|
||||
|
|
@ -605,7 +605,7 @@
|
|||
(defmethod read-section :v1/metadata
|
||||
[{:keys [::input]}]
|
||||
(let [{:keys [version files]} (read-obj! input)]
|
||||
(l/debug :hint "metadata readed" :version (:full version) :files files ::l/async false)
|
||||
(l/debug :hint "metadata readed" :version (:full version) :files files ::l/sync? true)
|
||||
(vswap! *state* update :index update-index files)
|
||||
(vswap! *state* assoc :version version :files files)))
|
||||
|
||||
|
|
@ -633,14 +633,14 @@
|
|||
:hint "the penpot file seems corrupt, found unexpected uuid (file-id)"))
|
||||
|
||||
;; Update index using with media
|
||||
(l/debug :hint "update index with media" ::l/async false)
|
||||
(l/debug :hint "update index with media" ::l/sync? true)
|
||||
(vswap! *state* update :index update-index (map :id media'))
|
||||
|
||||
;; Store file media for later insertion
|
||||
(l/debug :hint "update media references" ::l/async false)
|
||||
(l/debug :hint "update media references" ::l/sync? true)
|
||||
(vswap! *state* update :media into (map #(update % :id lookup-index)) media')
|
||||
|
||||
(l/debug :hint "processing file" :file-id file-id ::features features ::l/async false)
|
||||
(l/debug :hint "processing file" :file-id file-id ::features features ::l/sync? true)
|
||||
|
||||
(binding [ffeat/*current* features
|
||||
ffeat/*wrap-with-objects-map-fn* (if (features "storage/objects-map") omap/wrap identity)
|
||||
|
|
@ -666,7 +666,7 @@
|
|||
:created-at timestamp
|
||||
:modified-at timestamp}]
|
||||
|
||||
(l/debug :hint "create file" :id file-id' ::l/async false)
|
||||
(l/debug :hint "create file" :id file-id' ::l/sync? true)
|
||||
|
||||
(if overwrite?
|
||||
(create-or-update-file conn params)
|
||||
|
|
@ -689,7 +689,7 @@
|
|||
(l/debug :hint "create file library link"
|
||||
:file-id (:file-id rel)
|
||||
:lib-id (:library-file-id rel)
|
||||
::l/async false)
|
||||
::l/sync? true)
|
||||
(db/insert! conn :file-library-rel rel)))))
|
||||
|
||||
(defmethod read-section :v1/sobjects
|
||||
|
|
@ -706,7 +706,7 @@
|
|||
:code :inconsistent-penpot-file
|
||||
:hint "the penpot file seems corrupt, found unexpected uuid (storage-object-id)"))
|
||||
|
||||
(l/debug :hint "readed storage object" :id id ::l/async false)
|
||||
(l/debug :hint "readed storage object" :id id ::l/sync? true)
|
||||
|
||||
(let [[size resource] (read-stream! input)
|
||||
hash (sto/calculate-hash resource)
|
||||
|
|
@ -720,18 +720,18 @@
|
|||
|
||||
sobject @(sto/put-object! storage params)]
|
||||
|
||||
(l/debug :hint "persisted storage object" :id id :new-id (:id sobject) ::l/async false)
|
||||
(l/debug :hint "persisted storage object" :id id :new-id (:id sobject) ::l/sync? true)
|
||||
(vswap! *state* update :index assoc id (:id sobject)))))
|
||||
|
||||
(doseq [item (:media @*state*)]
|
||||
(l/debug :hint "inserting file media object"
|
||||
:id (:id item)
|
||||
:file-id (:file-id item)
|
||||
::l/async false)
|
||||
::l/sync? true)
|
||||
|
||||
(let [file-id (lookup-index (:file-id item))]
|
||||
(if (= file-id (:file-id item))
|
||||
(l/warn :hint "ignoring file media object" :file-id (:file-id item) ::l/async false)
|
||||
(l/warn :hint "ignoring file media object" :file-id (:file-id item) ::l/sync? true)
|
||||
(db/insert! conn :file-media-object
|
||||
(-> item
|
||||
(assoc :file-id file-id)
|
||||
|
|
@ -742,7 +742,7 @@
|
|||
(defn- lookup-index
|
||||
[id]
|
||||
(let [val (get-in @*state* [:index id])]
|
||||
(l/trace :fn "lookup-index" :id id :val val ::l/async false)
|
||||
(l/trace :fn "lookup-index" :id id :val val ::l/sync? true)
|
||||
(when (and (not (::ignore-index-errors? *options*)) (not val))
|
||||
(ex/raise :type :validation
|
||||
:code :incomplete-index
|
||||
|
|
@ -755,7 +755,7 @@
|
|||
index index]
|
||||
(if-let [id (first items)]
|
||||
(let [new-id (if (::overwrite? *options*) id (uuid/next))]
|
||||
(l/trace :fn "update-index" :id id :new-id new-id ::l/async false)
|
||||
(l/trace :fn "update-index" :id id :new-id new-id ::l/sync? true)
|
||||
(recur (rest items)
|
||||
(assoc index id new-id)))
|
||||
index)))
|
||||
|
|
@ -803,7 +803,7 @@
|
|||
(try
|
||||
(process-map-form form)
|
||||
(catch Throwable cause
|
||||
(l/warn :hint "failed form" :form (pr-str form) ::l/async false)
|
||||
(l/warn :hint "failed form" :form (pr-str form) ::l/sync? true)
|
||||
(throw cause)))
|
||||
form))
|
||||
data)))
|
||||
|
|
|
|||
|
|
@ -41,7 +41,6 @@
|
|||
(s/def ::project-id ::us/uuid)
|
||||
(s/def ::style valid-style)
|
||||
(s/def ::team-id ::us/uuid)
|
||||
(s/def ::team-id ::us/uuid)
|
||||
(s/def ::weight valid-weight)
|
||||
|
||||
;; --- QUERY: Get font variants
|
||||
|
|
|
|||
|
|
@ -363,7 +363,7 @@
|
|||
(let [state (read-config path)]
|
||||
(l/info :hint "config refreshed"
|
||||
:loaded-limits (count (::limits state))
|
||||
::l/async false)
|
||||
::l/sync? true)
|
||||
state)))))
|
||||
|
||||
(schedule-next [state]
|
||||
|
|
@ -380,10 +380,10 @@
|
|||
(when-not (instance? java.util.concurrent.RejectedExecutionException cause)
|
||||
(if-let [explain (-> cause ex-data ex/explain)]
|
||||
(l/warn ::l/raw (str "unable to refresh config, invalid format:\n" explain)
|
||||
::l/async false)
|
||||
::l/sync? true)
|
||||
(l/warn :hint "unexpected exception on loading config"
|
||||
:cause cause
|
||||
::l/async false))))
|
||||
::l/sync? true))))
|
||||
|
||||
(defn- get-config-path
|
||||
[]
|
||||
|
|
|
|||
|
|
@ -242,7 +242,7 @@
|
|||
(let [result (a/<! (handler wsp v))]
|
||||
;; (l/trace :hint "message received" :message v)
|
||||
(cond
|
||||
(ex/ex-info? result)
|
||||
(ex/error? result)
|
||||
(a/>! output-ch {:type :error :error (ex-data result)})
|
||||
|
||||
(ex/exception? result)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue