3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-07-12 08:47:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

✨ Backport changes from develop.

Browse source
This commit is contained in:
alonso.torres 2021-12-30 11:19:46 +01:00 • committed by Andrey Antukh
parent dea090e7d3
commit 71734df489
17 changed files with 420 additions and 207 deletions
Download patch file Download diff file Expand all files Collapse all files

10
backend/src/app/config.clj
View file

@ -51,6 +51,9 @@
:default-blob-version 3
:loggers-zmq-uri "tcp://localhost:45556"
:file-change-snapshot-every 5
:file-change-snapshot-timeout "3h"
:public-uri "http://localhost:3449"
:redis-uri "redis://redis/0"
@ -98,6 +101,10 @@
(s/def ::audit-log-archive-uri ::us/string)
(s/def ::audit-log-gc-max-age ::dt/duration)
(s/def ::admins ::us/set-of-str)
(s/def ::file-change-snapshot-every ::us/integer)
(s/def ::file-change-snapshot-timeout ::dt/duration)
(s/def ::secret-key ::us/string)
(s/def ::allow-demo-users ::us/boolean)
(s/def ::assets-path ::us/string)
@ -185,6 +192,7 @@
(s/def ::config
(s/keys :opt-un [::secret-key
::flags
::admins
::allow-demo-users
::audit-log-archive-uri
::audit-log-gc-max-age
@ -193,6 +201,8 @@
::database-username
::default-blob-version
::error-report-webhook
::file-change-snapshot-every
::file-change-snapshot-timeout
::user-feedback-destination
::github-client-id
::github-client-secret

21
backend/src/app/http.clj
View file

@ -12,6 +12,7 @@
[app.common.spec :as us]
[app.http.doc :as doc]
[app.http.errors :as errors]
[app.http.debug :as debug]
[app.http.middleware :as middleware]
[app.metrics :as mtx]
[clojure.spec.alpha :as s]
@ -104,17 +105,16 @@
(s/def ::storage map?)
(s/def ::assets map?)
(s/def ::feedback fn?)
(s/def ::error-report-handler fn?)
(s/def ::audit-http-handler fn?)
(s/def ::debug map?)
(defmethod ig/pre-init-spec ::router [_]
(s/keys :req-un [::rpc ::session ::mtx/metrics
::oauth ::storage ::assets ::feedback
::error-report-handler
::audit-http-handler]))
::debug ::audit-http-handler]))
(defmethod ig/init-key ::router
[_ {:keys [session rpc oauth metrics assets feedback] :as cfg}]
[_ {:keys [session rpc oauth metrics assets feedback debug] :as cfg}]
(rr/router
[["/metrics" {:get (:handler metrics)}]
["/assets" {:middleware [[middleware/format-response-body]
@ -125,8 +125,17 @@
["/by-file-media-id/:id" {:get (:file-objects-handler assets)}]
["/by-file-media-id/:id/thumbnail" {:get (:file-thumbnails-handler assets)}]]
["/dbg"
["/error-by-id/:id" {:get (:error-report-handler cfg)}]]
["/dbg" {:middleware [[middleware/params]
[middleware/keyword-params]
[middleware/format-response-body]
[middleware/errors errors/handle]
[middleware/cookies]
[(:middleware session)]]}
["/error-by-id/:id" {:get (:retrieve-error debug)}]
["/error/:id" {:get (:retrieve-error debug)}]
["/error" {:get (:retrieve-error-list debug)}]
["/file/data/:id" {:get (:retrieve-file-data debug)}]
["/file/changes/:id" {:get (:retrieve-file-changes debug)}]]
["/webhooks"
["/sns" {:post (:sns-webhook cfg)}]]

165
backend/src/app/http/debug.clj Normal file
View file

@ -0,0 +1,165 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) UXBOX Labs SL
(ns app.http.debug
(:require
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.common.transit :as t]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.rpc.queries.profile :as profile]
[app.util.blob :as blob]
[app.util.json :as json]
[app.util.template :as tmpl]
[app.util.time :as dt]
[clojure.java.io :as io]
[clojure.pprint :as ppr]
[clojure.spec.alpha :as s]
[cuerdas.core :as str]
[integrant.core :as ig]))
(def sql:retrieve-range-of-changes
"select revn, changes from file_change where file_id=? and revn >= ? and revn <= ? order by revn")
(def sql:retrieve-single-change
"select revn, changes, data from file_change where file_id=? and revn = ?")
(defn authorized?
[pool {:keys [profile-id]}]
(or (= "devenv" (cf/get :host))
(let [profile (ex/ignoring (profile/retrieve-profile-data pool profile-id))
admins (or (cf/get :admins) #{})]
(contains? admins (:email profile)))))
(defn prepare-response
[body]
(when-not body
(ex/raise :type :not-found
:code :enpty-data
:hint "empty response"))
{:status 200
:headers {"content-type" "application/transit+json"}
:body body})
(defn retrieve-file-data
[{:keys [pool]} request]
(when-not (authorized? pool request)
(ex/raise :type :authentication
:code :only-admins-allowed))
(let [id (some-> (get-in request [:path-params :id]) uuid/uuid)
revn (some-> (get-in request [:params :revn]) d/parse-integer)]
(when-not id
(ex/raise :type :validation
:code :missing-arguments))
(if (integer? revn)
(let [fchange (db/exec-one! pool [sql:retrieve-single-change id revn])]
(prepare-response (some-> fchange :data blob/decode)))
(let [file (db/get-by-id pool :file id)]
(prepare-response (some-> file :data blob/decode))))))
(defn retrieve-file-changes
[{:keys [pool]} {:keys [params path-params profile-id] :as request}]
(when-not (authorized? pool request)
(ex/raise :type :authentication
:code :only-admins-allowed))
(let [id (some-> (get-in request [:path-params :id]) uuid/uuid)
revn (get-in request [:params :revn] "latest")]
(when (or (not id) (not revn))
(ex/raise :type :validation
:code :invalid-arguments
:hint "missing arguments"))
(cond
(d/num-string? revn)
(let [item (db/exec-one! pool [sql:retrieve-single-change id (d/parse-integer revn)])]
(prepare-response (some-> item :changes blob/decode vec)))
(str/includes? revn ":")
(let [[start end] (->> (str/split revn #":")
(map str/trim)
(map d/parse-integer))
items (db/exec! pool [sql:retrieve-range-of-changes id start end])]
(prepare-response (some->> items
(map :changes)
(map blob/decode)
(mapcat identity)
(vec))))
:else
(ex/raise :type :validation :code :invalid-arguments))))
(defn retrieve-error
[{:keys [pool]} request]
(letfn [(parse-id [request]
(let [id (get-in request [:path-params :id])
id (us/uuid-conformer id)]
(when (uuid? id)
id)))
(retrieve-report [id]
(ex/ignoring
(some-> (db/get-by-id pool :server-error-report id) :content db/decode-transit-pgobject)))
(render-template [report]
(binding [ppr/*print-right-margin* 300]
(let [context (dissoc report :trace :cause :params :data :spec-prob :spec-problems)
params {:context (with-out-str (ppr/pprint context))
:data (:data report)
:trace (or (:cause report)
(:trace report)
(some-> report :error :trace))
:params (:params report)}]
(-> (io/resource "error-report.tmpl")
(tmpl/render params)))))
]
(when-not (authorized? pool request)
(ex/raise :type :authentication
:code :only-admins-allowed))
(let [result (some-> (parse-id request)
(retrieve-report)
(render-template))]
(if result
{:status 200
:headers {"content-type" "text/html; charset=utf-8"
"x-robots-tag" "noindex"}
:body result}
{:status 404
:body "not found"}))))
(def sql:error-reports
"select id, created_at from server_error_report order by created_at desc limit 100")
(defn retrieve-error-list
[{:keys [pool]} request]
(when-not (authorized? pool request)
(ex/raise :type :authentication
:code :only-admins-allowed))
(let [items (db/exec! pool [sql:error-reports])
items (map #(update % :created-at dt/format-instant :rfc1123) items)]
{:status 200
:headers {"content-type" "text/html; charset=utf-8"
"x-robots-tag" "noindex"}
:body (-> (io/resource "error-list.tmpl")
(tmpl/render {:items items}))}))
(defmethod ig/init-key ::handlers
[_ {:keys [pool] :as cfg}]
{:retrieve-file-data (partial retrieve-file-data cfg)
:retrieve-file-changes (partial retrieve-file-changes cfg)
:retrieve-error (partial retrieve-error cfg)
:retrieve-error-list (partial retrieve-error-list cfg)})

20
backend/src/app/http/errors.clj
View file

@ -30,14 +30,13 @@
:hint (or (:hint data) (ex-message error))
:params (l/stringify-data (:params request))
:spec-problems (some-> data ::s/problems)
:data (some-> data (dissoc ::s/problems))
:ip-addr (parse-client-ip request)
:profile-id (:profile-id request)}
(let [headers (:headers request)]
{:user-agent (get headers "user-agent")
:frontend-version (get headers "x-frontend-version" "unknown")})
(dissoc data ::s/problems))))
:frontend-version (get headers "x-frontend-version" "unknown")}))))
(defmulti handle-exception
(fn [err & _rest]
@ -54,18 +53,9 @@
{:status 400 :body (ex-data err)})
(defmethod handle-exception :validation
[err req]
(let [header (get-in req [:headers "accept"])
edata (ex-data err)]
(if (and (= :spec-validation (:code edata))
(str/starts-with? header "text/html"))
{:status 400
:headers {"content-type" "text/html"}
:body (str "<pre style='font-size:16px'>"
(:explain edata)
"</pre>\n")}
{:status 400
:body (dissoc edata ::s/problems)})))
[err _]
(let [edata (ex-data err)]
{:status 400 :body (dissoc edata ::s/problems)}))
(defmethod handle-exception :assertion
[error request]

1
backend/src/app/http/session.clj
View file

@ -73,6 +73,7 @@
(if-let [{:keys [id profile-id] :as session} (retrieve-from-request cfg request)]
(do
(a/>!! (::events-ch cfg) id)
(l/set-context! {:profile-id profile-id})
(handler (assoc request :profile-id profile-id)))
(handler request))))

44
backend/src/app/loggers/database.clj
View file

@ -14,10 +14,8 @@
[app.config :as cf]
[app.db :as db]
[app.util.async :as aa]
[app.util.template :as tmpl]
[app.worker :as wrk]
[clojure.core.async :as a]
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[cuerdas.core :as str]
[integrant.core :as ig]))
@ -60,8 +58,10 @@
[{:keys [executor] :as cfg} event]
(aa/with-thread executor
(try
(let [event (parse-event event)]
(l/debug :hint "registering error on database" :id (:id event))
(let [event (parse-event event)
uri (cf/get :public-uri)]
(l/debug :hint "registering error on database" :id (:id event)
:uri (str uri "/dbg/error/" (:id event)))
(persist-on-database! cfg event))
(catch Exception e
(l/warn :hint "unexpected exception on database error logger"
@ -89,39 +89,3 @@
(defmethod ig/halt-key! ::reporter
[_ output]
(a/close! output))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Http Handler
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defmethod ig/pre-init-spec ::handler [_]
(s/keys :req-un [::db/pool]))
(defmethod ig/init-key ::handler
[_ {:keys [pool] :as cfg}]
(letfn [(parse-id [request]
(let [id (get-in request [:path-params :id])
id (us/uuid-conformer id)]
(when (uuid? id)
id)))
(retrieve-report [id]
(ex/ignoring
(when-let [{:keys [content] :as row} (db/get-by-id pool :server-error-report id)]
(assoc row :content (db/decode-transit-pgobject content)))))
(render-template [{:keys [content] :as report}]
(some-> (io/resource "error-report.tmpl")
(tmpl/render content)))]
(fn [request]
(let [result (some-> (parse-id request)
(retrieve-report)
(render-template))]
(if result
{:status 200
:headers {"content-type" "text/html; charset=utf-8"
"x-robots-tag" "noindex"}
:body result}
{:status 404
:body "not found"})))))

2
backend/src/app/loggers/mattermost.clj
View file

@ -25,7 +25,7 @@
[cfg {:keys [host id public-uri] :as event}]
(try
(let [uri (:uri cfg)
text (str "Exception on (host: " host ", url: " public-uri "/dbg/error-by-id/" id ")\n"
text (str "Exception on (host: " host ", url: " public-uri "/dbg/error/" id ")\n"
(when-let [pid (:profile-id event)]
(str "- profile-id: #uuid-" pid "\n")))
rsp (http/send! {:uri uri

10
backend/src/app/main.clj
View file

@ -106,8 +106,11 @@
:storage (ig/ref :app.storage/storage)
:sns-webhook (ig/ref :app.http.awsns/handler)
:feedback (ig/ref :app.http.feedback/handler)
:audit-http-handler (ig/ref :app.loggers.audit/http-handler)
:error-report-handler (ig/ref :app.loggers.database/handler)}
:debug (ig/ref :app.http.debug/handlers)
:audit-http-handler (ig/ref :app.loggers.audit/http-handler)}
:app.http.debug/handlers
{:pool (ig/ref :app.db/pool)}
:app.http.assets/handlers
{:metrics (ig/ref :app.metrics/metrics)
@ -306,9 +309,6 @@
:pool (ig/ref :app.db/pool)
:executor (ig/ref :app.worker/executor)}
:app.loggers.database/handler
{:pool (ig/ref :app.db/pool)}
:app.loggers.sentry/reporter
{:dsn (cf/get :sentry-dsn)
:trace-sample-rate (cf/get :sentry-trace-sample-rate 1.0)

15
backend/src/app/rpc/mutations/files.clj
View file

@ -11,6 +11,7 @@
[app.common.pages.migrations :as pmg]
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.metrics :as mtx]
[app.rpc.permissions :as perms]
@ -280,11 +281,13 @@
(defn- take-snapshot?
"Defines the rule when file `data` snapshot should be saved."
[{:keys [revn modified-at] :as file}]
;; The snapshot will be saved every 20 changes or if the last
;; modification is older than 3 hour.
(or (zero? (mod revn 20))
(> (inst-ms (dt/diff modified-at (dt/now)))
(inst-ms (dt/duration {:hours 3})))))
(let [freq (or (cf/get :file-change-snapshot-every) 20)
timeout (or (cf/get :file-change-snapshot-timeout)
(dt/duration {:hours 1}))]
(or (= 1 freq)
(zero? (mod revn freq))
(> (inst-ms (dt/diff modified-at (dt/now)))
(inst-ms timeout)))))
(defn- delete-from-storage
[{:keys [storage] :as cfg} file]
@ -309,6 +312,8 @@
(mapcat :changes changes-with-metadata)
changes)
changes (vec changes)
;; Trace the number of changes processed
_ ((::mtx/fn mtx1) {:by (count changes)})