3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-05-03 00:45:53 +02:00
Code Issues Projects Releases Packages Wiki Activity

♻️ Minor code reorganization.

Browse source 
Improves modularity and reusability and allows usage
of backend code as a library.
This commit is contained in:
Andrey Antukh 2021-03-30 14:55:19 +02:00 • committed by Alonso Torres
parent 59a45530a8
commit 0926fbcbc6
27 changed files with 704 additions and 791 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
backend/dev/user.clj
View file

@ -70,7 +70,7 @@
[] []
(alter-var-root #'system (fn [sys] (alter-var-root #'system (fn [sys]
(when sys (ig/halt! sys)) (when sys (ig/halt! sys))
(-> (main/build-system-config cfg/config) (-> main/system-config
(ig/prep) (ig/prep)
(ig/init)))) (ig/init))))
:started) :started)

3
backend/src/app/cli/fixtures.clj
View file

@ -12,7 +12,6 @@
(:require (:require
[app.common.pages :as cp] [app.common.pages :as cp]
[app.common.uuid :as uuid] [app.common.uuid :as uuid]
[app.config :as cfg]
[app.db :as db] [app.db :as db]
[app.main :as main] [app.main :as main]
[app.rpc.mutations.profile :as profile] [app.rpc.mutations.profile :as profile]
@ -233,7 +232,7 @@
(defn run (defn run
[{:keys [preset] :or {preset :small}}] [{:keys [preset] :or {preset :small}}]
(let [config (select-keys (main/build-system-config cfg/config) (let [config (select-keys main/system-config
[:app.db/pool [:app.db/pool
:app.telemetry/migrations :app.telemetry/migrations
:app.migrations/migrations :app.migrations/migrations

5
backend/src/app/cli/manage.clj
View file

@ -5,12 +5,11 @@
;; This Source Code Form is "Incompatible With Secondary Licenses", as ;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0. ;; defined by the Mozilla Public License, v. 2.0.
;; ;;
;; Copyright (c) 2021 UXBOX Labs SL ;; Copyright (c) UXBOX Labs SL
(ns app.cli.manage (ns app.cli.manage
"A manage cli api." "A manage cli api."
(:require (:require
[app.config :as cfg]
[app.db :as db] [app.db :as db]
[app.main :as main] [app.main :as main]
[app.rpc.mutations.profile :as profile] [app.rpc.mutations.profile :as profile]
@ -26,7 +25,7 @@
(defn init-system (defn init-system
[] []
(let [data (-> (main/build-system-config cfg/config) (let [data (-> main/system-config
(select-keys [:app.db/pool :app.metrics/metrics]) (select-keys [:app.db/pool :app.metrics/metrics])
(assoc :app.migrations/all {}))] (assoc :app.migrations/all {}))]
(-> data ig/prep ig/init))) (-> data ig/prep ig/init)))

10
backend/src/app/cli/migrate_media.clj
View file

@ -10,7 +10,7 @@
(ns app.cli.migrate-media (ns app.cli.migrate-media
(:require (:require
[app.common.media :as cm] [app.common.media :as cm]
[app.config :as cfg] [app.config :as cf]
[app.db :as db] [app.db :as db]
[app.main :as main] [app.main :as main]
[app.storage :as sto] [app.storage :as sto]
@ -34,7 +34,7 @@
(defn run (defn run
[] []
(let [config (select-keys (main/build-system-config cfg/config) (let [config (select-keys main/system-config
[:app.db/pool [:app.db/pool
:app.migrations/migrations :app.migrations/migrations
:app.metrics/metrics :app.metrics/metrics
@ -60,7 +60,7 @@
(->> (db/exec! conn ["select * from profile"]) (->> (db/exec! conn ["select * from profile"])
(filter #(not (str/empty? (:photo %)))) (filter #(not (str/empty? (:photo %))))
(seq)))] (seq)))]
(let [base (fs/path (:storage-fs-old-directory cfg/config)) (let [base (fs/path (cf/get :storage-fs-old-directory))
storage (-> (:app.storage/storage system) storage (-> (:app.storage/storage system)
(assoc :conn conn))] (assoc :conn conn))]
(doseq [profile (retrieve-profiles conn)] (doseq [profile (retrieve-profiles conn)]
@ -81,7 +81,7 @@
(->> (db/exec! conn ["select * from team"]) (->> (db/exec! conn ["select * from team"])
(filter #(not (str/empty? (:photo %)))) (filter #(not (str/empty? (:photo %))))
(seq)))] (seq)))]
(let [base (fs/path (:storage-fs-old-directory cfg/config)) (let [base (fs/path (cf/get :storage-fs-old-directory))
storage (-> (:app.storage/storage system) storage (-> (:app.storage/storage system)
(assoc :conn conn))] (assoc :conn conn))]
(doseq [team (retrieve-teams conn)] (doseq [team (retrieve-teams conn)]
@ -105,7 +105,7 @@
from file_media_object as fmo from file_media_object as fmo
join file_media_thumbnail as fth on (fth.media_object_id = fmo.id)"]) join file_media_thumbnail as fth on (fth.media_object_id = fmo.id)"])
(seq)))] (seq)))]
(let [base (fs/path (:storage-fs-old-directory cfg/config)) (let [base (fs/path (cf/get :storage-fs-old-directory))
storage (-> (:app.storage/storage system) storage (-> (:app.storage/storage system)
(assoc :conn conn))] (assoc :conn conn))]
(doseq [mobj (retrieve-media-objects conn)] (doseq [mobj (retrieve-media-objects conn)]

50
backend/src/app/config.clj
View file

@ -5,7 +5,7 @@
;; This Source Code Form is "Incompatible With Secondary Licenses", as ;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0. ;; defined by the Mozilla Public License, v. 2.0.
;; ;;
;; Copyright (c) 2020-2021 UXBOX Labs SL ;; Copyright (c) UXBOX Labs SL
(ns app.config (ns app.config
"A configuration management." "A configuration management."
@ -15,10 +15,19 @@
[app.common.version :as v] [app.common.version :as v]
[app.util.time :as dt] [app.util.time :as dt]
[clojure.core :as c] [clojure.core :as c]
[clojure.pprint :as pprint]
[clojure.spec.alpha :as s] [clojure.spec.alpha :as s]
[cuerdas.core :as str] [cuerdas.core :as str]
[environ.core :refer [env]])) [environ.core :refer [env]]))
(prefer-method print-method
clojure.lang.IRecord
clojure.lang.IDeref)
(prefer-method pprint/simple-dispatch
clojure.lang.IPersistentMap
clojure.lang.IDeref)
(def defaults (def defaults
{:http-server-port 6060 {:http-server-port 6060
:host "devenv" :host "devenv"
@ -221,39 +230,31 @@
::telemetry-server-enabled ::telemetry-server-enabled
::telemetry-server-port ::telemetry-server-port
::telemetry-uri ::telemetry-uri
::telemetry-referer
::telemetry-with-taiga ::telemetry-with-taiga
::tenant])) ::tenant]))
(defn- env->config (defn read-env
[env] [prefix]
(let [prefix (str prefix "-")
len (count prefix)]
(reduce-kv (reduce-kv
(fn [acc k v] (fn [acc k v]
(cond-> acc (cond-> acc
(str/starts-with? (name k) "penpot-") (str/starts-with? (name k) prefix)
(assoc (keyword (subs (name k) 7)) v) (assoc (keyword (subs (name k) len)) v)))
(str/starts-with? (name k) "app-")
(assoc (keyword (subs (name k) 4)) v)))
{} {}
env)) env)))
(defn- read-config (defn- read-config
[env] []
(->> (env->config env) (->> (read-env "penpot")
(merge defaults) (merge defaults)
(us/conform ::config))) (us/conform ::config)))
(defn- read-test-config
[env]
(merge {:redis-uri "redis://redis/1"
:database-uri "postgresql://postgres/penpot_test"
:storage-fs-directory "/tmp/app/storage"
:migrations-verbose false}
(read-config env)))
(def version (v/parse "%version%")) (def version (v/parse "%version%"))
(def config (read-config env)) (def config (atom (read-config)))
(def test-config (read-test-config env))
(def deletion-delay (def deletion-delay
(dt/duration {:days 7})) (dt/duration {:days 7}))
@ -261,6 +262,9 @@
(defn get (defn get
"A configuration getter. Helps code be more testable." "A configuration getter. Helps code be more testable."
([key] ([key]
(c/get config key)) (c/get @config key))
([key default] ([key default]
(c/get config key default))) (c/get @config key default)))
;; Set value for all new threads bindings.
(alter-var-root #'*assert* (constantly (get :asserts-enabled)))

11
backend/src/app/db.clj
View file

@ -9,6 +9,7 @@
(ns app.db (ns app.db
(:require (:require
[app.common.data :as d]
[app.common.exceptions :as ex] [app.common.exceptions :as ex]
[app.common.geom.point :as gpt] [app.common.geom.point :as gpt]
[app.common.spec :as us] [app.common.spec :as us]
@ -48,8 +49,8 @@
(declare instrument-jdbc!) (declare instrument-jdbc!)
(s/def ::name keyword?)
(s/def ::uri ::us/not-empty-string) (s/def ::uri ::us/not-empty-string)
(s/def ::name ::us/not-empty-string)
(s/def ::min-pool-size ::us/integer) (s/def ::min-pool-size ::us/integer)
(s/def ::max-pool-size ::us/integer) (s/def ::max-pool-size ::us/integer)
(s/def ::migrations map?) (s/def ::migrations map?)
@ -59,14 +60,14 @@
(defmethod ig/init-key ::pool (defmethod ig/init-key ::pool
[_ {:keys [migrations metrics] :as cfg}] [_ {:keys [migrations metrics] :as cfg}]
(log/infof "initialize connection pool '%s' with uri '%s'" (:name cfg) (:uri cfg)) (log/infof "initialize connection pool '%s' with uri '%s'" (name (:name cfg)) (:uri cfg))
(instrument-jdbc! (:registry metrics)) (instrument-jdbc! (:registry metrics))
(let [pool (create-pool cfg)] (let [pool (create-pool cfg)]
(when (seq migrations) (when (seq migrations)
(with-open [conn ^AutoCloseable (open pool)] (with-open [conn ^AutoCloseable (open pool)]
(mg/setup! conn) (mg/setup! conn)
(doseq [[mname steps] migrations] (doseq [[name steps] migrations]
(mg/migrate! conn {:name (name mname) :steps steps})))) (mg/migrate! conn {:name (d/name name) :steps steps}))))
pool)) pool))
(defmethod ig/halt-key! ::pool (defmethod ig/halt-key! ::pool
@ -100,7 +101,7 @@
mtf (PrometheusMetricsTrackerFactory. (:registry metrics))] mtf (PrometheusMetricsTrackerFactory. (:registry metrics))]
(doto config (doto config
(.setJdbcUrl (str "jdbc:" dburi)) (.setJdbcUrl (str "jdbc:" dburi))
(.setPoolName (:name cfg "default")) (.setPoolName (d/name (:name cfg)))
(.setAutoCommit true) (.setAutoCommit true)
(.setReadOnly false) (.setReadOnly false)
(.setConnectionTimeout 8000) ;; 8seg (.setConnectionTimeout 8000) ;; 8seg

96
backend/src/app/emails.clj
View file

@ -5,7 +5,7 @@
;; This Source Code Form is "Incompatible With Secondary Licenses", as ;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0. ;; defined by the Mozilla Public License, v. 2.0.
;; ;;
;; Copyright (c) 2020-2021 UXBOX Labs SL ;; Copyright (c) UXBOX Labs SL
(ns app.emails (ns app.emails
"Main api for send emails." "Main api for send emails."
@ -14,36 +14,34 @@
[app.config :as cfg] [app.config :as cfg]
[app.db :as db] [app.db :as db]
[app.db.sql :as sql] [app.db.sql :as sql]
[app.tasks :as tasks]
[app.util.emails :as emails] [app.util.emails :as emails]
[clojure.spec.alpha :as s])) [app.worker :as wrk]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]))
;; --- Defaults
(defn default-context
[]
{:assets-uri (:assets-uri cfg/config)
:public-uri (:public-uri cfg/config)})
;; --- Public API
;; --- PUBLIC API
(defn render (defn render
[email-factory context] [email-factory context]
(email-factory context)) (email-factory context))
(defn send! (defn send!
"Schedule the email for sending." "Schedule the email for sending."
[conn email-factory context] [{:keys [::conn ::factory] :as context}]
(us/verify fn? email-factory) (us/verify fn? factory)
(us/verify map? context) (us/verify some? conn)
(let [email (email-factory context)] (let [email (factory context)]
(tasks/submit! conn {:name "sendmail" (wrk/submit! (assoc email
:delay 0 ::wrk/task :sendmail
:max-retries 1 ::wrk/delay 0
:priority 200 ::wrk/max-retries 1
:props email}))) ::wrk/priority 200
::wrk/conn conn))))
;; --- BOUNCE/COMPLAINS HANDLING
(def sql:profile-complaint-report (def sql:profile-complaint-report
"select (select count(*) "select (select count(*)
from profile_complaint_report from profile_complaint_report
@ -91,7 +89,7 @@
(>= (count reports) threshold)))) (>= (count reports) threshold))))
;; --- Emails ;; --- EMAIL FACTORIES
(s/def ::subject ::us/string) (s/def ::subject ::us/string)
(s/def ::content ::us/string) (s/def ::content ::us/string)
@ -101,7 +99,7 @@
(def feedback (def feedback
"A profile feedback email." "A profile feedback email."
(emails/template-factory ::feedback default-context)) (emails/template-factory ::feedback))
(s/def ::name ::us/string) (s/def ::name ::us/string)
(s/def ::register (s/def ::register
@ -109,7 +107,7 @@
(def register (def register
"A new profile registration welcome email." "A new profile registration welcome email."
(emails/template-factory ::register default-context)) (emails/template-factory ::register))
(s/def ::token ::us/string) (s/def ::token ::us/string)
(s/def ::password-recovery (s/def ::password-recovery
@ -117,7 +115,7 @@
(def password-recovery (def password-recovery
"A password recovery notification email." "A password recovery notification email."
(emails/template-factory ::password-recovery default-context)) (emails/template-factory ::password-recovery))
(s/def ::pending-email ::us/email) (s/def ::pending-email ::us/email)
(s/def ::change-email (s/def ::change-email
@ -125,7 +123,7 @@
(def change-email (def change-email
"Password change confirmation email" "Password change confirmation email"
(emails/template-factory ::change-email default-context)) (emails/template-factory ::change-email))
(s/def :internal.emails.invite-to-team/invited-by ::us/string) (s/def :internal.emails.invite-to-team/invited-by ::us/string)
(s/def :internal.emails.invite-to-team/team ::us/string) (s/def :internal.emails.invite-to-team/team ::us/string)
@ -138,4 +136,50 @@
(def invite-to-team (def invite-to-team
"Teams member invitation email." "Teams member invitation email."
(emails/template-factory ::invite-to-team default-context)) (emails/template-factory ::invite-to-team))
;; --- SENDMAIL TASK
(declare send-console!)
(s/def ::username ::cfg/smtp-username)
(s/def ::password ::cfg/smtp-password)
(s/def ::tls ::cfg/smtp-tls)
(s/def ::ssl ::cfg/smtp-ssl)
(s/def ::host ::cfg/smtp-host)
(s/def ::port ::cfg/smtp-port)
(s/def ::default-reply-to ::cfg/smtp-default-reply-to)
(s/def ::default-from ::cfg/smtp-default-from)
(s/def ::enabled ::cfg/smtp-enabled)
(defmethod ig/pre-init-spec ::sendmail-handler [_]
(s/keys :req-un [::enabled]
:opt-un [::username
::password
::tls
::ssl
::host
::port
::default-from
::default-reply-to]))
(defmethod ig/init-key ::sendmail-handler
[_ cfg]
(fn [{:keys [props] :as task}]
(if (:enabled cfg)
(emails/send! cfg props)
(send-console! cfg props))))
(defn- send-console!
[cfg email]
(let [baos (java.io.ByteArrayOutputStream.)
mesg (emails/smtp-message cfg email)]
(.writeTo mesg baos)
(let [out (with-out-str
(println "email console dump:")
(println "******** start email" (:id email) "**********")
(println (.toString baos))
(println "******** end email "(:id email) "**********"))]
(log/info out))))

77
backend/src/app/http.clj
View file

@ -5,13 +5,13 @@
;; This Source Code Form is "Incompatible With Secondary Licenses", as ;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0. ;; defined by the Mozilla Public License, v. 2.0.
;; ;;
;; Copyright (c) 2020-2021 UXBOX Labs SL ;; Copyright (c) UXBOX Labs SL
(ns app.http (ns app.http
(:require (:require
[app.common.data :as d] [app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.spec :as us] [app.common.spec :as us]
[app.config :as cfg]
[app.http.errors :as errors] [app.http.errors :as errors]
[app.http.middleware :as middleware] [app.http.middleware :as middleware]
[app.metrics :as mtx] [app.metrics :as mtx]
@ -26,22 +26,24 @@
org.eclipse.jetty.server.handler.ErrorHandler org.eclipse.jetty.server.handler.ErrorHandler
org.eclipse.jetty.server.handler.StatisticsHandler)) org.eclipse.jetty.server.handler.StatisticsHandler))
(declare router-handler)
(s/def ::handler fn?) (s/def ::handler fn?)
(s/def ::router some?)
(s/def ::ws (s/map-of ::us/string fn?)) (s/def ::ws (s/map-of ::us/string fn?))
(s/def ::port ::cfg/http-server-port) (s/def ::port ::us/integer)
(s/def ::name ::us/string) (s/def ::name ::us/string)
(defmethod ig/pre-init-spec ::server [_] (defmethod ig/pre-init-spec ::server [_]
(s/keys :req-un [::handler ::port] (s/keys :req-un [::port]
:opt-un [::ws ::name ::mtx/metrics])) :opt-un [::ws ::name ::mtx/metrics ::router ::handler]))
(defmethod ig/prep-key ::server (defmethod ig/prep-key ::server
[_ cfg] [_ cfg]
(merge {:name "http"} (merge {:name "http"} (d/without-nils cfg)))
(d/without-nils cfg)))
(defmethod ig/init-key ::server (defmethod ig/init-key ::server
[_ {:keys [handler ws port name metrics] :as opts}] [_ {:keys [handler router ws port name metrics] :as opts}]
(log/infof "starting '%s' server on port %s." name port) (log/infof "starting '%s' server on port %s." name port)
(let [pre-start (fn [^Server server] (let [pre-start (fn [^Server server]
(let [handler (doto (ErrorHandler.) (let [handler (doto (ErrorHandler.)
@ -49,7 +51,7 @@
(.setServer server))] (.setServer server))]
(.setErrorHandler server ^ErrorHandler handler) (.setErrorHandler server ^ErrorHandler handler)
(when metrics (when metrics
(let [stats (new StatisticsHandler)] (let [stats (StatisticsHandler.)]
(.setHandler ^StatisticsHandler stats (.getHandler server)) (.setHandler ^StatisticsHandler stats (.getHandler server))
(.setHandler server stats) (.setHandler server stats)
(mtx/instrument-jetty! (:registry metrics) stats))))) (mtx/instrument-jetty! (:registry metrics) stats)))))
@ -63,6 +65,13 @@
(when (seq ws) (when (seq ws)
{:websockets ws})) {:websockets ws}))
handler (cond
(fn? handler) handler
(some? router) (router-handler router)
:else (ex/raise :type :internal
:code :invalid-argument
:hint "Missing `handler` or `router` option."))
server (jetty/run-jetty handler options)] server (jetty/run-jetty handler options)]
(assoc opts :server server))) (assoc opts :server server)))
@ -71,27 +80,9 @@
(log/infof "stoping '%s' server on port %s." name port) (log/infof "stoping '%s' server on port %s." name port)
(jetty/stop-server server)) (jetty/stop-server server))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (defn- router-handler
;; Http Main Handler (Router) [router]
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (let [handler (rr/ring-handler router
(declare create-router)
(s/def ::rpc map?)
(s/def ::session map?)
(s/def ::metrics map?)
(s/def ::oauth map?)
(s/def ::storage map?)
(s/def ::assets map?)
(s/def ::feedback fn?)
(defmethod ig/pre-init-spec ::router [_]
(s/keys :req-un [::rpc ::session ::metrics ::oauth ::storage ::assets ::feedback]))
(defmethod ig/init-key ::router
[_ cfg]
(let [handler (rr/ring-handler
(create-router cfg)
(rr/routes (rr/routes
(rr/create-resource-handler {:path "/"}) (rr/create-resource-handler {:path "/"})
(rr/create-default-handler)) (rr/create-default-handler))
@ -104,18 +95,30 @@
(let [cdata (errors/get-error-context request e)] (let [cdata (errors/get-error-context request e)]
(update-thread-context! cdata) (update-thread-context! cdata)
(log/errorf e "unhandled exception: %s (id: %s)" (ex-message e) (str (:id cdata))) (log/errorf e "unhandled exception: %s (id: %s)" (ex-message e) (str (:id cdata)))
{:status 500 {:status 500 :body "internal server error"})
:body "internal server error"})
(catch Throwable e (catch Throwable e
(log/errorf e "unhandled exception: %s" (ex-message e)) (log/errorf e "unhandled exception: %s" (ex-message e))
{:status 500 {:status 500 :body "internal server error"})))))))
:body "internal server error"})))))))
(defn- create-router
[{:keys [session rpc oauth metrics svgparse assets feedback] :as cfg}] ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Http Main Handler (Router)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(s/def ::rpc map?)
(s/def ::session map?)
(s/def ::oauth map?)
(s/def ::storage map?)
(s/def ::assets map?)
(s/def ::feedback fn?)
(defmethod ig/pre-init-spec ::router [_]
(s/keys :req-un [::rpc ::session ::mtx/metrics ::oauth ::storage ::assets ::feedback]))
(defmethod ig/init-key ::router
[_ {:keys [session rpc oauth metrics svgparse assets feedback] :as cfg}]
(rr/router (rr/router
[["/metrics" {:get (:handler metrics)}] [["/metrics" {:get (:handler metrics)}]
["/assets" {:middleware [[middleware/format-response-body] ["/assets" {:middleware [[middleware/format-response-body]
[middleware/errors errors/handle]]} [middleware/errors errors/handle]]}
["/by-id/:id" {:get (:objects-handler assets)}] ["/by-id/:id" {:get (:objects-handler assets)}]

7
backend/src/app/http/feedback.clj
View file

@ -15,7 +15,7 @@
[app.common.spec :as us] [app.common.spec :as us]
[app.config :as cfg] [app.config :as cfg]
[app.db :as db] [app.db :as db]
[app.emails :as emails] [app.emails :as eml]
[app.rpc.queries.profile :as profile] [app.rpc.queries.profile :as profile]
[clojure.spec.alpha :as s] [clojure.spec.alpha :as s]
[integrant.core :as ig])) [integrant.core :as ig]))
@ -62,8 +62,9 @@
[pool profile params] [pool profile params]
(let [params (us/conform ::feedback params) (let [params (us/conform ::feedback params)
destination (cfg/get :feedback-destination)] destination (cfg/get :feedback-destination)]
(emails/send! pool emails/feedback (eml/send! {::eml/conn pool
{:to destination ::eml/factory eml/feedback
:to destination
:profile profile :profile profile
:reply-to (:from params) :reply-to (:from params)
:email (:from params) :email (:from params)

5
backend/src/app/http/oauth/github.clj
View file

@ -5,13 +5,12 @@
;; This Source Code Form is "Incompatible With Secondary Licenses", as ;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0. ;; defined by the Mozilla Public License, v. 2.0.
;; ;;
;; Copyright (c) 2020-2021 UXBOX Labs SL ;; Copyright (c) UXBOX Labs SL
(ns app.http.oauth.github (ns app.http.oauth.github
(:require (:require
[app.common.exceptions :as ex] [app.common.exceptions :as ex]
[app.common.spec :as us] [app.common.spec :as us]
[app.config :as cfg]
[app.http.oauth.google :as gg] [app.http.oauth.google :as gg]
[app.util.http :as http] [app.util.http :as http]
[app.util.time :as dt] [app.util.time :as dt]
@ -105,7 +104,7 @@
state (tokens :generate {:iss :github-oauth state (tokens :generate {:iss :github-oauth
:invitation-token invitation :invitation-token invitation
:exp (dt/in-future "15m")}) :exp (dt/in-future "15m")})
params {:client_id (:client-id cfg/config) params {:client_id (:client-id cfg)
:redirect_uri (build-redirect-url cfg) :redirect_uri (build-redirect-url cfg)
:state state :state state
:scope scope} :scope scope}

2
backend/src/app/loggers/mattermost.clj
View file

@ -65,7 +65,7 @@
(try (try
(let [uri (:uri cfg) (let [uri (:uri cfg)
text (str "Unhandled exception (@channel):\n" text (str "Unhandled exception (@channel):\n"
"- detail: " (:public-uri cfg/config) "/dbg/error-by-id/" id "\n" "- detail: " (cfg/get :public-uri) "/dbg/error-by-id/" id "\n"
"- host: `" host "`\n" "- host: `" host "`\n"
"- version: `" version "`\n" "- version: `" version "`\n"
(when error (when error

196
backend/src/app/main.clj
View file

@ -5,34 +5,24 @@
;; This Source Code Form is "Incompatible With Secondary Licenses", as ;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0. ;; defined by the Mozilla Public License, v. 2.0.
;; ;;
;; Copyright (c) 2020-2021 UXBOX Labs SL ;; Copyright (c) UXBOX Labs SL
(ns app.main (ns app.main
(:require (:require
[app.common.data :as d] [app.common.data :as d]
[app.config :as cfg] [app.config :as cf]
[app.util.time :as dt] [app.util.time :as dt]
[clojure.pprint :as pprint]
[clojure.tools.logging :as log] [clojure.tools.logging :as log]
[integrant.core :as ig])) [integrant.core :as ig]))
;; Set value for all new threads bindings. (def system-config
(alter-var-root #'*assert* (constantly (:asserts-enabled cfg/config)))
(derive :app.telemetry/server :app.http/server)
;; --- Entry point
(defn build-system-config
[config]
(d/deep-merge
{:app.db/pool {:app.db/pool
{:uri (:database-uri config) {:uri (cf/get :database-uri)
:username (:database-username config) :username (cf/get :database-username)
:password (:database-password config) :password (cf/get :database-password)
:metrics (ig/ref :app.metrics/metrics) :metrics (ig/ref :app.metrics/metrics)
:migrations (ig/ref :app.migrations/all) :migrations (ig/ref :app.migrations/all)
:name "main" :name :main
:min-pool-size 0 :min-pool-size 0
:max-pool-size 20} :max-pool-size 20}
@ -48,18 +38,14 @@
:type :counter}}} :type :counter}}}
:app.migrations/all :app.migrations/all
{:main (ig/ref :app.migrations/migrations) {:main (ig/ref :app.migrations/migrations)}
:telemetry (ig/ref :app.telemetry/migrations)}
:app.migrations/migrations :app.migrations/migrations
{} {}
:app.telemetry/migrations
{}
:app.msgbus/msgbus :app.msgbus/msgbus
{:backend (:msgbus-backend config :redis) {:backend (cf/get :msgbus-backend :redis)
:redis-uri (:redis-uri config)} :redis-uri (cf/get :redis-uri)}
:app.tokens/tokens :app.tokens/tokens
{:sprops (ig/ref :app.setup/props)} {:sprops (ig/ref :app.setup/props)}
@ -78,35 +64,36 @@
:app.http.session/session :app.http.session/session
{:pool (ig/ref :app.db/pool) {:pool (ig/ref :app.db/pool)
:cookie-name (:http-session-cookie-name config)} :cookie-name (cf/get :http-session-cookie-name)}
:app.http.session/gc-task :app.http.session/gc-task
{:pool (ig/ref :app.db/pool) {:pool (ig/ref :app.db/pool)
:max-age (:http-session-idle-max-age config)} :max-age (cf/get :http-session-idle-max-age)}
:app.http.session/updater :app.http.session/updater
{:pool (ig/ref :app.db/pool) {:pool (ig/ref :app.db/pool)
:metrics (ig/ref :app.metrics/metrics) :metrics (ig/ref :app.metrics/metrics)
:executor (ig/ref :app.worker/executor) :executor (ig/ref :app.worker/executor)
:session (ig/ref :app.http.session/session) :session (ig/ref :app.http.session/session)
:max-batch-age (:http-session-updater-batch-max-age config) :max-batch-age (cf/get :http-session-updater-batch-max-age)
:max-batch-size (:http-session-updater-batch-max-size config)} :max-batch-size (cf/get :http-session-updater-batch-max-size)}
:app.http.awsns/handler :app.http.awsns/handler
{:tokens (ig/ref :app.tokens/tokens) {:tokens (ig/ref :app.tokens/tokens)
:pool (ig/ref :app.db/pool)} :pool (ig/ref :app.db/pool)}
:app.http/server :app.http/server
{:port (:http-server-port config) {:port (cf/get :http-server-port)
:handler (ig/ref :app.http/router) :router (ig/ref :app.http/router)
:metrics (ig/ref :app.metrics/metrics) :metrics (ig/ref :app.metrics/metrics)
:ws {"/ws/notifications" (ig/ref :app.notifications/handler)}} :ws {"/ws/notifications" (ig/ref :app.notifications/handler)}}
:app.http/router :app.http/router
{:rpc (ig/ref :app.rpc/rpc) {
:rpc (ig/ref :app.rpc/rpc)
:session (ig/ref :app.http.session/session) :session (ig/ref :app.http.session/session)
:tokens (ig/ref :app.tokens/tokens) :tokens (ig/ref :app.tokens/tokens)
:public-uri (:public-uri config) :public-uri (cf/get :public-uri)
:metrics (ig/ref :app.metrics/metrics) :metrics (ig/ref :app.metrics/metrics)
:oauth (ig/ref :app.http.oauth/all) :oauth (ig/ref :app.http.oauth/all)
:assets (ig/ref :app.http.assets/handlers) :assets (ig/ref :app.http.assets/handlers)
@ -118,7 +105,7 @@
:app.http.assets/handlers :app.http.assets/handlers
{:metrics (ig/ref :app.metrics/metrics) {:metrics (ig/ref :app.metrics/metrics)
:assets-path (:assets-path config) :assets-path (cf/get :assets-path)
:storage (ig/ref :app.storage/storage) :storage (ig/ref :app.storage/storage)
:cache-max-age (dt/duration {:hours 24}) :cache-max-age (dt/duration {:hours 24})
:signature-max-age (dt/duration {:hours 24 :minutes 5})} :signature-max-age (dt/duration {:hours 24 :minutes 5})}
@ -135,38 +122,42 @@
{:rpc (ig/ref :app.rpc/rpc) {:rpc (ig/ref :app.rpc/rpc)
:session (ig/ref :app.http.session/session) :session (ig/ref :app.http.session/session)
:tokens (ig/ref :app.tokens/tokens) :tokens (ig/ref :app.tokens/tokens)
:public-uri (:public-uri config) :public-uri (cf/get :public-uri)
:client-id (:google-client-id config) :client-id (cf/get :google-client-id)
:client-secret (:google-client-secret config)} :client-secret (cf/get :google-client-secret)}
:app.http.oauth/github :app.http.oauth/github
{:rpc (ig/ref :app.rpc/rpc) {:rpc (ig/ref :app.rpc/rpc)
:session (ig/ref :app.http.session/session) :session (ig/ref :app.http.session/session)
:tokens (ig/ref :app.tokens/tokens) :tokens (ig/ref :app.tokens/tokens)
:public-uri (:public-uri config) :public-uri (cf/get :public-uri)
:client-id (:github-client-id config) :client-id (cf/get :github-client-id)
:client-secret (:github-client-secret config)} :client-secret (cf/get :github-client-secret)}
:app.http.oauth/gitlab :app.http.oauth/gitlab
{:rpc (ig/ref :app.rpc/rpc) {:rpc (ig/ref :app.rpc/rpc)
:session (ig/ref :app.http.session/session) :session (ig/ref :app.http.session/session)
:tokens (ig/ref :app.tokens/tokens) :tokens (ig/ref :app.tokens/tokens)
:public-uri (:public-uri config) :public-uri (cf/get :public-uri)
:base-uri (:gitlab-base-uri config) :base-uri (cf/get :gitlab-base-uri)
:client-id (:gitlab-client-id config) :client-id (cf/get :gitlab-client-id)
:client-secret (:gitlab-client-secret config)} :client-secret (cf/get :gitlab-client-secret)}
:app.svgparse/svgc
{:metrics (ig/ref :app.metrics/metrics)}
;; HTTP Handler for SVG parsing ;; HTTP Handler for SVG parsing
:app.svgparse/handler :app.svgparse/handler
{:metrics (ig/ref :app.metrics/metrics)} {:metrics (ig/ref :app.metrics/metrics)
:svgc (ig/ref :app.svgparse/svgc)}
;; RLimit definition for password hashing ;; RLimit definition for password hashing
:app.rlimits/password :app.rlimits/password
(:rlimits-password config) (cf/get :rlimits-password)
;; RLimit definition for image processing ;; RLimit definition for image processing
:app.rlimits/image :app.rlimits/image
(:rlimits-image config) (cf/get :rlimits-image)
;; A collection of rlimits as hash-map. ;; A collection of rlimits as hash-map.
:app.rlimits/all :app.rlimits/all
@ -180,7 +171,9 @@
:metrics (ig/ref :app.metrics/metrics) :metrics (ig/ref :app.metrics/metrics)
:storage (ig/ref :app.storage/storage) :storage (ig/ref :app.storage/storage)
:msgbus (ig/ref :app.msgbus/msgbus) :msgbus (ig/ref :app.msgbus/msgbus)
:rlimits (ig/ref :app.rlimits/all)} :rlimits (ig/ref :app.rlimits/all)
:svgc (ig/ref :app.svgparse/svgc)
:public-uri (cf/get :public-uri)}
:app.notifications/handler :app.notifications/handler
{:msgbus (ig/ref :app.msgbus/msgbus) {:msgbus (ig/ref :app.msgbus/msgbus)
@ -190,56 +183,52 @@
:executor (ig/ref :app.worker/executor)} :executor (ig/ref :app.worker/executor)}
:app.worker/executor :app.worker/executor
{:name "worker"} {:min-threads 0
:max-threads 256
:idle-timeout 60000
:name :worker}
:app.worker/worker :app.worker/worker
{:executor (ig/ref :app.worker/executor) {:executor (ig/ref :app.worker/executor)
:pool (ig/ref :app.db/pool) :tasks (ig/ref :app.worker/registry)
:tasks (ig/ref :app.tasks/registry)} :metrics (ig/ref :app.metrics/metrics)
:pool (ig/ref :app.db/pool)}
:app.worker/scheduler :app.worker/scheduler
{:executor (ig/ref :app.worker/executor) {:executor (ig/ref :app.worker/executor)
:tasks (ig/ref :app.worker/registry)
:pool (ig/ref :app.db/pool) :pool (ig/ref :app.db/pool)
:tasks (ig/ref :app.tasks/registry)
:schedule :schedule
[{:id "file-media-gc" [{:cron #app/cron "0 0 0 */1 * ? *" ;; daily
:cron #app/cron "0 0 0 */1 * ? *" ;; daily
:task :file-media-gc} :task :file-media-gc}
{:id "file-xlog-gc" {:cron #app/cron "0 0 */1 * * ?" ;; hourly
:cron #app/cron "0 0 */1 * * ?" ;; hourly
:task :file-xlog-gc} :task :file-xlog-gc}
{:id "storage-deleted-gc" {:cron #app/cron "0 0 1 */1 * ?" ;; daily (1 hour shift)
:cron #app/cron "0 0 1 */1 * ?" ;; daily (1 hour shift)
:task :storage-deleted-gc} :task :storage-deleted-gc}
{:id "storage-touched-gc" {:cron #app/cron "0 0 2 */1 * ?" ;; daily (2 hour shift)
:cron #app/cron "0 0 2 */1 * ?" ;; daily (2 hour shift)
:task :storage-touched-gc} :task :storage-touched-gc}
{:id "session-gc" {:cron #app/cron "0 0 3 */1 * ?" ;; daily (3 hour shift)
:cron #app/cron "0 0 3 */1 * ?" ;; daily (3 hour shift)
:task :session-gc} :task :session-gc}
{:id "storage-recheck" {:cron #app/cron "0 0 */1 * * ?" ;; hourly
:cron #app/cron "0 0 */1 * * ?" ;; hourly
:task :storage-recheck} :task :storage-recheck}
{:id "tasks-gc" {:cron #app/cron "0 0 0 */1 * ?" ;; daily
:cron #app/cron "0 0 0 */1 * ?" ;; daily
:task :tasks-gc} :task :tasks-gc}
(when (:telemetry-enabled config) (when (cf/get :telemetry-enabled)
{:id "telemetry" {:cron #app/cron "0 0 */6 * * ?" ;; every 6h
:cron #app/cron "0 0 */6 * * ?" ;; every 6h :uri (cf/get :telemetry-uri)
:uri (:telemetry-uri config)
:task :telemetry})]} :task :telemetry})]}
:app.tasks/registry :app.worker/registry
{:metrics (ig/ref :app.metrics/metrics) {:metrics (ig/ref :app.metrics/metrics)
:tasks :tasks
{:sendmail (ig/ref :app.tasks.sendmail/handler) {:sendmail (ig/ref :app.emails/sendmail-handler)
:delete-object (ig/ref :app.tasks.delete-object/handler) :delete-object (ig/ref :app.tasks.delete-object/handler)
:delete-profile (ig/ref :app.tasks.delete-profile/handler) :delete-profile (ig/ref :app.tasks.delete-profile/handler)
:file-media-gc (ig/ref :app.tasks.file-media-gc/handler) :file-media-gc (ig/ref :app.tasks.file-media-gc/handler)
@ -251,17 +240,17 @@
:telemetry (ig/ref :app.tasks.telemetry/handler) :telemetry (ig/ref :app.tasks.telemetry/handler)
:session-gc (ig/ref :app.http.session/gc-task)}} :session-gc (ig/ref :app.http.session/gc-task)}}
:app.tasks.sendmail/handler :app.emails/sendmail-handler
{:host (:smtp-host config) {:host (cf/get :smtp-host)
:port (:smtp-port config) :port (cf/get :smtp-port)
:ssl (:smtp-ssl config) :ssl (cf/get :smtp-ssl)
:tls (:smtp-tls config) :tls (cf/get :smtp-tls)
:enabled (:smtp-enabled config) :enabled (cf/get :smtp-enabled)
:username (:smtp-username config) :username (cf/get :smtp-username)
:password (:smtp-password config) :password (cf/get :smtp-password)
:metrics (ig/ref :app.metrics/metrics) :metrics (ig/ref :app.metrics/metrics)
:default-reply-to (:smtp-default-reply-to config) :default-reply-to (cf/get :smtp-default-reply-to)
:default-from (:smtp-default-from config)} :default-from (cf/get :smtp-default-from)}
:app.tasks.tasks-gc/handler :app.tasks.tasks-gc/handler
{:pool (ig/ref :app.db/pool) {:pool (ig/ref :app.db/pool)
@ -293,27 +282,27 @@
:app.tasks.telemetry/handler :app.tasks.telemetry/handler
{:pool (ig/ref :app.db/pool) {:pool (ig/ref :app.db/pool)
:version (:full cfg/version) :version (:full cf/version)
:uri (:telemetry-uri config) :uri (cf/get :telemetry-uri)
:sprops (ig/ref :app.setup/props)} :sprops (ig/ref :app.setup/props)}
:app.srepl/server :app.srepl/server
{:port (:srepl-port config) {:port (cf/get :srepl-port)
:host (:srepl-host config)} :host (cf/get :srepl-host)}
:app.setup/props :app.setup/props
{:pool (ig/ref :app.db/pool)} {:pool (ig/ref :app.db/pool)}
:app.loggers.zmq/receiver :app.loggers.zmq/receiver
{:endpoint (:loggers-zmq-uri config)} {:endpoint (cf/get :loggers-zmq-uri)}
:app.loggers.loki/reporter :app.loggers.loki/reporter
{:uri (:loggers-loki-uri config) {:uri (cf/get :loggers-loki-uri)
:receiver (ig/ref :app.loggers.zmq/receiver) :receiver (ig/ref :app.loggers.zmq/receiver)
:executor (ig/ref :app.worker/executor)} :executor (ig/ref :app.worker/executor)}
:app.loggers.mattermost/reporter :app.loggers.mattermost/reporter
{:uri (:error-report-webhook config) {:uri (cf/get :error-report-webhook)
:receiver (ig/ref :app.loggers.zmq/receiver) :receiver (ig/ref :app.loggers.zmq/receiver)
:pool (ig/ref :app.db/pool) :pool (ig/ref :app.db/pool)
:executor (ig/ref :app.worker/executor)} :executor (ig/ref :app.worker/executor)}
@ -324,34 +313,24 @@
:app.storage/storage :app.storage/storage
{:pool (ig/ref :app.db/pool) {:pool (ig/ref :app.db/pool)
:executor (ig/ref :app.worker/executor) :executor (ig/ref :app.worker/executor)
:backend (:storage-backend config :fs) :backend (cf/get :storage-backend :fs)
:backends {:s3 (ig/ref [::main :app.storage.s3/backend]) :backends {:s3 (ig/ref [::main :app.storage.s3/backend])
:db (ig/ref [::main :app.storage.db/backend]) :db (ig/ref [::main :app.storage.db/backend])
:fs (ig/ref [::main :app.storage.fs/backend]) :fs (ig/ref [::main :app.storage.fs/backend])
:tmp (ig/ref [::tmp :app.storage.fs/backend])}} :tmp (ig/ref [::tmp :app.storage.fs/backend])}}
[::main :app.storage.s3/backend] [::main :app.storage.s3/backend]
{:region (:storage-s3-region config) {:region (cf/get :storage-s3-region)
:bucket (:storage-s3-bucket config)} :bucket (cf/get :storage-s3-bucket)}
[::main :app.storage.fs/backend] [::main :app.storage.fs/backend]
{:directory (:storage-fs-directory config)} {:directory (cf/get :storage-fs-directory)}
[::tmp :app.storage.fs/backend] [::tmp :app.storage.fs/backend]
{:directory "/tmp/penpot"} {:directory "/tmp/penpot"}
[::main :app.storage.db/backend] [::main :app.storage.db/backend]
{:pool (ig/ref :app.db/pool)}} {:pool (ig/ref :app.db/pool)}})
(when (:telemetry-server-enabled config)
{:app.telemetry/handler
{:pool (ig/ref :app.db/pool)
:executor (ig/ref :app.worker/executor)}
:app.telemetry/server
{:port (:telemetry-server-port config 6063)
:handler (ig/ref :app.telemetry/handler)
:name "telemetry"}})))
(defmethod ig/init-key :default [_ data] data) (defmethod ig/init-key :default [_ data] data)
(defmethod ig/prep-key :default (defmethod ig/prep-key :default
@ -364,7 +343,6 @@
(defn start (defn start
[] []
(let [system-config (build-system-config cfg/config)]
(ig/load-namespaces system-config) (ig/load-namespaces system-config)
(alter-var-root #'system (fn [sys] (alter-var-root #'system (fn [sys]
(when sys (ig/halt! sys)) (when sys (ig/halt! sys))
@ -372,7 +350,7 @@
(ig/prep) (ig/prep)
(ig/init)))) (ig/init))))
(log/infof "welcome to penpot (version: '%s')" (log/infof "welcome to penpot (version: '%s')"
(:full cfg/version)))) (:full cf/version)))
(defn stop (defn stop
[] []
@ -380,14 +358,6 @@
(when sys (ig/halt! sys)) (when sys (ig/halt! sys))
nil))) nil)))
(prefer-method print-method
clojure.lang.IRecord
clojure.lang.IDeref)
(prefer-method pprint/simple-dispatch
clojure.lang.IPersistentMap
clojure.lang.IDeref)
(defn -main (defn -main
[& _args] [& _args]
(start)) (start))

13
backend/src/app/rpc/mutations/demo.clj
View file

@ -5,7 +5,7 @@
;; This Source Code Form is "Incompatible With Secondary Licenses", as ;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0. ;; defined by the Mozilla Public License, v. 2.0.
;; ;;
;; Copyright (c) 2020-2021 UXBOX Labs SL ;; Copyright (c) UXBOX Labs SL
(ns app.rpc.mutations.demo (ns app.rpc.mutations.demo
"A demo specific mutations." "A demo specific mutations."
@ -16,8 +16,8 @@
[app.db :as db] [app.db :as db]
[app.rpc.mutations.profile :as profile] [app.rpc.mutations.profile :as profile]
[app.setup.initial-data :as sid] [app.setup.initial-data :as sid]
[app.tasks :as tasks]
[app.util.services :as sv] [app.util.services :as sv]
[app.worker :as wrk]
[buddy.core.codecs :as bc] [buddy.core.codecs :as bc]
[buddy.core.nonce :as bn] [buddy.core.nonce :as bn]
[clojure.spec.alpha :as s])) [clojure.spec.alpha :as s]))
@ -40,7 +40,7 @@
:password password :password password
:props {:onboarding-viewed true}}] :props {:onboarding-viewed true}}]
(when-not (:allow-demo-users cfg/config) (when-not (cfg/get :allow-demo-users)
(ex/raise :type :validation (ex/raise :type :validation
:code :demo-users-not-allowed :code :demo-users-not-allowed
:hint "Demo users are disabled by config.")) :hint "Demo users are disabled by config."))
@ -51,9 +51,10 @@
(sid/load-initial-project! conn)) (sid/load-initial-project! conn))
;; Schedule deletion of the demo profile ;; Schedule deletion of the demo profile
(tasks/submit! conn {:name "delete-profile" (wrk/submit! {::wrk/task :delete-profile
:delay cfg/deletion-delay ::wrk/delay cfg/deletion-delay
:props {:profile-id id}}) ::wrk/conn conn
:profile-id id})
{:email email {:email email
:password password}))) :password password})))

12
backend/src/app/rpc/mutations/files.clj
View file

@ -5,7 +5,7 @@
;; This Source Code Form is "Incompatible With Secondary Licenses", as ;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0. ;; defined by the Mozilla Public License, v. 2.0.
;; ;;
;; Copyright (c) 2020 UXBOX Labs SL ;; Copyright (c) UXBOX Labs SL
(ns app.rpc.mutations.files (ns app.rpc.mutations.files
(:require (:require
@ -19,10 +19,10 @@
[app.rpc.permissions :as perms] [app.rpc.permissions :as perms]
[app.rpc.queries.files :as files] [app.rpc.queries.files :as files]
[app.rpc.queries.projects :as proj] [app.rpc.queries.projects :as proj]
[app.tasks :as tasks]
[app.util.blob :as blob] [app.util.blob :as blob]
[app.util.services :as sv] [app.util.services :as sv]
[app.util.time :as dt] [app.util.time :as dt]
[app.worker :as wrk]
[clojure.spec.alpha :as s])) [clojure.spec.alpha :as s]))
;; --- Helpers & Specs ;; --- Helpers & Specs
@ -126,9 +126,11 @@
(files/check-edition-permissions! conn profile-id id) (files/check-edition-permissions! conn profile-id id)
;; Schedule object deletion ;; Schedule object deletion
(tasks/submit! conn {:name "delete-object" (wrk/submit! {::wrk/task :delete-object
:delay cfg/deletion-delay ::wrk/delay cfg/deletion-delay
:props {:id id :type :file}}) ::wrk/conn conn
:id id
:type :file})
(mark-file-deleted conn params))) (mark-file-deleted conn params)))

42
backend/src/app/rpc/mutations/profile.clj
View file

@ -14,16 +14,16 @@
[app.common.uuid :as uuid] [app.common.uuid :as uuid]
[app.config :as cfg] [app.config :as cfg]
[app.db :as db] [app.db :as db]
[app.emails :as emails] [app.emails :as eml]
[app.media :as media] [app.media :as media]
[app.rpc.mutations.projects :as projects] [app.rpc.mutations.projects :as projects]
[app.rpc.mutations.teams :as teams] [app.rpc.mutations.teams :as teams]
[app.rpc.queries.profile :as profile] [app.rpc.queries.profile :as profile]
[app.setup.initial-data :as sid] [app.setup.initial-data :as sid]
[app.storage :as sto] [app.storage :as sto]
[app.tasks :as tasks]
[app.util.services :as sv] [app.util.services :as sv]
[app.util.time :as dt] [app.util.time :as dt]
[app.worker :as wrk]
[buddy.hashers :as hashers] [buddy.hashers :as hashers]
[clojure.spec.alpha :as s] [clojure.spec.alpha :as s]
[cuerdas.core :as str])) [cuerdas.core :as str]))
@ -117,16 +117,19 @@
;; Don't allow proceed in register page if the email is ;; Don't allow proceed in register page if the email is
;; already reported as permanent bounced ;; already reported as permanent bounced
(when (emails/has-bounce-reports? conn (:email profile)) (when (eml/has-bounce-reports? conn (:email profile))
(ex/raise :type :validation (ex/raise :type :validation
:code :email-has-permanent-bounces :code :email-has-permanent-bounces
:hint "looks like the email has one or many bounces reported")) :hint "looks like the email has one or many bounces reported"))
(emails/send! conn emails/register (eml/send! {::eml/conn conn
{:to (:email profile) ::eml/factory eml/register
:public-uri (:public-uri cfg)
:to (:email profile)
:name (:fullname profile) :name (:fullname profile)
:token vtoken :token vtoken
:extra-data ptoken}) :extra-data ptoken})
(with-meta profile (with-meta profile
{:before-complete (annotate-profile-register metrics profile)}))))) {:before-complete (annotate-profile-register metrics profile)})))))
@ -439,7 +442,7 @@
{:changed true}) {:changed true})
(defn- request-email-change (defn- request-email-change
[{:keys [conn tokens]} {:keys [profile email] :as params}] [{:keys [conn tokens] :as cfg} {:keys [profile email] :as params}]
(let [token (tokens :generate (let [token (tokens :generate
{:iss :change-email {:iss :change-email
:exp (dt/in-future "15m") :exp (dt/in-future "15m")
@ -452,18 +455,20 @@
(when (not= email (:email profile)) (when (not= email (:email profile))
(check-profile-existence! conn params)) (check-profile-existence! conn params))
(when-not (emails/allow-send-emails? conn profile) (when-not (eml/allow-send-emails? conn profile)
(ex/raise :type :validation (ex/raise :type :validation
:code :profile-is-muted :code :profile-is-muted
:hint "looks like the profile has reported repeatedly as spam or has permanent bounces.")) :hint "looks like the profile has reported repeatedly as spam or has permanent bounces."))
(when (emails/has-bounce-reports? conn email) (when (eml/has-bounce-reports? conn email)
(ex/raise :type :validation (ex/raise :type :validation
:code :email-has-permanent-bounces :code :email-has-permanent-bounces
:hint "looks like the email you invite has been repeatedly reported as spam or permanent bounce")) :hint "looks like the email you invite has been repeatedly reported as spam or permanent bounce"))
(emails/send! conn emails/change-email (eml/send! {::eml/conn conn
{:to (:email profile) ::eml/factory eml/change-email
:public-uri (:public-uri cfg)
:to (:email profile)
:name (:fullname profile) :name (:fullname profile)
:pending-email email :pending-email email
:token token :token token
@ -493,8 +498,10 @@
(let [ptoken (tokens :generate-predefined (let [ptoken (tokens :generate-predefined
{:iss :profile-identity {:iss :profile-identity
:profile-id (:id profile)})] :profile-id (:id profile)})]
(emails/send! conn emails/password-recovery (eml/send! {::eml/conn conn
{:to (:email profile) ::eml/factory eml/password-recovery
:public-uri (:public-uri cfg)
:to (:email profile)
:token (:token profile) :token (:token profile)
:name (:fullname profile) :name (:fullname profile)
:extra-data ptoken}) :extra-data ptoken})
@ -502,7 +509,7 @@
(db/with-atomic [conn pool] (db/with-atomic [conn pool]
(when-let [profile (profile/retrieve-profile-data-by-email conn email)] (when-let [profile (profile/retrieve-profile-data-by-email conn email)]
(when-not (emails/allow-send-emails? conn profile) (when-not (eml/allow-send-emails? conn profile)
(ex/raise :type :validation (ex/raise :type :validation
:code :profile-is-muted :code :profile-is-muted
:hint "looks like the profile has reported repeatedly as spam or has permanent bounces.")) :hint "looks like the profile has reported repeatedly as spam or has permanent bounces."))
@ -512,7 +519,7 @@
:code :profile-not-verified :code :profile-not-verified
:hint "the user need to validate profile before recover password")) :hint "the user need to validate profile before recover password"))
(when (emails/has-bounce-reports? conn (:email profile)) (when (eml/has-bounce-reports? conn (:email profile))
(ex/raise :type :validation (ex/raise :type :validation
:code :email-has-permanent-bounces :code :email-has-permanent-bounces
:hint "looks like the email you invite has been repeatedly reported as spam or permanent bounce")) :hint "looks like the email you invite has been repeatedly reported as spam or permanent bounce"))
@ -579,9 +586,10 @@
(check-can-delete-profile! conn profile-id) (check-can-delete-profile! conn profile-id)
;; Schedule a complete deletion of profile ;; Schedule a complete deletion of profile
(tasks/submit! conn {:name "delete-profile" (wrk/submit! {::wrk/task :delete-profile
:delay cfg/deletion-delay ::wrk/dalay cfg/deletion-delay
:props {:profile-id profile-id}}) ::wrk/conn conn
:profile-id profile-id})
(db/update! conn :profile (db/update! conn :profile
{:deleted-at (dt/now)} {:deleted-at (dt/now)}

10
backend/src/app/rpc/mutations/projects.clj
View file

@ -16,9 +16,9 @@
[app.rpc.permissions :as perms] [app.rpc.permissions :as perms]
[app.rpc.queries.projects :as proj] [app.rpc.queries.projects :as proj]
[app.rpc.queries.teams :as teams] [app.rpc.queries.teams :as teams]
[app.tasks :as tasks]
[app.util.services :as sv] [app.util.services :as sv]
[app.util.time :as dt] [app.util.time :as dt]
[app.worker :as wrk]
[clojure.spec.alpha :as s])) [clojure.spec.alpha :as s]))
;; --- Helpers & Specs ;; --- Helpers & Specs
@ -128,9 +128,11 @@
(proj/check-edition-permissions! conn profile-id id) (proj/check-edition-permissions! conn profile-id id)
;; Schedule object deletion ;; Schedule object deletion
(tasks/submit! conn {:name "delete-object" (wrk/submit! {::wrk/task :delete-object
:delay cfg/deletion-delay ::wrk/delay cfg/deletion-delay
:props {:id id :type :project}}) ::wrk/conn conn
:id id
:type :project})
(db/update! conn :project (db/update! conn :project
{:deleted-at (dt/now)} {:deleted-at (dt/now)}

26
backend/src/app/rpc/mutations/teams.clj
View file

@ -5,7 +5,7 @@
;; This Source Code Form is "Incompatible With Secondary Licenses", as ;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0. ;; defined by the Mozilla Public License, v. 2.0.
;; ;;
;; Copyright (c) 2020-2021 UXBOX Labs SL ;; Copyright (c) UXBOX Labs SL
(ns app.rpc.mutations.teams (ns app.rpc.mutations.teams
(:require (:require
@ -15,16 +15,16 @@
[app.common.uuid :as uuid] [app.common.uuid :as uuid]
[app.config :as cfg] [app.config :as cfg]
[app.db :as db] [app.db :as db]
[app.emails :as emails] [app.emails :as eml]
[app.media :as media] [app.media :as media]
[app.rpc.mutations.projects :as projects] [app.rpc.mutations.projects :as projects]
[app.rpc.permissions :as perms] [app.rpc.permissions :as perms]
[app.rpc.queries.profile :as profile] [app.rpc.queries.profile :as profile]
[app.rpc.queries.teams :as teams] [app.rpc.queries.teams :as teams]
[app.storage :as sto] [app.storage :as sto]
[app.tasks :as tasks]
[app.util.services :as sv] [app.util.services :as sv]
[app.util.time :as dt] [app.util.time :as dt]
[app.worker :as wrk]
[clojure.spec.alpha :as s] [clojure.spec.alpha :as s]
[datoteka.core :as fs])) [datoteka.core :as fs]))
@ -139,9 +139,11 @@
:code :only-owner-can-delete-team)) :code :only-owner-can-delete-team))
;; Schedule object deletion ;; Schedule object deletion
(tasks/submit! conn {:name "delete-object" (wrk/submit! {::wrk/task :delete-object
:delay cfg/deletion-delay ::wrk/delay cfg/deletion-delay
:props {:id id :type :team}}) ::wrk/conn conn
:id id
:type :team})
(db/update! conn :team (db/update! conn :team
{:deleted-at (dt/now)} {:deleted-at (dt/now)}
@ -323,25 +325,27 @@
:code :insufficient-permissions)) :code :insufficient-permissions))
;; First check if the current profile is allowed to send emails. ;; First check if the current profile is allowed to send emails.
(when-not (emails/allow-send-emails? conn profile) (when-not (eml/allow-send-emails? conn profile)
(ex/raise :type :validation (ex/raise :type :validation
:code :profile-is-muted :code :profile-is-muted
:hint "looks like the profile has reported repeatedly as spam or has permanent bounces")) :hint "looks like the profile has reported repeatedly as spam or has permanent bounces"))
(when (and member (not (emails/allow-send-emails? conn member))) (when (and member (not (eml/allow-send-emails? conn member)))
(ex/raise :type :validation (ex/raise :type :validation
:code :member-is-muted :code :member-is-muted
:hint "looks like the profile has reported repeatedly as spam or has permanent bounces")) :hint "looks like the profile has reported repeatedly as spam or has permanent bounces"))
;; Secondly check if the invited member email is part of the ;; Secondly check if the invited member email is part of the
;; global spam/bounce report. ;; global spam/bounce report.
(when (emails/has-bounce-reports? conn email) (when (eml/has-bounce-reports? conn email)
(ex/raise :type :validation (ex/raise :type :validation
:code :email-has-permanent-bounces :code :email-has-permanent-bounces
:hint "looks like the email you invite has been repeatedly reported as spam or permanent bounce")) :hint "looks like the email you invite has been repeatedly reported as spam or permanent bounce"))
(emails/send! conn emails/invite-to-team (eml/send! {::eml/conn conn
{:to email ::eml/factory eml/invite-to-team
:public-uri (:public-uri cfg)
:to email
:invited-by (:fullname profile) :invited-by (:fullname profile)
:team (:name team) :team (:name team)
:token itoken :token itoken

110
backend/src/app/tasks.clj
View file

@ -1,110 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
(ns app.tasks
(:require
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.db :as db]
[app.metrics :as mtx]
[app.util.time :as dt]
[app.worker]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]))
(s/def ::name ::us/string)
(s/def ::delay
(s/or :int ::us/integer
:duration dt/duration?))
(s/def ::queue ::us/string)
(s/def ::task-options
(s/keys :req-un [::name]
:opt-un [::delay ::props ::queue]))
(def ^:private sql:insert-new-task
"insert into task (id, name, props, queue, priority, max_retries, scheduled_at)
values (?, ?, ?, ?, ?, ?, clock_timestamp() + ?)
returning id")
(defn submit!
[conn {:keys [name delay props queue priority max-retries]
:or {delay 0 props {} queue "default" priority 100 max-retries 3}
:as options}]
(us/verify ::task-options options)
(let [duration (dt/duration delay)
interval (db/interval duration)
props (db/tjson props)
id (uuid/next)]
(log/debugf "submit task '%s' to be executed in '%s'" name (str duration))
(db/exec-one! conn [sql:insert-new-task id name props queue priority max-retries interval])
id))
(defn- instrument!
[registry]
(mtx/instrument-vars!
[#'submit!]
{:registry registry
:type :counter
:labels ["name"]
:name "tasks_submit_total"
:help "A counter of task submissions."
:wrap (fn [rootf mobj]
(let [mdata (meta rootf)
origf (::original mdata rootf)]
(with-meta
(fn [conn params]
(let [tname (:name params)]
(mobj :inc [tname])
(origf conn params)))
{::original origf})))})
(mtx/instrument-vars!
[#'app.worker/run-task]
{:registry registry
:type :summary
:quantiles []
:name "tasks_checkout_timing"
:help "Latency measured between scheduld_at and execution time."
:wrap (fn [rootf mobj]
(let [mdata (meta rootf)
origf (::original mdata rootf)]
(with-meta
(fn [tasks item]
(let [now (inst-ms (dt/now))
sat (inst-ms (:scheduled-at item))]
(mobj :observe (- now sat))
(origf tasks item)))
{::original origf})))}))
;; --- STATE INIT: REGISTRY
(s/def ::tasks
(s/map-of keyword? fn?))
(defmethod ig/pre-init-spec ::registry [_]
(s/keys :req-un [::mtx/metrics ::tasks]))
(defmethod ig/init-key ::registry
[_ {:keys [metrics tasks]}]
(instrument! (:registry metrics))
(let [mobj (mtx/create
{:registry (:registry metrics)
:type :summary
:labels ["name"]
:quantiles []
:name "tasks_timing"
:help "Background task execution timing."})]
(reduce-kv (fn [res k v]
(let [tname (name k)]
(log/debugf "registring task '%s'" tname)
(assoc res tname (mtx/wrap-summary v mobj [tname]))))
{}
tasks)))

2
backend/src/app/tasks/sendmail.clj
View file

@ -5,7 +5,7 @@
;; This Source Code Form is "Incompatible With Secondary Licenses", as ;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0. ;; defined by the Mozilla Public License, v. 2.0.
;; ;;
;; Copyright (c) 2020-2021 UXBOX Labs SL ;; Copyright (c) UXBOX Labs SL
(ns app.tasks.sendmail (ns app.tasks.sendmail
(:require (:require

17
backend/src/app/tasks/telemetry.clj
View file

@ -5,13 +5,14 @@
;; This Source Code Form is "Incompatible With Secondary Licenses", as ;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0. ;; defined by the Mozilla Public License, v. 2.0.
;; ;;
;; Copyright (c) 2020 UXBOX Labs SL ;; Copyright (c) UXBOX Labs SL
(ns app.tasks.telemetry (ns app.tasks.telemetry
"A task that is reponsible to collect anonymous statistical "A task that is reponsible to collect anonymous statistical
information about the current instance and send it to the telemetry information about the current instance and send it to the telemetry
server." server."
(:require (:require
[app.common.data :as d]
[app.common.exceptions :as ex] [app.common.exceptions :as ex]
[app.common.spec :as us] [app.common.spec :as us]
[app.config :as cfg] [app.config :as cfg]
@ -32,7 +33,6 @@
(s/def ::sprops (s/def ::sprops
(s/keys :req-un [::instance-id])) (s/keys :req-un [::instance-id]))
(defmethod ig/pre-init-spec ::handler [_] (defmethod ig/pre-init-spec ::handler [_]
(s/keys :req-un [::db/pool ::version ::uri ::sprops])) (s/keys :req-un [::db/pool ::version ::uri ::sprops]))
@ -128,11 +128,16 @@
(defn- retrieve-stats (defn- retrieve-stats
[{:keys [conn version]}] [{:keys [conn version]}]
(merge (let [referer (if (cfg/get :telemetry-with-taiga)
{:version version "taiga"
:with-taiga (:telemetry-with-taiga cfg/config false) (cfg/get :telemetry-referer))]
(-> {:version version
:referer referer
:total-teams (retrieve-num-teams conn) :total-teams (retrieve-num-teams conn)
:total-projects (retrieve-num-projects conn) :total-projects (retrieve-num-projects conn)
:total-files (retrieve-num-files conn)} :total-files (retrieve-num-files conn)}
(d/merge
(retrieve-team-averages conn) (retrieve-team-averages conn)
(retrieve-jvm-stats))) (retrieve-jvm-stats))
(d/without-nils))))

121
backend/src/app/telemetry.clj
View file

@ -1,121 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
(ns app.telemetry
(:require
[app.common.spec :as us]
[app.db :as db]
[app.http.middleware :refer [wrap-parse-request-body]]
[clojure.pprint :refer [pprint]]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]
[promesa.exec :as px]
[ring.middleware.keyword-params :refer [wrap-keyword-params]]
[ring.middleware.params :refer [wrap-params]]))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Migrations
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def sql:create-instance-table
"CREATE TABLE IF NOT EXISTS telemetry.instance (
id uuid PRIMARY KEY,
created_at timestamptz NOT NULL DEFAULT now()
);")
(def sql:create-info-table
"CREATE TABLE telemetry.info (
instance_id uuid,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
data jsonb NOT NULL,
PRIMARY KEY (instance_id, created_at)
) PARTITION BY RANGE(created_at);
CREATE TABLE telemetry.info_default (LIKE telemetry.info INCLUDING ALL);
ALTER TABLE telemetry.info
ATTACH PARTITION telemetry.info_default DEFAULT;")
(def migrations
[{:name "0001-add-telemetry-schema"
:fn #(db/exec! % ["CREATE SCHEMA IF NOT EXISTS telemetry;"])}
{:name "0002-add-instance-table"
:fn #(db/exec! % [sql:create-instance-table])}
{:name "0003-add-info-table"
:fn #(db/exec! % [sql:create-info-table])}
{:name "0004-del-instance-table"
:fn #(db/exec! % ["DROP TABLE telemetry.instance;"])}])
(defmethod ig/init-key ::migrations [_ _] migrations)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Router Handler
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(declare handler)
(declare process-request)
(defmethod ig/init-key ::handler
[_ cfg]
(-> (partial handler cfg)
(wrap-keyword-params)
(wrap-params)
(wrap-parse-request-body)))
(s/def ::instance-id ::us/uuid)
(s/def ::params (s/keys :req-un [::instance-id]))
(defn handler
[{:keys [executor] :as cfg} {:keys [params] :as request}]
(try
(let [params (us/conform ::params params)
cfg (assoc cfg
:instance-id (:instance-id params)
:data (dissoc params :instance-id))]
(px/run! executor (partial process-request cfg)))
(catch Exception e
;; We don't want notify user of a error, just log it for posible
;; future investigation.
(log/warn e (str "unexpected error on telemetry:\n"
(when-let [edata (ex-data e)]
(str "ex-data: \n"
(with-out-str (pprint edata))))
(str "params: \n"
(with-out-str (pprint params)))))))
{:status 200
:body "OK\n"})
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Request Processing
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def sql:insert-instance-info
"insert into telemetry.info (instance_id, data, created_at)
values (?, ?, date_trunc('day', now()))
on conflict (instance_id, created_at)
do update set data = ?")
(defn- process-request
[{:keys [pool instance-id data]}]
(try
(db/with-atomic [conn pool]
(let [data (db/json data)]
(db/exec! conn [sql:insert-instance-info
instance-id
data
data])))
(catch Exception e
(log/errorf e "error on procesing request"))))

16
backend/src/app/util/blob.clj
View file

@ -5,13 +5,13 @@
;; This Source Code Form is "Incompatible With Secondary Licenses", as ;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0. ;; defined by the Mozilla Public License, v. 2.0.
;; ;;
;; Copyright (c) 2016-2020 Andrey Antukh <niwi@niwi.nz> ;; Copyright (c) UXBOX Labs SL
(ns app.util.blob (ns app.util.blob
"A generic blob storage encoding. Mainly used for "A generic blob storage encoding. Mainly used for page data, page
page data, page options and txlog payload storage." options and txlog payload storage."
(:require (:require
[app.config :as cfg] [app.config :as cf]
[app.util.transit :as t] [app.util.transit :as t]
[taoensso.nippy :as n]) [taoensso.nippy :as n])
(:import (:import
@ -33,17 +33,15 @@
(declare encode-v2) (declare encode-v2)
(declare encode-v3) (declare encode-v3)
(def default-version
(:default-blob-version cfg/config 1))
(defn encode (defn encode
([data] (encode data nil)) ([data] (encode data nil))
([data {:keys [version] :or {version default-version}}] ([data {:keys [version]}]
(let [version (or version (cf/get :default-blob-version 1))]
(case (long version) (case (long version)
1 (encode-v1 data) 1 (encode-v1 data)
2 (encode-v2 data) 2 (encode-v2 data)
3 (encode-v3 data) 3 (encode-v3 data)
(throw (ex-info "unsupported version" {:version version}))))) (throw (ex-info "unsupported version" {:version version}))))))
(defn decode (defn decode
"A function used for decode persisted blobs in the database." "A function used for decode persisted blobs in the database."

1
backend/src/app/util/time.clj
View file

@ -60,7 +60,6 @@
[t1 t2] [t1 t2]
(Duration/between t1 t2)) (Duration/between t1 t2))
(letfn [(conformer [v] (letfn [(conformer [v]
(cond (cond
(duration? v) v (duration? v) v

182
backend/src/app/worker.clj
View file

@ -5,15 +5,17 @@
;; This Source Code Form is "Incompatible With Secondary Licenses", as ;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0. ;; defined by the Mozilla Public License, v. 2.0.
;; ;;
;; Copyright (c) 2020 Andrey Antukh <niwi@niwi.nz> ;; Copyright (c) UXBOX Labs SL
(ns app.worker (ns app.worker
"Async tasks abstraction (impl)." "Async tasks abstraction (impl)."
(:require (:require
[app.common.data :as d]
[app.common.exceptions :as ex] [app.common.exceptions :as ex]
[app.common.spec :as us] [app.common.spec :as us]
[app.common.uuid :as uuid] [app.common.uuid :as uuid]
[app.db :as db] [app.db :as db]
[app.metrics :as mtx]
[app.util.async :as aa] [app.util.async :as aa]
[app.util.log4j :refer [update-thread-context!]] [app.util.log4j :refer [update-thread-context!]]
[app.util.time :as dt] [app.util.time :as dt]
@ -35,21 +37,13 @@
;; Executor ;; Executor
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(s/def ::name ::us/string) (s/def ::name keyword?)
(s/def ::min-threads ::us/integer) (s/def ::min-threads ::us/integer)
(s/def ::max-threads ::us/integer) (s/def ::max-threads ::us/integer)
(s/def ::idle-timeout ::us/integer) (s/def ::idle-timeout ::us/integer)
(defmethod ig/pre-init-spec ::executor [_] (defmethod ig/pre-init-spec ::executor [_]
(s/keys :opt-un [::min-threads ::max-threads ::idle-timeout ::name])) (s/keys :req-un [::min-threads ::max-threads ::idle-timeout ::name]))
(defmethod ig/prep-key ::executor
[_ cfg]
(merge {:min-threads 0
:max-threads 256
:idle-timeout 60000
:name "worker"}
cfg))
(defmethod ig/init-key ::executor (defmethod ig/init-key ::executor
[_ {:keys [min-threads max-threads idle-timeout name]}] [_ {:keys [min-threads max-threads idle-timeout name]}]
@ -57,28 +51,29 @@
(int min-threads) (int min-threads)
(int idle-timeout)) (int idle-timeout))
(.setStopTimeout 500) (.setStopTimeout 500)
(.setName name) (.setName (d/name name))
(.start))) (.start)))
(defmethod ig/halt-key! ::executor (defmethod ig/halt-key! ::executor
[_ instance] [_ instance]
(.stop ^QueuedThreadPool instance)) (.stop ^QueuedThreadPool instance))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Worker ;; Worker
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(declare event-loop-fn) (declare event-loop-fn)
(declare instrument-tasks)
(s/def ::queue ::us/string) (s/def ::queue keyword?)
(s/def ::parallelism ::us/integer) (s/def ::parallelism ::us/integer)
(s/def ::batch-size ::us/integer) (s/def ::batch-size ::us/integer)
(s/def ::tasks (s/map-of string? fn?)) (s/def ::tasks (s/map-of keyword? fn?))
(s/def ::poll-interval ::dt/duration) (s/def ::poll-interval ::dt/duration)
(defmethod ig/pre-init-spec ::worker [_] (defmethod ig/pre-init-spec ::worker [_]
(s/keys :req-un [::executor (s/keys :req-un [::executor
::mtx/metrics
::db/pool ::db/pool
::batch-size ::batch-size
::name ::name
@ -88,29 +83,29 @@
(defmethod ig/prep-key ::worker (defmethod ig/prep-key ::worker
[_ cfg] [_ cfg]
(merge {:batch-size 2 (d/merge {:batch-size 2
:name "worker" :name :worker
:poll-interval (dt/duration {:seconds 5}) :poll-interval (dt/duration {:seconds 5})
:queue "default"} :queue :default}
cfg)) (d/without-nils cfg)))
(defmethod ig/init-key ::worker (defmethod ig/init-key ::worker
[_ {:keys [pool poll-interval name queue] :as cfg}] [_ {:keys [pool poll-interval name queue] :as cfg}]
(log/infof "starting worker '%s' on queue '%s'" name queue) (log/infof "starting worker '%s' on queue '%s'" (d/name name) (d/name queue))
(let [cch (a/chan 1) (let [close-ch (a/chan 1)
poll-ms (inst-ms poll-interval)] poll-ms (inst-ms poll-interval)]
(a/go-loop [] (a/go-loop []
(let [[val port] (a/alts! [cch (event-loop-fn cfg)] :priority true)] (let [[val port] (a/alts! [close-ch (event-loop-fn cfg)] :priority true)]
(cond (cond
;; Terminate the loop if close channel is closed or ;; Terminate the loop if close channel is closed or
;; event-loop-fn returns nil. ;; event-loop-fn returns nil.
(or (= port cch) (nil? val)) (or (= port close-ch) (nil? val))
(log/infof "stop condition found; shutdown worker: '%s'" name) (log/infof "stop condition found; shutdown worker: '%s'" (d/name name))
(db/pool-closed? pool) (db/pool-closed? pool)
(do (do
(log/info "worker eventloop is aborted because pool is closed") (log/info "worker eventloop is aborted because pool is closed")
(a/close! cch)) (a/close! close-ch))
(and (instance? java.sql.SQLException val) (and (instance? java.sql.SQLException val)
(contains? #{"08003" "08006" "08001" "08004"} (.getSQLState ^java.sql.SQLException val))) (contains? #{"08003" "08006" "08001" "08004"} (.getSQLState ^java.sql.SQLException val)))
@ -143,13 +138,55 @@
(reify (reify
java.lang.AutoCloseable java.lang.AutoCloseable
(close [_] (close [_]
(a/close! cch))))) (a/close! close-ch)))))
(defmethod ig/halt-key! ::worker (defmethod ig/halt-key! ::worker
[_ instance] [_ instance]
(.close ^java.lang.AutoCloseable instance)) (.close ^java.lang.AutoCloseable instance))
;; --- SUBMIT
(s/def ::task keyword?)
(s/def ::delay (s/or :int ::us/integer :duration dt/duration?))
(s/def ::conn some?)
(s/def ::priority ::us/integer)
(s/def ::max-retries ::us/integer)
(s/def ::submit-options
(s/keys :req [::task ::conn]
:opt [::delay ::queue ::priority ::max-retries]))
(def ^:private sql:insert-new-task
"insert into task (id, name, props, queue, priority, max_retries, scheduled_at)
values (?, ?, ?, ?, ?, ?, clock_timestamp() + ?)
returning id")
(defn- extract-props
[options]
(persistent!
(reduce-kv (fn [res k v]
(cond-> res
(not (qualified-keyword? k))
(assoc! k v)))
(transient {})
options)))
(defn submit!
[{:keys [::task ::delay ::queue ::priority ::max-retries ::conn]
:or {delay 0 queue :default priority 100 max-retries 3}
:as options}]
(us/verify ::submit-options options)
(let [duration (dt/duration delay)
interval (db/interval duration)
props (-> options extract-props db/tjson)
id (uuid/next)]
(log/debugf "submit task '%s' to be executed in '%s'" (d/name task) (str duration))
(db/exec-one! conn [sql:insert-new-task id (d/name task) props (d/name queue) priority max-retries interval])
id))
;; --- RUNNER
(def ^:private (def ^:private
sql:mark-as-retry sql:mark-as-retry
@ -194,17 +231,18 @@
nil)) nil))
(defn- decode-task-row (defn- decode-task-row
[{:keys [props] :as row}] [{:keys [props name] :as row}]
(when row (when row
(cond-> row (cond-> row
(db/pgobject? props) (assoc :props (db/decode-transit-pgobject props))))) (db/pgobject? props) (assoc :props (db/decode-transit-pgobject props))
(string? name) (assoc :name (keyword name)))))
(defn- handle-task (defn- handle-task
[tasks {:keys [name] :as item}] [tasks {:keys [name] :as item}]
(let [task-fn (get tasks name)] (let [task-fn (get tasks name)]
(if task-fn (if task-fn
(task-fn item) (task-fn item)
(log/warnf "no task handler found for '%s'" (pr-str name))) (log/warnf "no task handler found for '%s'" (d/name name)))
{:status :completed :task item})) {:status :completed :task item}))
(defn get-error-context (defn get-error-context
@ -236,13 +274,14 @@
(defn- run-task (defn- run-task
[{:keys [tasks]} item] [{:keys [tasks]} item]
(let [name (d/name (:name item))]
(try (try
(log/debugf "started task '%s/%s/%s'" (:name item) (:id item) (:retry-num item)) (log/debugf "started task '%s/%s/%s'" name (:id item) (:retry-num item))
(handle-task tasks item) (handle-task tasks item)
(catch Exception e (catch Exception e
(handle-exception e item)) (handle-exception e item))
(finally (finally
(log/debugf "finished task '%s/%s/%s'" (:name item) (:id item) (:retry-num item))))) (log/debugf "finished task '%s/%s/%s'" name (:id item) (:retry-num item))))))
(def sql:select-next-tasks (def sql:select-next-tasks
"select * from task as t "select * from task as t
@ -256,7 +295,7 @@
(defn- event-loop-fn* (defn- event-loop-fn*
[{:keys [pool executor batch-size] :as cfg}] [{:keys [pool executor batch-size] :as cfg}]
(db/with-atomic [conn pool] (db/with-atomic [conn pool]
(let [queue (:queue cfg) (let [queue (name (:queue cfg))
items (->> (db/exec! conn [sql:select-next-tasks queue batch-size]) items (->> (db/exec! conn [sql:select-next-tasks queue batch-size])
(map decode-task-row) (map decode-task-row)
(seq)) (seq))
@ -288,16 +327,16 @@
(declare synchronize-schedule) (declare synchronize-schedule)
(s/def ::fn (s/or :var var? :fn fn?)) (s/def ::fn (s/or :var var? :fn fn?))
(s/def ::id ::us/string) (s/def ::id keyword?)
(s/def ::cron dt/cron?) (s/def ::cron dt/cron?)
(s/def ::props (s/nilable map?)) (s/def ::props (s/nilable map?))
(s/def ::task keyword?) (s/def ::task keyword?)
(s/def ::scheduled-task-spec (s/def ::scheduled-task
(s/keys :req-un [::id ::cron ::task] (s/keys :req-un [::cron ::task]
:opt-un [::props])) :opt-un [::props ::id]))
(s/def ::schedule (s/coll-of (s/nilable ::scheduled-task-spec))) (s/def ::schedule (s/coll-of (s/nilable ::scheduled-task)))
(defmethod ig/pre-init-spec ::scheduler [_] (defmethod ig/pre-init-spec ::scheduler [_]
(s/keys :req-un [::executor ::db/pool ::schedule ::tasks])) (s/keys :req-un [::executor ::db/pool ::schedule ::tasks]))
@ -307,8 +346,13 @@
(let [scheduler (Executors/newScheduledThreadPool (int 1)) (let [scheduler (Executors/newScheduledThreadPool (int 1))
schedule (->> schedule schedule (->> schedule
(filter some?) (filter some?)
;; If id is not defined, use the task as id.
(map (fn [{:keys [id task] :as item}]
(if (some? id)
item
(assoc item :id task))))
(map (fn [{:keys [task] :as item}] (map (fn [{:keys [task] :as item}]
(let [f (get tasks (name task))] (let [f (get tasks task)]
(when-not f (when-not f
(ex/raise :type :internal (ex/raise :type :internal
:code :task-not-found :code :task-not-found
@ -341,7 +385,8 @@
(defn- synchronize-schedule-item (defn- synchronize-schedule-item
[conn {:keys [id cron]}] [conn {:keys [id cron]}]
(let [cron (str cron)] (let [cron (str cron)
id (name id)]
(log/infof "initialize scheduled task '%s' (cron: '%s')" id cron) (log/infof "initialize scheduled task '%s' (cron: '%s')" id cron)
(db/exec-one! conn [sql:upsert-scheduled-task id cron cron]))) (db/exec-one! conn [sql:upsert-scheduled-task id cron cron])))
@ -390,3 +435,62 @@
[{:keys [scheduler] :as cfg} {:keys [cron] :as task}] [{:keys [scheduler] :as cfg} {:keys [cron] :as task}]
(let [ms (ms-until-valid cron)] (let [ms (ms-until-valid cron)]
(px/schedule! scheduler ms (partial execute-scheduled-task cfg task)))) (px/schedule! scheduler ms (partial execute-scheduled-task cfg task))))
;; --- INSTRUMENTATION
(defn instrument!
[registry]
(mtx/instrument-vars!
[#'submit!]
{:registry registry
:type :counter
:labels ["name"]
:name "tasks_submit_total"
:help "A counter of task submissions."
:wrap (fn [rootf mobj]
(let [mdata (meta rootf)
origf (::original mdata rootf)]
(with-meta
(fn [conn params]
(let [tname (:name params)]
(mobj :inc [tname])
(origf conn params)))
{::original origf})))})
(mtx/instrument-vars!
[#'app.worker/run-task]
{:registry registry
:type :summary
:quantiles []
:name "tasks_checkout_timing"
:help "Latency measured between scheduld_at and execution time."
:wrap (fn [rootf mobj]
(let [mdata (meta rootf)
origf (::original mdata rootf)]
(with-meta
(fn [tasks item]
(let [now (inst-ms (dt/now))
sat (inst-ms (:scheduled-at item))]
(mobj :observe (- now sat))
(origf tasks item)))
{::original origf})))}))
(defmethod ig/pre-init-spec ::registry [_]
(s/keys :req-un [::mtx/metrics ::tasks]))
(defmethod ig/init-key ::registry
[_ {:keys [metrics tasks]}]
(let [mobj (mtx/create
{:registry (:registry metrics)
:type :summary
:labels ["name"]
:quantiles []
:name "tasks_timing"
:help "Background task execution timing."})]
(reduce-kv (fn [res k v]
(let [tname (name k)]
(log/debugf "registring task '%s'" tname)
(assoc res k (mtx/wrap-summary v mobj [tname]))))
{}
tasks)))

20
backend/tests/app/tests/helpers.clj
View file

@ -13,7 +13,7 @@
[app.common.pages :as cp] [app.common.pages :as cp]
[app.common.spec :as us] [app.common.spec :as us]
[app.common.uuid :as uuid] [app.common.uuid :as uuid]
[app.config :as cfg] [app.config :as cf]
[app.db :as db] [app.db :as db]
[app.main :as main] [app.main :as main]
[app.media] [app.media]
@ -38,16 +38,12 @@
(def ^:dynamic *system* nil) (def ^:dynamic *system* nil)
(def ^:dynamic *pool* nil) (def ^:dynamic *pool* nil)
(def config
(merge {:redis-uri "redis://redis/1"
:database-uri "postgresql://postgres/penpot_test"
:storage-fs-directory "/tmp/app/storage"
:migrations-verbose false}
cfg/config))
(defn state-init (defn state-init
[next] [next]
(let [config (-> (main/build-system-config config) (let [config (-> main/system-config
(assoc-in [:app.msgbus/msgbus :redis-uri] "redis://redis/1")
(assoc-in [:app.db/pool :uri] "postgresql://postgres/penpot_test")
(assoc-in [[:app.main/main :app.storage.fs/backend] :directory] "/tmp/app/storage")
(dissoc :app.srepl/server (dissoc :app.srepl/server
:app.http/server :app.http/server
:app.http/router :app.http/router
@ -328,8 +324,10 @@
"Helper for mock app.config/get" "Helper for mock app.config/get"
[data] [data]
(fn (fn
([key] (get (merge config data) key)) ([key]
([key default] (get (merge config data) key default)))) (get data key (cf/get key)))
([key default]
(get data key (cf/get key default)))))
(defn reset-mock! (defn reset-mock!
[m] [m]

3
common/app/common/data.cljc
View file

@ -401,6 +401,9 @@
(keyword? maybe-keyword) (keyword? maybe-keyword)
(core/name maybe-keyword) (core/name maybe-keyword)
(string? maybe-keyword)
maybe-keyword
(nil? maybe-keyword) default-value (nil? maybe-keyword) default-value
:else :else

2
common/app/common/exceptions.cljc
View file

@ -2,7 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this ;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/. ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;; ;;
;; Copyright (c) 2016 Andrey Antukh <niwi@niwi.nz> ;; Copyright (c) Andrey Antukh <niwi@niwi.nz>
(ns app.common.exceptions (ns app.common.exceptions
"A helpers for work with exceptions." "A helpers for work with exceptions."