diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index c572c1ada..e9aaeb7e7 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -24,7 +24,6 @@ :database-uri "postgresql://127.0.0.1/penpot" :database-username "penpot" :database-password "penpot" - :secret-key "default" :asserts-enabled true :public-uri "http://localhost:3449" @@ -92,7 +91,6 @@ (s/def ::media-uri ::us/string) (s/def ::media-directory ::us/string) -(s/def ::secret-key ::us/string) (s/def ::asserts-enabled ::us/boolean) (s/def ::error-report-webhook ::us/string) @@ -141,6 +139,7 @@ (s/def ::ldap-auth-avatar-attribute ::us/string) (s/def ::telemetry-enabled ::us/boolean) +(s/def ::telemetry-with-taiga ::us/boolean) (s/def ::telemetry-uri ::us/string) (s/def ::telemetry-server-enabled ::us/boolean) (s/def ::telemetry-server-port ::us/integer) @@ -182,7 +181,6 @@ ::redis-uri ::registration-domain-whitelist ::registration-enabled - ::secret-key ::rlimits-password ::rlimits-image ::smtp-default-from @@ -202,6 +200,7 @@ ::storage-s3-bucket ::storage-s3-region ::telemetry-enabled + ::telemetry-with-taiga ::telemetry-server-enabled ::telemetry-server-port ::telemetry-uri diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index 970a77ec2..fad23167e 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -53,7 +53,7 @@ {:uri (:redis-uri config)} :app.tokens/tokens - {:secret-key (:secret-key config)} + {:sprops (ig/ref :app.sprops/props)} :app.storage/gc-task {:pool (ig/ref :app.db/pool) @@ -96,43 +96,6 @@ :cache-max-age (dt/duration {:hours 24}) :signature-max-age (dt/duration {:hours 24 :minutes 5})} - :app.svgparse/svgc - {:metrics (ig/ref :app.metrics/metrics)} - - ;; HTTP Handler for SVG parsing - :app.svgparse/handler - {:metrics (ig/ref :app.metrics/metrics) - :svgc (ig/ref :app.svgparse/svgc)} - - ;; RLimit definition for password hashing - :app.rlimits/password - (:rlimits-password cfg/config) - - ;; RLimit definition for image processing - :app.rlimits/image - (:rlimits-image cfg/config) - - ;; A collection of rlimits as hash-map. - :app.rlimits/all - {:password (ig/ref :app.rlimits/password) - :image (ig/ref :app.rlimits/image)} - - :app.rpc/rpc - {:pool (ig/ref :app.db/pool) - :session (ig/ref :app.http.session/session) - :tokens (ig/ref :app.tokens/tokens) - :metrics (ig/ref :app.metrics/metrics) - :storage (ig/ref :app.storage/storage) - :redis (ig/ref :app.redis/redis) - :rlimits (ig/ref :app.rlimits/all) - :svgc (ig/ref :app.svgparse/svgc)} - - :app.notifications/handler - {:redis (ig/ref :app.redis/redis) - :pool (ig/ref :app.db/pool) - :session (ig/ref :app.http.session/session) - :metrics (ig/ref :app.metrics/metrics)} - :app.http.auth/google {:rpc (ig/ref :app.rpc/rpc) :session (ig/ref :app.http.session/session) @@ -172,6 +135,43 @@ :session (ig/ref :app.http.session/session) :rpc (ig/ref :app.rpc/rpc)} + :app.svgparse/svgc + {:metrics (ig/ref :app.metrics/metrics)} + + ;; HTTP Handler for SVG parsing + :app.svgparse/handler + {:metrics (ig/ref :app.metrics/metrics) + :svgc (ig/ref :app.svgparse/svgc)} + + ;; RLimit definition for password hashing + :app.rlimits/password + (:rlimits-password cfg/config) + + ;; RLimit definition for image processing + :app.rlimits/image + (:rlimits-image cfg/config) + + ;; A collection of rlimits as hash-map. + :app.rlimits/all + {:password (ig/ref :app.rlimits/password) + :image (ig/ref :app.rlimits/image)} + + :app.rpc/rpc + {:pool (ig/ref :app.db/pool) + :session (ig/ref :app.http.session/session) + :tokens (ig/ref :app.tokens/tokens) + :metrics (ig/ref :app.metrics/metrics) + :storage (ig/ref :app.storage/storage) + :redis (ig/ref :app.redis/redis) + :rlimits (ig/ref :app.rlimits/all) + :svgc (ig/ref :app.svgparse/svgc)} + + :app.notifications/handler + {:redis (ig/ref :app.redis/redis) + :pool (ig/ref :app.db/pool) + :session (ig/ref :app.http.session/session) + :metrics (ig/ref :app.metrics/metrics)} + :app.worker/executor {:name "worker"} @@ -205,8 +205,8 @@ :fn (ig/ref :app.tasks.tasks-gc/handler)} (when (:telemetry-enabled cfg/config) - {:id "telemetry" - :cron #app/cron "0 0 */3 * * ?" ;; every 3h + {:id "telemetry" + :cron #app/cron "0 0 */6 * * ?" ;; every 6h :uri (:telemetry-uri cfg/config) :fn (ig/ref :app.tasks.telemetry/handler)})]} @@ -259,12 +259,16 @@ :app.tasks.telemetry/handler {:pool (ig/ref :app.db/pool) :version (:full cfg/version) - :uri (:telemetry-uri cfg/config)} + :uri (:telemetry-uri cfg/config) + :sprops (ig/ref :app.sprops/props)} :app.srepl/server {:port (:srepl-port cfg/config) :host (:srepl-host cfg/config)} + :app.sprops/props + {:pool (ig/ref :app.db/pool)} + :app.error-reporter/reporter {:uri (:error-report-webhook cfg/config) :pool (ig/ref :app.db/pool) diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index 7ca57c23c..1be3db2cc 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -137,6 +137,9 @@ {:name "0041-mod-pg-storage-options" :fn (mg/resource "app/migrations/sql/0041-mod-pg-storage-options.sql")} + + {:name "0042-add-server-prop-table" + :fn (mg/resource "app/migrations/sql/0042-add-server-prop-table.sql")} ]) diff --git a/backend/src/app/migrations/sql/0042-add-server-prop-table.sql b/backend/src/app/migrations/sql/0042-add-server-prop-table.sql new file mode 100644 index 000000000..a291b098b --- /dev/null +++ b/backend/src/app/migrations/sql/0042-add-server-prop-table.sql @@ -0,0 +1,8 @@ +CREATE TABLE server_prop ( + id text PRIMARY KEY, + content jsonb +); + +ALTER TABLE server_prop + ALTER COLUMN id SET STORAGE external, + ALTER COLUMN content SET STORAGE external; diff --git a/backend/src/app/sprops.clj b/backend/src/app/sprops.clj new file mode 100644 index 000000000..252e42ef8 --- /dev/null +++ b/backend/src/app/sprops.clj @@ -0,0 +1,63 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; This Source Code Form is "Incompatible With Secondary Licenses", as +;; defined by the Mozilla Public License, v. 2.0. +;; +;; Copyright (c) 2020-2021 UXBOX Labs SL + +(ns app.sprops + "Server props module." + (:require + [app.common.exceptions :as ex] + [app.common.spec :as us] + [app.common.uuid :as uuid] + [app.config :as cfg] + [app.db :as db] + [app.util.time :as dt] + [buddy.core.codecs :as bc] + [buddy.core.nonce :as bn] + [clojure.spec.alpha :as s] + [clojure.tools.logging :as log] + [integrant.core :as ig])) + +(declare initialize) + +(defmethod ig/pre-init-spec ::props [_] + (s/keys :req-un [::db/pool])) + +(defmethod ig/init-key ::props + [_ cfg] + (initialize cfg)) + +(defn- initialize-secret-key + [{:keys [conn] :as cfg}] + (let [key (-> (bn/random-bytes 64) + (bc/bytes->b64u) + (bc/bytes->str))] + (db/exec-one! conn ["insert into server_prop (id, content) + values ('secret-key', ?) on conflict do nothing" + (db/tjson key)]))) + +(defn- initialize-instance-id + [{:keys [conn] :as cfg}] + (let [iid (uuid/random)] + (db/exec-one! conn ["insert into server_prop (id, content) + values ('instance-id', ?::jsonb) on conflict do nothing" + (db/tjson iid)]))) + +(defn- retrieve-all + [{:keys [conn] :as cfg}] + (reduce (fn [acc row] + (assoc acc (keyword (:id row)) (db/decode-transit-pgobject (:content row)))) + {} + (db/exec! conn ["select * from server_prop;"]))) + +(defn- initialize + [{:keys [pool] :as cfg}] + (db/with-atomic [conn pool] + (let [cfg (assoc cfg :conn conn)] + (initialize-secret-key cfg) + (initialize-instance-id cfg) + (retrieve-all cfg)))) diff --git a/backend/src/app/tasks/telemetry.clj b/backend/src/app/tasks/telemetry.clj index e278a7fe4..e29cdacdb 100644 --- a/backend/src/app/tasks/telemetry.clj +++ b/backend/src/app/tasks/telemetry.clj @@ -12,6 +12,7 @@ information about the current instance and send it to the telemetry server." (:require + [app.config :as cfg] [app.common.exceptions :as ex] [app.common.spec :as us] [app.common.uuid :as uuid] @@ -29,9 +30,13 @@ (s/def ::version ::us/string) (s/def ::uri ::us/string) +(s/def ::instance-id ::us/uuid) +(s/def ::sprops + (s/keys :req-un [::instance-id])) + (defmethod ig/pre-init-spec ::handler [_] - (s/keys :req-un [::db/pool ::version ::uri])) + (s/keys :req-un [::db/pool ::version ::uri ::sprops])) (defmethod ig/init-key ::handler [_ {:keys [pool] :as cfg}] @@ -51,22 +56,11 @@ [conn] (db/exec-one! conn ["select pg_advisory_unlock_all();"])) -(defn- get-or-create-instance-id - [{:keys [conn] :as cfg}] - (if-let [result (db/exec-one! conn ["select id from telemetry.instance"])] - (:id result) - (let [result (db/exec-one! conn ["insert into telemetry.instance (id) values (?) returning *" - (uuid/random)])] - (:id result)))) - -(defonce debug {}) - (defn- handler - [cfg] - (let [instance-id (get-or-create-instance-id cfg) + [{:keys [sprops] :as cfg}] + (let [instance-id (:instance-id sprops) data (retrieve-stats cfg) data (assoc data :instance-id instance-id)] - (alter-var-root #'debug (constantly data)) (let [response (http/send! {:method :post :uri (:uri cfg) :headers {"content-type" "application/json"} @@ -77,7 +71,6 @@ :context {:status (:status response) :body (:body response)}))))) - (defn retrieve-num-teams [conn] (-> (db/exec-one! conn ["select count(*) as count from team;"]) :count)) @@ -138,6 +131,7 @@ [{:keys [conn version]}] (merge {:version version + :with-taiga (:telemetry-with-taiga cfg/config) :total-teams (retrieve-num-teams conn) :total-projects (retrieve-num-projects conn) :total-files (retrieve-num-files conn)} diff --git a/backend/src/app/telemetry.clj b/backend/src/app/telemetry.clj index c79ce5ffc..4bce0f038 100644 --- a/backend/src/app/telemetry.clj +++ b/backend/src/app/telemetry.clj @@ -9,13 +9,13 @@ (ns app.telemetry (:require - [clojure.tools.logging :as log] [app.common.spec :as us] [app.db :as db] [app.http.middleware :refer [wrap-parse-request-body wrap-errors]] - [promesa.exec :as px] [clojure.spec.alpha :as s] + [clojure.tools.logging :as log] [integrant.core :as ig] + [promesa.exec :as px] [ring.middleware.keyword-params :refer [wrap-keyword-params]] [ring.middleware.params :refer [wrap-params]])) @@ -52,11 +52,13 @@ :fn #(db/exec! % [sql:create-instance-table])} {:name "0003-add-info-table" - :fn #(db/exec! % [sql:create-info-table])}]) + :fn #(db/exec! % [sql:create-info-table])} + + {:name "0004-del-instance-table" + :fn #(db/exec! % ["DROP TABLE telemetry.instance;"])}]) (defmethod ig/init-key ::migrations [_ _] migrations) - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Router Handler ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/backend/src/app/tokens.clj b/backend/src/app/tokens.clj index 6885b858a..e83a4a36c 100644 --- a/backend/src/app/tokens.clj +++ b/backend/src/app/tokens.clj @@ -5,7 +5,7 @@ ;; This Source Code Form is "Incompatible With Secondary Licenses", as ;; defined by the Mozilla Public License, v. 2.0. ;; -;; Copyright (c) 2020 UXBOX Labs SL +;; Copyright (c) 2020-2021 UXBOX Labs SL (ns app.tokens "Tokens generation service." @@ -23,8 +23,6 @@ (defn- derive-tokens-secret [key] - (when (= key "default") - (log/warn "Using default PENPOT_SECRET_KEY, the system will generate insecure tokens.")) (let [engine (bk/engine {:key key :salt "tokens" :alg :hkdf @@ -57,14 +55,16 @@ :params params)) claims)) -(s/def ::secret-key ::us/not-empty-string) - -(defmethod ig/pre-init-spec ::tokens [_] +(s/def ::secret-key ::us/string) +(s/def ::sprops (s/keys :req-un [::secret-key])) +(defmethod ig/pre-init-spec ::tokens [_] + (s/keys :req-un [::sprops])) + (defmethod ig/init-key ::tokens - [_ cfg] - (let [secret (derive-tokens-secret (:secret-key cfg)) + [_ {:keys [sprops] :as cfg}] + (let [secret (derive-tokens-secret (:secret-key sprops)) cfg (assoc cfg ::secret secret)] (fn [action params] (case action