✨ Simplify internal props handling and telemetry.

backend/src/app/config.clj

@@ -24,7 +24,6 @@
    :database-uri "postgresql://127.0.0.1/penpot"
    :database-username "penpot"
    :database-password "penpot"
-   :secret-key "default"
    :asserts-enabled true
 
    :public-uri "http://localhost:3449"
@@ -92,7 +91,6 @@
 
 (s/def ::media-uri ::us/string)
 (s/def ::media-directory ::us/string)
-(s/def ::secret-key ::us/string)
 (s/def ::asserts-enabled ::us/boolean)
 
 (s/def ::error-report-webhook ::us/string)
@@ -141,6 +139,7 @@
 (s/def ::ldap-auth-avatar-attribute ::us/string)
 
 (s/def ::telemetry-enabled ::us/boolean)
+(s/def ::telemetry-with-taiga ::us/boolean)
 (s/def ::telemetry-uri ::us/string)
 (s/def ::telemetry-server-enabled ::us/boolean)
 (s/def ::telemetry-server-port ::us/integer)
@@ -182,7 +181,6 @@
                    ::redis-uri
                    ::registration-domain-whitelist
                    ::registration-enabled
-                   ::secret-key
                    ::rlimits-password
                    ::rlimits-image
                    ::smtp-default-from
@@ -202,6 +200,7 @@
                    ::storage-s3-bucket
                    ::storage-s3-region
                    ::telemetry-enabled
+                   ::telemetry-with-taiga
                    ::telemetry-server-enabled
                    ::telemetry-server-port
                    ::telemetry-uri

backend/src/app/main.clj

@@ -53,7 +53,7 @@
     {:uri (:redis-uri config)}
 
     :app.tokens/tokens
-    {:secret-key (:secret-key config)}
+    {:sprops (ig/ref :app.sprops/props)}
 
     :app.storage/gc-task
     {:pool     (ig/ref :app.db/pool)
@@ -96,43 +96,6 @@
      :cache-max-age     (dt/duration {:hours 24})
      :signature-max-age (dt/duration {:hours 24 :minutes 5})}
 
-    :app.svgparse/svgc
-    {:metrics (ig/ref :app.metrics/metrics)}
-
-    ;; HTTP Handler for SVG parsing
-    :app.svgparse/handler
-    {:metrics (ig/ref :app.metrics/metrics)
-     :svgc    (ig/ref :app.svgparse/svgc)}
-
-    ;; RLimit definition for password hashing
-    :app.rlimits/password
-    (:rlimits-password cfg/config)
-
-    ;; RLimit definition for image processing
-    :app.rlimits/image
-    (:rlimits-image cfg/config)
-
-    ;; A collection of rlimits as hash-map.
-    :app.rlimits/all
-    {:password (ig/ref :app.rlimits/password)
-     :image    (ig/ref :app.rlimits/image)}
-
-    :app.rpc/rpc
-    {:pool    (ig/ref :app.db/pool)
-     :session (ig/ref :app.http.session/session)
-     :tokens  (ig/ref :app.tokens/tokens)
-     :metrics (ig/ref :app.metrics/metrics)
-     :storage (ig/ref :app.storage/storage)
-     :redis   (ig/ref :app.redis/redis)
-     :rlimits (ig/ref :app.rlimits/all)
-     :svgc    (ig/ref :app.svgparse/svgc)}
-
-    :app.notifications/handler
-    {:redis   (ig/ref :app.redis/redis)
-     :pool    (ig/ref :app.db/pool)
-     :session (ig/ref :app.http.session/session)
-     :metrics (ig/ref :app.metrics/metrics)}
-
     :app.http.auth/google
     {:rpc           (ig/ref :app.rpc/rpc)
      :session       (ig/ref :app.http.session/session)
@@ -172,6 +135,43 @@
      :session            (ig/ref :app.http.session/session)
      :rpc                (ig/ref :app.rpc/rpc)}
 
+    :app.svgparse/svgc
+    {:metrics (ig/ref :app.metrics/metrics)}
+
+    ;; HTTP Handler for SVG parsing
+    :app.svgparse/handler
+    {:metrics (ig/ref :app.metrics/metrics)
+     :svgc    (ig/ref :app.svgparse/svgc)}
+
+    ;; RLimit definition for password hashing
+    :app.rlimits/password
+    (:rlimits-password cfg/config)
+
+    ;; RLimit definition for image processing
+    :app.rlimits/image
+    (:rlimits-image cfg/config)
+
+    ;; A collection of rlimits as hash-map.
+    :app.rlimits/all
+    {:password (ig/ref :app.rlimits/password)
+     :image    (ig/ref :app.rlimits/image)}
+
+    :app.rpc/rpc
+    {:pool    (ig/ref :app.db/pool)
+     :session (ig/ref :app.http.session/session)
+     :tokens  (ig/ref :app.tokens/tokens)
+     :metrics (ig/ref :app.metrics/metrics)
+     :storage (ig/ref :app.storage/storage)
+     :redis   (ig/ref :app.redis/redis)
+     :rlimits (ig/ref :app.rlimits/all)
+     :svgc    (ig/ref :app.svgparse/svgc)}
+
+    :app.notifications/handler
+    {:redis   (ig/ref :app.redis/redis)
+     :pool    (ig/ref :app.db/pool)
+     :session (ig/ref :app.http.session/session)
+     :metrics (ig/ref :app.metrics/metrics)}
+
     :app.worker/executor
     {:name "worker"}
 
@@ -205,8 +205,8 @@
        :fn (ig/ref :app.tasks.tasks-gc/handler)}
 
       (when (:telemetry-enabled cfg/config)
-        {:id "telemetry"
+        {:id   "telemetry"
-         :cron #app/cron "0 0 */3 * * ?" ;; every 3h
+         :cron #app/cron "0 0 */6 * * ?" ;; every 6h
          :uri  (:telemetry-uri cfg/config)
          :fn   (ig/ref :app.tasks.telemetry/handler)})]}
 
@@ -259,12 +259,16 @@
     :app.tasks.telemetry/handler
     {:pool        (ig/ref :app.db/pool)
      :version     (:full cfg/version)
-     :uri         (:telemetry-uri cfg/config)}
+     :uri         (:telemetry-uri cfg/config)
+     :sprops      (ig/ref :app.sprops/props)}
 
     :app.srepl/server
     {:port (:srepl-port cfg/config)
      :host (:srepl-host cfg/config)}
 
+    :app.sprops/props
+    {:pool (ig/ref :app.db/pool)}
+
     :app.error-reporter/reporter
     {:uri      (:error-report-webhook cfg/config)
      :pool     (ig/ref :app.db/pool)

backend/src/app/migrations.clj

@@ -137,6 +137,9 @@
 
    {:name "0041-mod-pg-storage-options"
     :fn (mg/resource "app/migrations/sql/0041-mod-pg-storage-options.sql")}
+
+   {:name "0042-add-server-prop-table"
+    :fn (mg/resource "app/migrations/sql/0042-add-server-prop-table.sql")}
    ])
 
 

backend/src/app/migrations/sql/0042-add-server-prop-table.sql

@@ -0,0 +1,8 @@
+CREATE TABLE server_prop (
+  id text PRIMARY KEY,
+  content jsonb
+);
+
+ALTER TABLE server_prop
+  ALTER COLUMN id SET STORAGE external,
+  ALTER COLUMN content SET STORAGE external;

backend/src/app/sprops.clj

@@ -0,0 +1,63 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; This Source Code Form is "Incompatible With Secondary Licenses", as
+;; defined by the Mozilla Public License, v. 2.0.
+;;
+;; Copyright (c) 2020-2021 UXBOX Labs SL
+
+(ns app.sprops
+  "Server props module."
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.spec :as us]
+   [app.common.uuid :as uuid]
+   [app.config :as cfg]
+   [app.db :as db]
+   [app.util.time :as dt]
+   [buddy.core.codecs :as bc]
+   [buddy.core.nonce :as bn]
+   [clojure.spec.alpha :as s]
+   [clojure.tools.logging :as log]
+   [integrant.core :as ig]))
+
+(declare initialize)
+
+(defmethod ig/pre-init-spec ::props [_]
+  (s/keys :req-un [::db/pool]))
+
+(defmethod ig/init-key ::props
+  [_ cfg]
+  (initialize cfg))
+
+(defn- initialize-secret-key
+  [{:keys [conn] :as cfg}]
+  (let [key (-> (bn/random-bytes 64)
+                (bc/bytes->b64u)
+                (bc/bytes->str))]
+    (db/exec-one! conn ["insert into server_prop (id, content)
+                         values ('secret-key', ?) on conflict do nothing"
+                        (db/tjson key)])))
+
+(defn- initialize-instance-id
+  [{:keys [conn] :as cfg}]
+  (let [iid (uuid/random)]
+    (db/exec-one! conn ["insert into server_prop (id, content)
+                         values ('instance-id', ?::jsonb) on conflict do nothing"
+                        (db/tjson iid)])))
+
+(defn- retrieve-all
+  [{:keys [conn] :as cfg}]
+  (reduce (fn [acc row]
+            (assoc acc (keyword (:id row)) (db/decode-transit-pgobject (:content row))))
+          {}
+          (db/exec! conn ["select * from server_prop;"])))
+
+(defn- initialize
+  [{:keys [pool] :as cfg}]
+  (db/with-atomic [conn pool]
+    (let [cfg (assoc cfg :conn conn)]
+      (initialize-secret-key cfg)
+      (initialize-instance-id cfg)
+      (retrieve-all cfg))))

backend/src/app/tasks/telemetry.clj

@@ -12,6 +12,7 @@
   information about the current instance and send it to the telemetry
   server."
   (:require
+   [app.config :as cfg]
    [app.common.exceptions :as ex]
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
@@ -29,9 +30,13 @@
 
 (s/def ::version ::us/string)
 (s/def ::uri ::us/string)
+(s/def ::instance-id ::us/uuid)
+(s/def ::sprops
+  (s/keys :req-un [::instance-id]))
+
 
 (defmethod ig/pre-init-spec ::handler [_]
-  (s/keys :req-un [::db/pool ::version ::uri]))
+  (s/keys :req-un [::db/pool ::version ::uri ::sprops]))
 
 (defmethod ig/init-key ::handler
   [_ {:keys [pool] :as cfg}]
@@ -51,22 +56,11 @@
   [conn]
   (db/exec-one! conn ["select pg_advisory_unlock_all();"]))
 
-(defn- get-or-create-instance-id
-  [{:keys [conn] :as cfg}]
-  (if-let [result (db/exec-one! conn ["select id from telemetry.instance"])]
-    (:id result)
-    (let [result (db/exec-one! conn ["insert into telemetry.instance (id) values (?) returning *"
-                                     (uuid/random)])]
-      (:id result))))
-
-(defonce debug {})
-
 (defn- handler
-  [cfg]
+  [{:keys [sprops] :as cfg}]
-  (let [instance-id (get-or-create-instance-id cfg)
+  (let [instance-id (:instance-id sprops)
         data        (retrieve-stats cfg)
         data        (assoc data :instance-id instance-id)]
-    (alter-var-root #'debug (constantly data))
     (let [response (http/send! {:method :post
                                 :uri (:uri cfg)
                                 :headers {"content-type" "application/json"}
@@ -77,7 +71,6 @@
                   :context {:status (:status response)
                             :body (:body response)})))))
 
-
 (defn retrieve-num-teams
   [conn]
   (-> (db/exec-one! conn ["select count(*) as count from team;"]) :count))
@@ -138,6 +131,7 @@
   [{:keys [conn version]}]
   (merge
    {:version version
+    :with-taiga  (:telemetry-with-taiga cfg/config)
     :total-teams (retrieve-num-teams conn)
     :total-projects (retrieve-num-projects conn)
     :total-files (retrieve-num-files conn)}

backend/src/app/telemetry.clj

@@ -9,13 +9,13 @@
 
 (ns app.telemetry
   (:require
-   [clojure.tools.logging :as log]
    [app.common.spec :as us]
    [app.db :as db]
    [app.http.middleware :refer [wrap-parse-request-body wrap-errors]]
-   [promesa.exec :as px]
    [clojure.spec.alpha :as s]
+   [clojure.tools.logging :as log]
    [integrant.core :as ig]
+   [promesa.exec :as px]
    [ring.middleware.keyword-params :refer [wrap-keyword-params]]
    [ring.middleware.params :refer [wrap-params]]))
 
@@ -52,11 +52,13 @@
     :fn #(db/exec! % [sql:create-instance-table])}
 
    {:name "0003-add-info-table"
-    :fn #(db/exec! % [sql:create-info-table])}])
+    :fn #(db/exec! % [sql:create-info-table])}
+
+   {:name "0004-del-instance-table"
+    :fn #(db/exec! % ["DROP TABLE telemetry.instance;"])}])
 
 (defmethod ig/init-key ::migrations [_ _] migrations)
 
-
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Router Handler
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

backend/src/app/tokens.clj

@@ -5,7 +5,7 @@
 ;; This Source Code Form is "Incompatible With Secondary Licenses", as
 ;; defined by the Mozilla Public License, v. 2.0.
 ;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) 2020-2021 UXBOX Labs SL
 
 (ns app.tokens
   "Tokens generation service."
@@ -23,8 +23,6 @@
 
 (defn- derive-tokens-secret
   [key]
-  (when (= key "default")
-    (log/warn "Using default PENPOT_SECRET_KEY, the system will generate insecure tokens."))
   (let [engine (bk/engine {:key key
                            :salt "tokens"
                            :alg :hkdf
@@ -57,14 +55,16 @@
                 :params params))
     claims))
 
-(s/def ::secret-key ::us/not-empty-string)
+(s/def ::secret-key ::us/string)
-
+(s/def ::sprops
-(defmethod ig/pre-init-spec ::tokens [_]
   (s/keys :req-un [::secret-key]))
 
+(defmethod ig/pre-init-spec ::tokens [_]
+  (s/keys :req-un [::sprops]))
+
 (defmethod ig/init-key ::tokens
-  [_ cfg]
+  [_ {:keys [sprops] :as cfg}]
-  (let [secret (derive-tokens-secret (:secret-key cfg))
+  (let [secret (derive-tokens-secret (:secret-key sprops))
         cfg    (assoc cfg ::secret secret)]
     (fn [action params]
       (case action