♻️ Unify flags parsing on backend.

backend/scripts/repl

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-export PENPOT_ASSERTS_ENABLED=true
+export PENPOT_FLAGS="enable-asserts $PENPOT_FLAGS"
 
 export OPTIONS="-A:jmx-remote:dev -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -J-Dlog4j2.configurationFile=log4j2-devenv.xml -J-Djdk.attach.allowAttachSelf -J-XX:+UseZGC -J-XX:ConcGCThreads=1 -J-XX:-OmitStackTraceInFastThrow -J-Xms50m -J-Xmx512m";
 # export OPTIONS="$OPTIONS -J-XX:+UnlockDiagnosticVMOptions";

backend/src/app/config.clj

@@ -10,6 +10,7 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.flags :as flags]
    [app.common.spec :as us]
    [app.common.version :as v]
    [app.util.time :as dt]
@@ -50,8 +51,6 @@
    :default-blob-version 3
    :loggers-zmq-uri "tcp://localhost:45556"
 
-   :asserts-enabled false
-
    :public-uri "http://localhost:3449"
    :redis-uri "redis://redis/0"
 
@@ -61,15 +60,11 @@
    :assets-storage-backend :assets-fs
    :storage-assets-fs-directory "assets"
 
-   :feedback-destination "info@example.com"
-   :feedback-enabled false
-
    :assets-path "/internal/assets/"
 
    :rlimits-password 10
    :rlimits-image 2
 
-   :smtp-enabled false
    :smtp-default-reply-to "Penpot <no-reply@example.com>"
    :smtp-default-from "Penpot <no-reply@example.com>"
 
@@ -79,10 +74,6 @@
    :profile-bounce-max-age (dt/duration {:days 7})
    :profile-bounce-threshold 10
 
-   :allow-demo-users true
-   :registration-enabled true
-
-   :telemetry-enabled false
    :telemetry-uri "https://telemetry.penpot.app/"
 
    :ldap-user-query "(|(uid=:username)(mail=:username))"
@@ -95,24 +86,27 @@
    :initial-project-skey "initial-project"
    })
 
-(s/def ::audit-enabled ::us/boolean)
+(s/def ::flags ::us/words)
-(s/def ::audit-archive-enabled ::us/boolean)
+
-(s/def ::audit-archive-uri ::us/string)
+;; DEPRECATED PROPERTIES: should be removed in 1.10
-(s/def ::audit-archive-gc-enabled ::us/boolean)
+(s/def ::registration-enabled ::us/boolean)
-(s/def ::audit-archive-gc-max-age ::dt/duration)
+(s/def ::smtp-enabled ::us/boolean)
+(s/def ::telemetry-enabled ::us/boolean)
+(s/def ::asserts-enabled ::us/boolean)
+;; END DEPRECATED
+
+(s/def ::audit-log-archive-uri ::us/string)
+(s/def ::audit-log-gc-max-age ::dt/duration)
 
 (s/def ::secret-key ::us/string)
 (s/def ::allow-demo-users ::us/boolean)
-(s/def ::asserts-enabled ::us/boolean)
 (s/def ::assets-path ::us/string)
 (s/def ::database-password (s/nilable ::us/string))
 (s/def ::database-uri ::us/string)
 (s/def ::database-username (s/nilable ::us/string))
 (s/def ::default-blob-version ::us/integer)
 (s/def ::error-report-webhook ::us/string)
-(s/def ::feedback-destination ::us/string)
+(s/def ::user-feedback-destination ::us/string)
-(s/def ::feedback-enabled ::us/boolean)
-(s/def ::feedback-token ::us/string)
 (s/def ::github-client-id ::us/string)
 (s/def ::github-client-secret ::us/string)
 (s/def ::gitlab-base-uri ::us/string)
@@ -158,12 +152,10 @@
 (s/def ::public-uri ::us/string)
 (s/def ::redis-uri ::us/string)
 (s/def ::registration-domain-whitelist ::us/set-of-str)
-(s/def ::registration-enabled ::us/boolean)
 (s/def ::rlimits-image ::us/integer)
 (s/def ::rlimits-password ::us/integer)
 (s/def ::smtp-default-from ::us/string)
 (s/def ::smtp-default-reply-to ::us/string)
-(s/def ::smtp-enabled ::us/boolean)
 (s/def ::smtp-host ::us/string)
 (s/def ::smtp-password (s/nilable ::us/string))
 (s/def ::smtp-port ::us/integer)
@@ -180,28 +172,22 @@
 (s/def ::storage-fdata-s3-bucket ::us/string)
 (s/def ::storage-fdata-s3-region ::us/keyword)
 (s/def ::storage-fdata-s3-prefix ::us/string)
-(s/def ::telemetry-enabled ::us/boolean)
 (s/def ::telemetry-uri ::us/string)
 (s/def ::telemetry-with-taiga ::us/boolean)
 (s/def ::tenant ::us/string)
 
 (s/def ::config
   (s/keys :opt-un [::secret-key
+                   ::flags
                    ::allow-demo-users
-                   ::audit-enabled
+                   ::audit-log-archive-uri
-                   ::audit-archive-enabled
+                   ::audit-log-gc-max-age
-                   ::audit-archive-uri
-                   ::audit-archive-gc-enabled
-                   ::audit-archive-gc-max-age
-                   ::asserts-enabled
                    ::database-password
                    ::database-uri
                    ::database-username
                    ::default-blob-version
                    ::error-report-webhook
-                   ::feedback-destination
+                   ::user-feedback-destination
-                   ::feedback-enabled
-                   ::feedback-token
                    ::github-client-id
                    ::github-client-secret
                    ::gitlab-base-uri
@@ -258,26 +244,26 @@
                    ::smtp-ssl
                    ::smtp-tls
                    ::smtp-username
-
                    ::srepl-host
                    ::srepl-port
-
                    ::assets-storage-backend
                    ::storage-assets-fs-directory
                    ::storage-assets-s3-bucket
                    ::storage-assets-s3-region
-
                    ::fdata-storage-backend
                    ::storage-fdata-s3-bucket
                    ::storage-fdata-s3-region
                    ::storage-fdata-s3-prefix
-
                    ::telemetry-enabled
                    ::telemetry-uri
                    ::telemetry-referer
                    ::telemetry-with-taiga
                    ::tenant]))
 
+(defn- parse-flags
+  [{:keys [flags]}]
+  (flags/parse flags flags/default))
+
 (defn read-env
   [prefix]
   (let [prefix (str prefix "-")
@@ -304,11 +290,14 @@
         (println ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;")))
       (throw e))))
 
-(def version (v/parse (or (some-> (io/resource "version.txt")
+(def version
+  (v/parse (or (some-> (io/resource "version.txt")
                        (slurp)
                        (str/trim))
                "%version%")))
-(def config (atom (read-config)))
+
+(def ^:dynamic config (read-config))
+(def ^:dynamic flags  (parse-flags config))
 
 (def deletion-delay
   (dt/duration {:days 7}))
@@ -316,9 +305,9 @@
 (defn get
   "A configuration getter. Helps code be more testable."
   ([key]
-   (c/get @config key))
+   (c/get config key))
   ([key default]
-   (c/get @config key default)))
+   (c/get config key default)))
 
 ;; Set value for all new threads bindings.
-(alter-var-root #'*assert* (constantly (get :asserts-enabled)))
+(alter-var-root #'*assert* (constantly (contains? flags :backend-asserts)))

backend/src/app/emails.clj

@@ -8,7 +8,7 @@
   "Main api for send emails."
   (:require
    [app.common.spec :as us]
-   [app.config :as cfg]
+   [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
    [app.util.emails :as emails]
@@ -54,10 +54,10 @@
 (defn allow-send-emails?
   [conn profile]
   (when-not (:is-muted profile false)
-    (let [complaint-threshold (cfg/get :profile-complaint-threshold)
+    (let [complaint-threshold (cf/get :profile-complaint-threshold)
-          complaint-max-age   (cfg/get :profile-complaint-max-age)
+          complaint-max-age   (cf/get :profile-complaint-max-age)
-          bounce-threshold    (cfg/get :profile-bounce-threshold)
+          bounce-threshold    (cf/get :profile-bounce-threshold)
-          bounce-max-age      (cfg/get :profile-bounce-max-age)
+          bounce-max-age      (cf/get :profile-bounce-max-age)
 
           {:keys [complaints bounces] :as result}
           (db/exec-one! conn [sql:profile-complaint-report
@@ -140,19 +140,17 @@
 
 (declare send-console!)
 
-(s/def ::username ::cfg/smtp-username)
+(s/def ::username ::cf/smtp-username)
-(s/def ::password ::cfg/smtp-password)
+(s/def ::password ::cf/smtp-password)
-(s/def ::tls ::cfg/smtp-tls)
+(s/def ::tls ::cf/smtp-tls)
-(s/def ::ssl ::cfg/smtp-ssl)
+(s/def ::ssl ::cf/smtp-ssl)
-(s/def ::host ::cfg/smtp-host)
+(s/def ::host ::cf/smtp-host)
-(s/def ::port ::cfg/smtp-port)
+(s/def ::port ::cf/smtp-port)
-(s/def ::default-reply-to ::cfg/smtp-default-reply-to)
+(s/def ::default-reply-to ::cf/smtp-default-reply-to)
-(s/def ::default-from ::cfg/smtp-default-from)
+(s/def ::default-from ::cf/smtp-default-from)
-(s/def ::enabled ::cfg/smtp-enabled)
 
 (defmethod ig/pre-init-spec ::sendmail-handler [_]
-  (s/keys :req-un [::enabled]
+  (s/keys :opt-un [::username
-          :opt-un [::username
                    ::password
                    ::tls
                    ::ssl
@@ -164,9 +162,12 @@
 (defmethod ig/init-key ::sendmail-handler
   [_ cfg]
   (fn [{:keys [props] :as task}]
-    (if (:enabled cfg)
+    (let [enabled? (or (contains? cf/flags :smtp)
+                       (cf/get :smtp-enabled)
+                       (:enabled task))]
+      (if enabled?
         (emails/send! cfg props)
-      (send-console! cfg props))))
+        (send-console! cfg props)))))
 
 (defn- send-console!
   [cfg email]

backend/src/app/http/feedback.clj

@@ -10,7 +10,7 @@
    [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.spec :as us]
-   [app.config :as cfg]
+   [app.config :as cf]
    [app.db :as db]
    [app.emails :as eml]
    [app.rpc.queries.profile :as profile]
@@ -24,8 +24,8 @@
 
 (defmethod ig/init-key ::handler
   [_ {:keys [pool] :as scfg}]
-  (let [ftoken   (cfg/get :feedback-token ::no-token)
+  (let [ftoken   (cf/get :feedback-token ::no-token)
-        enabled  (cfg/get :feedback-enabled)]
+        enabled  (contains? cf/flags :user-feedback)]
     (fn [{:keys [profile-id] :as request}]
       (let [token  (get-in request [:headers "x-feedback-token"])
             params (d/merge (:params request)
@@ -58,7 +58,7 @@
 (defn send-feedback
   [pool profile params]
   (let [params      (us/conform ::feedback params)
-        destination (cfg/get :feedback-destination)]
+        destination (cf/get :feedback-destination)]
     (eml/send! {::eml/conn pool
                 ::eml/factory eml/feedback
                 :to       destination

backend/src/app/loggers/audit.clj

@@ -88,9 +88,9 @@
 (s/def ::events (s/every ::event))
 
 (defmethod ig/init-key ::http-handler
-  [_  {:keys [executor enabled] :as cfg}]
+  [_  {:keys [executor] :as cfg}]
-  (fn [{:keys [params _headers _cookies profile-id] :as request}]
+  (fn [{:keys [params profile-id] :as request}]
-    (when enabled
+    (when (contains? cf/flags :audit-log)
       (let [events  (->> (:events params)
                          (remove #(not= profile-id (:profile-id %)))
                          (us/conform ::events))
@@ -136,10 +136,9 @@
 ;; an external storage and data cleared.
 
 (declare persist-events)
-(s/def ::enabled ::us/boolean)
 
 (defmethod ig/pre-init-spec ::collector [_]
-  (s/keys :req-un [::db/pool ::wrk/executor ::enabled]))
+  (s/keys :req-un [::db/pool ::wrk/executor]))
 
 (def event-xform
   (comp
@@ -147,9 +146,9 @@
    (map clean-props)))
 
 (defmethod ig/init-key ::collector
-  [_ {:keys [enabled] :as cfg}]
+  [_ cfg]
-  (when enabled
+  (when (contains? cf/flags :audit-log)
-    (l/info :msg "intializing audit collector")
+    (l/info :msg "intializing audit log collector")
     (let [input  (a/chan 512 event-xform)
           buffer (aa/batch input {:max-batch-size 100
                                   :max-batch-age (* 10 1000) ; 10s
@@ -202,15 +201,16 @@
 (s/def ::tokens fn?)
 
 (defmethod ig/pre-init-spec ::archive-task [_]
-  (s/keys :req-un [::db/pool ::tokens ::enabled]
+  (s/keys :req-un [::db/pool ::tokens]
           :opt-un [::uri]))
 
 (defmethod ig/init-key ::archive-task
-  [_ {:keys [uri enabled] :as cfg}]
+  [_ {:keys [uri] :as cfg}]
   (fn [props]
     ;; NOTE: this let allows overwrite default configured values from
     ;; the repl, when manually invoking the task.
-    (let [enabled (or enabled (:enabled props false))
+    (let [enabled (or (contains? cf/flags :audit-log-archive)
+                      (:enabled props false))
           uri     (or uri (:uri props))
           cfg     (assoc cfg :uri uri)]
       (when (and enabled (not uri))
@@ -298,18 +298,6 @@
 ;; GC Task
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(declare clean-archived)
-
-(s/def ::max-age ::cf/audit-archive-gc-max-age)
-
-(defmethod ig/pre-init-spec ::archive-gc-task [_]
-  (s/keys :req-un [::db/pool ::enabled ::max-age]))
-
-(defmethod ig/init-key ::archive-gc-task
-  [_ cfg]
-  (fn [_]
-    (clean-archived cfg)))
-
 (def sql:clean-archived
   "delete from audit_log
     where archived_at is not null
@@ -322,3 +310,13 @@
         result   (:next.jdbc/update-count result)]
     (l/debug :action "clean archived audit log" :removed result)
     result))
+
+(s/def ::max-age ::cf/audit-log-gc-max-age)
+
+(defmethod ig/pre-init-spec ::gc-task [_]
+  (s/keys :req-un [::db/pool ::max-age]))
+
+(defmethod ig/init-key ::gc-task
+  [_ cfg]
+  (fn [_]
+    (clean-archived cfg)))

backend/src/app/main.clj

@@ -210,15 +210,16 @@
        {:cron #app/cron "0 0 * * * ?"  ;; hourly
         :task :file-offload})
 
-     (when (cf/get :audit-archive-enabled)
+     (when (contains? cf/flags :audit-log-archive)
        {:cron #app/cron "0 0 * * * ?" ;; every 1h
-        :task :audit-archive})
+        :task :audit-log-archive})
 
-     (when (cf/get :audit-archive-gc-enabled)
+     (when (contains? cf/flags :audit-log-gc)
        {:cron #app/cron "0 0 * * * ?" ;; every 1h
-        :task :audit-archive-gc})
+        :task :audit-log-gc})
 
-     (when (cf/get :telemetry-enabled)
+     (when (or (contains? cf/flags :telemetry)
+               (cf/get :telemetry-enabled))
        {:cron #app/cron "0 0 */6 * * ?" ;; every 6h
         :task :telemetry})]}
 
@@ -238,15 +239,14 @@
      :telemetry          (ig/ref :app.tasks.telemetry/handler)
      :session-gc         (ig/ref :app.http.session/gc-task)
      :file-offload       (ig/ref :app.tasks.file-offload/handler)
-     :audit-archive      (ig/ref :app.loggers.audit/archive-task)
+     :audit-log-archive  (ig/ref :app.loggers.audit/archive-task)
-     :audit-archive-gc   (ig/ref :app.loggers.audit/archive-gc-task)}}
+     :audit-log-gc       (ig/ref :app.loggers.audit/gc-task)}}
 
    :app.emails/sendmail-handler
    {:host             (cf/get :smtp-host)
     :port             (cf/get :smtp-port)
     :ssl              (cf/get :smtp-ssl)
     :tls              (cf/get :smtp-tls)
-    :enabled          (cf/get :smtp-enabled)
     :username         (cf/get :smtp-username)
     :password         (cf/get :smtp-password)
     :metrics          (ig/ref :app.metrics/metrics)
@@ -304,24 +304,20 @@
    {:endpoint (cf/get :loggers-zmq-uri)}
 
    :app.loggers.audit/http-handler
-   {:enabled  (cf/get :audit-enabled false)
+   {:pool     (ig/ref :app.db/pool)
-    :pool     (ig/ref :app.db/pool)
     :executor (ig/ref :app.worker/executor)}
 
    :app.loggers.audit/collector
-   {:enabled  (cf/get :audit-enabled false)
+   {:pool     (ig/ref :app.db/pool)
-    :pool     (ig/ref :app.db/pool)
     :executor (ig/ref :app.worker/executor)}
 
    :app.loggers.audit/archive-task
-   {:uri      (cf/get :audit-archive-uri)
+   {:uri      (cf/get :audit-log-archive-uri)
-    :enabled  (cf/get :audit-archive-enabled false)
     :tokens   (ig/ref :app.tokens/tokens)
     :pool     (ig/ref :app.db/pool)}
 
-   :app.loggers.audit/archive-gc-task
+   :app.loggers.audit/gc-task
-   {:enabled  (cf/get :audit-archive-gc-enabled false)
+   {:max-age  (cf/get :audit-log-gc-max-age cf/deletion-delay)
-    :max-age  (cf/get :audit-archive-gc-max-age cf/deletion-delay)
     :pool     (ig/ref :app.db/pool)}
 
    :app.loggers.loki/reporter

backend/src/app/rpc/mutations/demo.clj

@@ -9,7 +9,7 @@
   (:require
    [app.common.exceptions :as ex]
    [app.common.uuid :as uuid]
-   [app.config :as cfg]
+   [app.config :as cf]
    [app.db :as db]
    [app.loggers.audit :as audit]
    [app.rpc.mutations.profile :as profile]
@@ -35,11 +35,11 @@
                   :email email
                   :fullname fullname
                   :is-demo true
-                  :deleted-at (dt/in-future cfg/deletion-delay)
+                  :deleted-at (dt/in-future cf/deletion-delay)
                   :password password
                   :props {:onboarding-viewed true}}]
 
-    (when-not (cfg/get :allow-demo-users)
+    (when-not (contains? cf/flags :demo-users)
       (ex/raise :type :validation
                 :code :demo-users-not-allowed
                 :hint "Demo users are disabled by config."))

backend/src/app/rpc/mutations/profile.clj

@@ -100,10 +100,9 @@
 
 (sv/defmethod ::prepare-register-profile {:auth false}
   [{:keys [pool tokens] :as cfg} params]
-  (when-not (cf/get :registration-enabled)
+  (when-not (contains? cf/flags :registration)
     (ex/raise :type :restriction
               :code :registration-disabled))
-
   (when-let [domains (cf/get :registration-domain-whitelist)]
     (when-not (email-domain-in-whitelist? domains (:email params))
       (ex/raise :type :validation
@@ -458,7 +457,8 @@
           params  (assoc params
                          :profile profile
                          :email (str/lower email))]
-      (if (cf/get :smtp-enabled)
+      (if (or (cf/get :smtp-enabled)
+              (contains? cf/flags :smtp))
         (request-email-change cfg params)
         (change-email-inmediatelly cfg params)))))
 

backend/test/app/services_profile_test.clj

@@ -211,10 +211,7 @@
     ))
 
 (t/deftest prepare-register-with-registration-disabled
-  (with-mocks [mock {:target 'app.config/get
+  (th/with-mocks {#'app.config/flags nil}
-                     :return (th/mock-config-get-with
-                              {:registration-enabled false})}]
-
     (let [data  {::th/type :prepare-register-profile
                  :email "user@example.com"
                  :password "foobar"}]

backend/test/app/test_helpers.clj

@@ -7,6 +7,7 @@
 (ns app.test-helpers
   (:require
    [app.common.data :as d]
+   [app.common.flags :as flags]
    [app.common.pages :as cp]
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
@@ -336,9 +337,15 @@
   [data]
   (fn
     ([key]
-     (get data key (get @cf/config key)))
+     (get data key (get cf/config key)))
     ([key default]
-     (get data key (get @cf/config key default)))))
+     (get data key (get cf/config key default)))))
+
+
+(defmacro with-mocks
+  [rebinds & body]
+  `(with-redefs-fn ~rebinds
+     (fn [] ~@body)))
 
 (defn reset-mock!
   [m]

common/src/app/common/flags.cljc

@@ -9,8 +9,14 @@
   (:require
    [cuerdas.core :as str]))
 
+(def default
+  #{:backend-asserts
+    :registration
+    :demo-users})
+
 (defn parse
-  [default flags]
+  ([flags] (parse flags #{}))
+  ([flags default]
    (loop [flags  (seq flags)
           result default]
      (let [item (first flags)]
@@ -27,6 +33,6 @@
                     (disj result (keyword (subs sname 8))))
 
              :else
-            (recur (rest flags) result)))))))
+             (recur (rest flags) result))))))))
 
 

common/src/app/common/spec.cljc

@@ -111,6 +111,16 @@
 (s/def ::point gpt/point?)
 (s/def ::id ::uuid)
 
+(s/def ::words
+  (s/conformer
+   (fn [s]
+     (cond
+       (set? s)    s
+       (string? s) (into #{} (map keyword) (str/words s))
+       :else       ::s/invalid))
+   (fn [s]
+     (str/join " " (map name s)))))
+
 (defn bytes?
   "Test if a first parameter is a byte
   array or not."

frontend/src/app/config.cljs

@@ -54,22 +54,11 @@
     :browser
     :webworker))
 
-(def available-flags
-  #{:registration
-    :audit-log
-    :demo-users
-    :user-feedback
-    :demo-warning
-    :login-with-ldap})
-
-(def default-flags
-  #{:registration :demo-users})
-
 (defn- parse-flags
   [global]
   (let [flags (obj/get global "penpotFlags" "")
         flags (into #{} (map keyword) (str/words flags))]
-    (flags/parse default-flags flags)))
+    (flags/parse flags flags/default)))
 
 (defn- parse-version
   [global]