Merge pull request #4389 from penpot/test

✨ Several improvements

.yarnrc.yml

@@ -6,4 +6,6 @@ enableImmutableInstalls: false
 
 enableTelemetry: false
 
+httpTimeout: 600000
+
 nodeLinker: node-modules

backend/scripts/manage.py

@@ -160,7 +160,6 @@ available_commands = (
     "delete-profile",
     "search-profile",
     "derive-password",
-    "migrate-components-v2",
 )
 
 parser = argparse.ArgumentParser(
@@ -233,7 +232,4 @@ elif args.action == "search-profile":
 
     search_profile(email)
 
-elif args.action == "migrate-components-v2":
-    migrate_components_v2()
-
 

backend/scripts/repl

@@ -29,6 +29,8 @@ export PENPOT_FLAGS="\
        enable-file-validation \
        enable-file-schema-validation";
 
+# Default deletion delay for devenv
+export PENPOT_DELETION_DELAY="24h"
 
 # Setup default upload media file size to 100MiB
 export PENPOT_MEDIA_MAX_FILE_SIZE=104857600

backend/scripts/run.template.sh

@@ -18,7 +18,9 @@ if [ -f ./environ ]; then
     source ./environ
 fi
 
-export JVM_OPTS="-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dlog4j2.configurationFile=log4j2.xml -XX:-OmitStackTraceInFastThrow --enable-preview $JVM_OPTS"
+export JVM_OPTS="-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dlog4j2.configurationFile=log4j2.xml -XX:-OmitStackTraceInFastThrow -Dpolyglot.engine.WarnInterpreterOnly=false --enable-preview $JVM_OPTS"
 
-set -x
+ENTRYPOINT=${1:-app.main};
-exec $JAVA_CMD $JVM_OPTS "$@" -jar penpot.jar -m app.main
+
+set -ex
+exec $JAVA_CMD $JVM_OPTS "$@" -jar penpot.jar -m $ENTRYPOINT

backend/scripts/start-dev

@@ -32,35 +32,18 @@ export OPTIONS="
        -J-XX:+UnlockDiagnosticVMOptions \
        -J-XX:+DebugNonSafepoints"
 
+# Default deletion delay for devenv
+export PENPOT_DELETION_DELAY="24h"
+
 # Setup default upload media file size to 100MiB
 export PENPOT_MEDIA_MAX_FILE_SIZE=104857600
 
 # Setup default multipart upload size to 300MiB
 export PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE=314572800
 
-# Setup HEAP
-# export OPTIONS="$OPTIONS -J-Xms50m -J-Xmx1024m"
-# export OPTIONS="$OPTIONS -J-Xms1100m -J-Xmx1100m -J-XX:+AlwaysPreTouch"
-
-# Increase virtual thread pool size
-# export OPTIONS="$OPTIONS -J-Djdk.virtualThreadScheduler.parallelism=16"
-
-# Disable C2 Compiler
-# export OPTIONS="$OPTIONS -J-XX:TieredStopAtLevel=1"
-
-# Disable all compilers
-# export OPTIONS="$OPTIONS -J-Xint"
-
-# Setup GC
-# export OPTIONS="$OPTIONS -J-XX:+UseG1GC"
-
-# Setup GC
-# export OPTIONS="$OPTIONS -J-XX:+UseZGC"
-
 # Enable ImageMagick v7.x support
 # export OPTIONS="-J-Dim4java.useV7=true $OPTIONS";
 
-
 # Initialize MINIO config
 mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin -q
 mc admin user add penpot-s3 penpot-devenv penpot-devenv -q
@@ -76,24 +59,8 @@ export PENPOT_ASSETS_STORAGE_BACKEND=assets-s3
 export PENPOT_STORAGE_ASSETS_S3_ENDPOINT=http://minio:9000
 export PENPOT_STORAGE_ASSETS_S3_BUCKET=penpot
 
-if [ "$1" = "--watch" ]; then
+entrypoint=${1:-app.main};
-    trap "exit" INT TERM ERR
-    trap "kill 0" EXIT
 
-    echo "Start Watch..."
+set -ex
 
-    clojure $OPTIONS -A:dev -M -m app.main &
+clojure $OPTIONS -A:dev -M -m $entrypoint;
-
-    npx nodemon \
-        --watch src \
-        --watch ../common \
-        --ext "clj" \
-        --signal SIGKILL \
-        --exec 'echo "(app.main/stop)\n\r(repl/refresh)\n\r(app.main/start)\n" | nc -N localhost 6062'
-
-    wait;
-
-else
-    set -x
-    clojure $OPTIONS -A:dev -M -m app.main;
-fi

backend/src/app/config.clj

@@ -101,6 +101,8 @@
 (s/def ::audit-log-archive-uri ::us/string)
 (s/def ::audit-log-http-handler-concurrency ::us/integer)
 
+(s/def ::deletion-delay ::dt/duration)
+
 (s/def ::admins ::us/set-of-valid-emails)
 (s/def ::file-change-snapshot-every ::us/integer)
 (s/def ::file-change-snapshot-timeout ::dt/duration)
@@ -214,6 +216,7 @@
   (s/keys :opt-un [::secret-key
                    ::flags
                    ::admins
+                   ::deletion-delay
                    ::allow-demo-users
                    ::audit-log-archive-uri
                    ::audit-log-http-handler-concurrency
@@ -335,7 +338,8 @@
    :enable-backend-openapi-doc
    :enable-backend-worker
    :enable-secure-session-cookies
-   :enable-email-verification])
+   :enable-email-verification
+   :enable-v2-migration])
 
 (defn- parse-flags
   [config]
@@ -380,7 +384,8 @@
 (defonce ^:dynamic flags (parse-flags config))
 
 (def deletion-delay
-  (dt/duration {:days 7}))
+  (or (c/get config :deletion-delay)
+      (dt/duration {:days 7})))
 
 (defn get
   "A configuration getter. Helps code be more testable."

backend/src/app/features/components_v2.clj

@@ -53,7 +53,6 @@
    [app.storage.tmp :as tmp]
    [app.svgo :as svgo]
    [app.util.blob :as blob]
-   [app.util.events :as events]
    [app.util.pointer-map :as pmap]
    [app.util.time :as dt]
    [buddy.core.codecs :as bc]
@@ -1196,9 +1195,6 @@
             add-instance-grid
             (fn [fdata frame-id grid assets]
               (reduce (fn [result [component position]]
-                        (events/tap :progress {:op :migrate-component
-                                               :id (:id component)
-                                               :name (:name component)})
                         (add-main-instance result component frame-id (gpt/add position
                                                                               (gpt/point grid-gap grid-gap))))
                       fdata
@@ -1518,9 +1514,6 @@
 
     (->> (d/zip media-group grid)
          (reduce (fn [fdata [mobj position]]
-                   (events/tap :progress {:op :migrate-graphic
-                                          :id (:id mobj)
-                                          :name (:name mobj)})
                    (or (process fdata mobj position) fdata))
                  (assoc-in fdata [:options :components-v2] true)))))
 
@@ -1759,11 +1752,6 @@
                         (let [file (get-file system file-id)
                               file (process-file! system file :validate? validate?)]
 
-                          (events/tap :progress
-                                      {:op :migrate-file
-                                       :name (:name file)
-                                       :id (:id file)})
-
                           (persist-file! system file)))))
 
         (catch Throwable cause
@@ -1791,10 +1779,11 @@
             (some-> *team-stats* (swap! update :processed-files (fnil inc 0)))))))))
 
 (defn migrate-team!
-  [system team-id & {:keys [validate? skip-on-graphic-error? label]}]
+  [system team-id & {:keys [validate? rown skip-on-graphic-error? label]}]
 
   (l/dbg :hint "migrate:team:start"
-         :team-id (dm/str team-id))
+         :team-id (dm/str team-id)
+         :rown rown)
 
   (let [tpoint (dt/tpoint)
         err    (volatile! false)
@@ -1816,11 +1805,6 @@
                                  (conj "layout/grid")
                                  (conj "styles/v2"))]
 
-                (events/tap :progress
-                            {:op :migrate-team
-                             :name (:name team)
-                             :id id})
-
                 (run! (partial migrate-file system)
                       (get-and-lock-team-files conn id))
 
@@ -1849,6 +1833,7 @@
 
             (l/dbg :hint "migrate:team:end"
                    :team-id (dm/str team-id)
+                   :rown rown
                    :files files
                    :components components
                    :graphics graphics

backend/src/app/http/errors.clj

@@ -99,7 +99,7 @@
       (= code :invalid-image)
       (binding [l/*context* (request->context request)]
         (let [cause (or parent-cause err)]
-          (l/error :hint "unexpected error on processing image" :cause cause)
+          (l/warn :hint "unexpected error on processing image" :cause cause)
           {::rres/status 400 ::rres/body data}))
 
       :else

backend/src/app/loggers/mattermost.clj

@@ -23,17 +23,20 @@
 
 (defn- send-mattermost-notification!
   [cfg {:keys [id public-uri] :as report}]
+
+
   (let [text (str "Exception: " public-uri "/dbg/error/" id " "
                   (when-let [pid (:profile-id report)]
                     (str "(pid: #uuid-" pid ")"))
                   "\n"
-                  "```\n"
+                  "- host: #" (:host report) "\n"
-                  "- host: `" (:host report) "`\n"
+                  "- tenant: #" (:tenant report) "\n"
-                  "- tenant: `" (:tenant report) "`\n"
+                  "- logger: #" (:logger report) "\n"
                   "- request-path: `" (:request-path report) "`\n"
                   "- frontend-version: `" (:frontend-version report) "`\n"
                   "- backend-version: `" (:backend-version report) "`\n"
                   "\n"
+                  "```\n"
                   "Trace:\n"
                   (:trace report)
                   "```")
@@ -60,6 +63,7 @@
    :frontend-version (:version/frontend context)
    :profile-id       (:request/profile-id context)
    :request-path     (:request/path context)
+   :logger           (::l/logger record)
    :trace            (ex/format-throwable cause :detail? false :header? false)})
 
 (defn handle-event

backend/src/app/loggers/webhooks.clj

@@ -15,9 +15,9 @@
    [app.config :as cf]
    [app.db :as db]
    [app.http.client :as http]
-   [app.util.json :as json]
    [app.util.time :as dt]
    [app.worker :as wrk]
+   [clojure.data.json :as json]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [integrant.core :as ig]))
@@ -86,11 +86,9 @@
 (declare interpret-exception)
 (declare interpret-response)
 
-(def ^:private json-mapper
+(def json-write-opts
-  (json/mapper
+  {:key-fn str/camel
-   {:encode-key-fn str/camel
+   :indent true})
-    :decode-key-fn (comp keyword str/kebab)
-    :pretty true}))
 
 (defmethod ig/pre-init-spec ::run-webhook-handler [_]
   (s/keys :req [::http/client ::db/pool]))
@@ -134,15 +132,15 @@
             whook (::config props)
 
             body  (case (:mtype whook)
-                    "application/json" (json/encode-str event json-mapper)
+                    "application/json" (json/write-str event json-write-opts)
                     "application/transit+json" (t/encode-str event)
                     "application/x-www-form-urlencoded" (uri/map->query-string event))]
 
-        (l/debug :hint "run webhook"
+        (l/dbg :hint "run webhook"
-                 :event-name (:name event)
+               :event-name (:name event)
-                 :webhook-id (:id whook)
+               :webhook-id (:id whook)
-                 :webhook-uri (:uri whook)
+               :webhook-uri (:uri whook)
-                 :webhook-mtype (:mtype whook))
+               :webhook-mtype (:mtype whook))
 
         (let [req {:uri (:uri whook)
                    :headers {"content-type" (:mtype whook)
@@ -160,8 +158,8 @@
                 (report-delivery! whook req nil err)
                 (update-webhook! whook err)
                 (when (= err "unknown")
-                  (l/error :hint "unknown error on webhook request"
+                  (l/err :hint "unknown error on webhook request"
-                           :cause cause))))))))))
+                         :cause cause))))))))))
 
 (defn interpret-response
   [{:keys [status] :as response}]

backend/src/app/main.clj

@@ -24,6 +24,7 @@
    [app.loggers.webhooks :as-alias webhooks]
    [app.metrics :as-alias mtx]
    [app.metrics.definition :as-alias mdef]
+   [app.migrations.v2 :as migrations.v2]
    [app.msgbus :as-alias mbus]
    [app.redis :as-alias rds]
    [app.rpc :as-alias rpc]
@@ -527,6 +528,15 @@
          :worker? (contains? cf/flags :backend-worker)
          :version (:full cf/version)))
 
+(defn start-custom
+  [config]
+  (ig/load-namespaces config)
+  (alter-var-root #'system (fn [sys]
+                             (when sys (ig/halt! sys))
+                             (-> config
+                                 (ig/prep)
+                                 (ig/init)))))
+
 (defn stop
   []
   (alter-var-root #'system (fn [sys]
@@ -573,6 +583,11 @@
         (nrepl/start-server :bind "0.0.0.0" :port 6064 :handler cider-nrepl-handler))
 
       (start)
+
+      (when (contains? cf/flags :v2-migration)
+        (px/sleep 5000)
+        (migrations.v2/migrate app.main/system))
+
       (deref p))
     (catch Throwable cause
       (binding [*out* *err*]

backend/src/app/migrations/v2.clj

@@ -0,0 +1,104 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.migrations.v2
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [app.db :as db]
+   [app.features.components-v2 :as feat]
+   [app.setup :as setup]
+   [app.util.time :as dt]))
+
+(def ^:private sql:get-teams
+  "SELECT id, features,
+          row_number() OVER (ORDER BY created_at DESC) AS rown
+    FROM team
+   WHERE deleted_at IS NULL
+     AND (features <@ '{components/v2}' OR features IS NULL)
+   ORDER BY created_at DESC")
+
+(defn- get-teams
+  [conn]
+  (->> (db/cursor conn [sql:get-teams] {:chunk-size 1})
+       (map feat/decode-row)))
+
+(defn- migrate-teams
+  [{:keys [::db/conn] :as system}]
+  ;; Allow long running transaction for this connection
+  (db/exec-one! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"])
+
+  ;; Do not allow other migration running in the same time
+  (db/xact-lock! conn 0)
+
+  ;; Run teams migration
+  (run! (fn [{:keys [id rown]}]
+          (try
+            (-> (assoc system ::db/rollback true)
+                (feat/migrate-team! id
+                                    :rown rown
+                                    :label "v2-migration"
+                                    :validate? false
+                                    :skip-on-graphics-error? true))
+            (catch Throwable _
+              (swap! feat/*stats* update :errors (fnil inc 0))
+              (l/wrn :hint "error on migrating team (skiping)"))))
+        (get-teams conn))
+
+  (setup/set-prop! system :v2-migrated true))
+
+(defn migrate
+  [system]
+  (let [tpoint    (dt/tpoint)
+        stats     (atom {})
+        migrated? (setup/get-prop system :v2-migrated false)]
+
+    (when-not migrated?
+      (l/inf :hint "v2 migration started"
+             :files (:processed-files stats))
+      (try
+        (binding [feat/*stats* stats]
+          (db/tx-run! system migrate-teams))
+
+        (let [stats   (deref stats)
+              elapsed (dt/format-duration (tpoint))]
+          (l/inf :hint "v2 migration finished"
+                 :files (:processed-files stats)
+                 :teams (:processed-teams stats)
+                 :errors (:errors stats)
+                 :elapsed elapsed))
+
+        (catch Throwable cause
+          (l/err :hint "error on aplying v2 migration" :cause cause))))))
+
+(def ^:private required-services
+  [[:app.main/assets :app.storage.s3/backend]
+   [:app.main/assets :app.storage.fs/backend]
+   :app.storage/storage
+   :app.db/pool
+   :app.setup/props
+   :app.svgo/optimizer
+   :app.metrics/metrics
+   :app.migrations/migrations
+   :app.http.client/client])
+
+(defn -main
+  [& _args]
+  (try
+    (let [config-var (requiring-resolve 'app.main/system-config)
+          start-var  (requiring-resolve 'app.main/start-custom)
+          stop-var   (requiring-resolve 'app.main/stop)
+          system-var (requiring-resolve 'app.main/system)
+          config     (select-keys @config-var required-services)]
+
+      (start-var config)
+      (migrate @system-var)
+      (stop-var)
+      (System/exit 0))
+    (catch Throwable cause
+      (ex/print-throwable cause)
+      (flush)
+      (System/exit -1))))

backend/src/app/setup.clj

@@ -7,6 +7,7 @@
 (ns app.setup
   "Initial data setup of instance."
   (:require
+   [app.common.data :as d]
    [app.common.logging :as l]
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
@@ -25,7 +26,7 @@
       (bc/bytes->b64u)
       (bc/bytes->str)))
 
-(defn- retrieve-all
+(defn- get-all-props
   [conn]
   (->> (db/query conn :server-prop {:preload true})
        (filter #(not= "secret-key" (:id %)))
@@ -50,6 +51,28 @@
                       :cause cause))))
         instance-id)))
 
+
+(def sql:add-prop
+  "INSERT INTO server_prop (id, content, preload)
+   VALUES (?, ?, ?)
+       ON CONFLICT (id)
+       DO UPDATE SET content=?, preload=?")
+
+(defn get-prop
+  ([system prop] (get-prop system prop nil))
+  ([system prop default]
+   (let [prop (d/name prop)]
+     (db/run! system (fn [{:keys [::db/conn]}]
+                       (or (db/get* conn :server-prop {:id prop})
+                           default))))))
+
+(defn set-prop!
+  [system prop value]
+  (let [value (db/tjson value)
+        prop  (d/name prop)]
+    (db/run! system (fn [{:keys [::db/conn]}]
+                      (db/exec-one! conn [sql:add-prop prop value false value false])))))
+
 (s/def ::key ::us/string)
 (s/def ::props (s/map-of ::us/keyword some?))
 
@@ -67,7 +90,7 @@
                          "PENPOT_SECRET_KEY environment variable")))
 
     (let [secret (or key (generate-random-key))]
-      (-> (retrieve-all conn)
+      (-> (get-all-props conn)
           (assoc :secret-key secret)
           (assoc :tokens-key (keys/derive secret :salt "tokens"))
           (update :instance-id handle-instance-id conn (db/read-only? pool))))))

backend/src/app/srepl/cli.clj

@@ -11,10 +11,7 @@
    [app.common.exceptions :as ex]
    [app.common.uuid :as uuid]
    [app.db :as db]
-   [app.main :as main]
    [app.rpc.commands.auth :as cmd.auth]
-   [app.srepl.components-v2 :refer [migrate-teams!]]
-   [app.util.events :as events]
    [app.util.json :as json]
    [app.util.time :as dt]
    [cuerdas.core :as str]))
@@ -105,39 +102,6 @@
   [{:keys [password]}]
   (auth/derive-password password))
 
-(defmethod exec-command :migrate-v2
-  [_]
-  (letfn [(on-progress-report [{:keys [elapsed completed errors]}]
-            (println (str/ffmt "-> Progress: completed: %, errors: %, elapsed: %"
-                               completed errors elapsed)))
-
-          (on-progress [{:keys [op name]}]
-            (case op
-              :migrate-team
-              (println (str/ffmt "-> Migrating team: \"%\"" name))
-              :migrate-file
-              (println (str/ffmt "=> Migrating file: \"%\"" name))
-              nil))
-
-          (on-event [[type payload]]
-            (case type
-              :progress-report (on-progress-report payload)
-              :progress        (on-progress payload)
-              :error           (on-error payload)
-              nil))
-
-          (on-error [cause]
-            (println "EE:" (ex-message cause)))]
-
-    (println "The components/v2 migration started...")
-
-    (try
-      (let [result (-> (partial migrate-teams! main/system {:rollback? true})
-                       (events/run-with! on-event))]
-        (println (str/ffmt "Migration process finished (elapsed: %)" (:elapsed result))))
-      (catch Throwable cause
-        (on-error cause)))))
-
 (defmethod exec-command :default
   [{:keys [::cmd]}]
   (ex/raise :type :internal

backend/src/app/srepl/components_v2.clj

@@ -6,18 +6,15 @@
 
 (ns app.srepl.components-v2
   (:require
-   [app.common.data :as d]
    [app.common.fressian :as fres]
    [app.common.logging :as l]
    [app.db :as db]
    [app.features.components-v2 :as feat]
    [app.main :as main]
    [app.srepl.helpers :as h]
-   [app.svgo :as svgo]
    [app.util.events :as events]
    [app.util.time :as dt]
    [app.worker :as-alias wrk]
-   [cuerdas.core :as str]
    [datoteka.fs :as fs]
    [datoteka.io :as io]
    [promesa.exec :as px]
@@ -31,86 +28,6 @@
 ;; PRIVATE HELPERS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn- report-progress-files
-  [tpoint]
-  (fn [_ _ oldv newv]
-    (when (or (not= (:processed-files oldv)
-                    (:processed-files newv))
-              (not= (:errors oldv)
-                    (:errors newv)))
-      (let [completed (:processed-files newv 0)
-            errors    (:errors newv 0)
-            elapsed   (dt/format-duration (tpoint))]
-        (events/tap :progress-report
-                    {:elapsed elapsed
-                     :completed completed
-                     :errors errors})
-        (l/dbg :hint "progress"
-               :completed completed
-               :elapsed elapsed)))))
-
-(defn- report-progress-teams
-  [tpoint]
-  (fn [_ _ oldv newv]
-    (when (or (not= (:processed-teams oldv)
-                    (:processed-teams newv))
-              (not= (:errors oldv)
-                    (:errors newv)))
-      (let [completed (:processed-teams newv 0)
-            errors    (:errors newv 0)
-            elapsed   (dt/format-duration (tpoint))]
-        (events/tap :progress-report
-                    {:elapsed elapsed
-                     :completed completed
-                     :errors errors})
-        (l/dbg :hint "progress"
-               :completed completed
-               :elapsed elapsed)))))
-
-(def ^:private sql:get-teams-by-created-at
-  "WITH teams AS (
-     SELECT id, features,
-            row_number() OVER (ORDER BY created_at) AS rown
-       FROM team
-      WHERE deleted_at IS NULL
-      ORDER BY created_at DESC
-   ) SELECT * FROM TEAMS %(pred)s")
-
-(def ^:private sql:get-teams-by-graphics
-  "WITH teams AS (
-     SELECT t.id, t.features,
-            row_number() OVER (ORDER BY t.created_at) AS rown,
-            (SELECT count(*)
-               FROM file_media_object AS fmo
-               JOIN file AS f ON (f.id = fmo.file_id)
-               JOIN project AS p ON (p.id = f.project_id)
-              WHERE p.team_id = t.id
-                AND fmo.mtype = 'image/svg+xml'
-                AND fmo.is_local = false) AS graphics
-       FROM team AS t
-      WHERE t.deleted_at IS NULL
-      ORDER BY 3 ASC
-   )
-   SELECT * FROM teams %(pred)s")
-
-(def ^:private sql:get-teams-by-activity
-  "WITH teams AS (
-     SELECT t.id, t.features,
-            row_number() OVER (ORDER BY t.created_at) AS rown,
-            (SELECT coalesce(max(date_trunc('month', f.modified_at)), date_trunc('month', t.modified_at))
-               FROM file AS f
-               JOIN project AS p ON (f.project_id = p.id)
-              WHERE p.team_id = t.id) AS updated_at,
-            (SELECT coalesce(count(*), 0)
-               FROM file AS f
-               JOIN project AS p ON (f.project_id = p.id)
-              WHERE p.team_id = t.id) AS total_files
-       FROM team AS t
-      WHERE t.deleted_at IS NULL
-      ORDER BY 3 DESC, 4 DESC
-   )
-   SELECT * FROM teams %(pred)s")
-
 (def ^:private sql:get-files-by-created-at
   "SELECT id, features,
           row_number() OVER (ORDER BY created_at DESC) AS rown
@@ -118,87 +35,12 @@
     WHERE deleted_at IS NULL
     ORDER BY created_at DESC")
 
-(def ^:private sql:get-files-by-modified-at
-  "SELECT id, features
-          row_number() OVER (ORDER BY modified_at DESC) AS rown
-     FROM file
-    WHERE deleted_at IS NULL
-    ORDER BY modified_at DESC")
-
-(def ^:private sql:get-files-by-graphics
-  "WITH files AS (
-     SELECT f.id, f.features,
-            row_number() OVER (ORDER BY modified_at) AS rown,
-            (SELECT count(*) FROM file_media_object AS fmo
-              WHERE fmo.mtype = 'image/svg+xml'
-                AND fmo.is_local = false
-                AND fmo.file_id = f.id) AS graphics
-       FROM file AS f
-      WHERE f.deleted_at IS NULL
-      ORDER BY 3 ASC
-   ) SELECT * FROM files %(pred)s")
-
-(defn- read-pred
-  [entries]
-  (let [entries (if (and (vector? entries)
-                         (keyword? (first entries)))
-                  [entries]
-                  entries)]
-    (loop [params  []
-           queries []
-           entries (seq entries)]
-      (if-let [[op val field] (first entries)]
-        (let [field (name field)
-              cond  (case op
-                      :lt  (str/ffmt "% < ?" field)
-                      :lte (str/ffmt "% <= ?" field)
-                      :gt  (str/ffmt "% > ?" field)
-                      :gte (str/ffmt "% >= ?" field)
-                      :eq  (str/ffmt "% = ?" field))]
-          (recur (conj params val)
-                 (conj queries cond)
-                 (rest entries)))
-
-        (let [sql (apply str "WHERE " (str/join " AND " queries))]
-          (apply vector sql params))))))
-
-(defn- get-teams
-  [conn query pred]
-  (let [query (d/nilv query :created-at)
-        sql   (case query
-                :created-at sql:get-teams-by-created-at
-                :activity   sql:get-teams-by-activity
-                :graphics   sql:get-teams-by-graphics)
-        sql  (if pred
-               (let [[pred-sql & pred-params] (read-pred pred)]
-                 (apply vector
-                        (str/format sql {:pred pred-sql})
-                        pred-params))
-               [(str/format sql {:pred ""})])]
-
-    (->> (db/cursor conn sql {:chunk-size 500})
-         (map feat/decode-row)
-         (remove (fn [{:keys [features]}]
-                   (contains? features "components/v2"))))))
-
 (defn- get-files
-  [conn query pred]
+  [conn]
-  (let [query (d/nilv query :created-at)
+  (->> (db/cursor conn [sql:get-files-by-created-at] {:chunk-size 500})
-        sql   (case query
+       (map feat/decode-row)
-                :created-at  sql:get-files-by-created-at
+       (remove (fn [{:keys [features]}]
-                :modified-at sql:get-files-by-modified-at
+                 (contains? features "components/v2")))))
-                :graphics    sql:get-files-by-graphics)
-        sql  (if pred
-               (let [[pred-sql & pred-params] (read-pred pred)]
-                 (apply vector
-                        (str/format sql {:pred pred-sql})
-                        pred-params))
-               [(str/format sql {:pred ""})])]
-
-    (->> (db/cursor conn sql {:chunk-size 500})
-         (map feat/decode-row)
-         (remove (fn [{:keys [features]}]
-                   (contains? features "components/v2"))))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; PUBLIC API
@@ -244,8 +86,6 @@
         stats   (atom {})
         tpoint  (dt/tpoint)]
 
-    (add-watch stats :progress-report (report-progress-files tpoint))
-
     (binding [feat/*stats* stats
               feat/*cache* cache]
       (try
@@ -265,127 +105,6 @@
           (let [elapsed (dt/format-duration (tpoint))]
             (l/dbg :hint "migrate:end" :rollback rollback? :elapsed elapsed)))))))
 
-(defn migrate-teams!
-  "A REPL helper for migrate all teams.
-
-  This function starts multiple concurrent team migration processes
-  until the maximum number of jobs is reached which by default has the
-  value of `1`. This is controled with the `:max-jobs` option.
-
-  If you want to run this on multiple machines you will need to specify
-  the total number of partitions and the current partition.
-
-  In order to get the report table populated, you will need to provide
-  a correct `:label`. That label is also used for persist a file
-  snaphot before continue with the migration."
-  [& {:keys [max-jobs max-items max-time rollback? validate? query
-             pred max-procs cache skip-on-graphic-error?
-             label partitions current-partition]
-      :or {validate? false
-           rollback? true
-           max-jobs 1
-           current-partition 1
-           skip-on-graphic-error? true
-           max-items Long/MAX_VALUE}}]
-
-  (when (int? partitions)
-    (when-not (int? current-partition)
-      (throw (IllegalArgumentException. "missing `current-partition` parameter")))
-    (when-not (<= 0 current-partition partitions)
-      (throw (IllegalArgumentException. "invalid value on `current-partition` parameter"))))
-
-  (let [stats     (atom {})
-        tpoint    (dt/tpoint)
-        mtime     (some-> max-time dt/duration)
-
-        factory   (px/thread-factory :virtual false :prefix "penpot/migration/")
-        executor  (px/cached-executor :factory factory)
-
-        max-procs (or max-procs max-jobs)
-        sjobs     (ps/create :permits max-jobs)
-        sprocs    (ps/create :permits max-procs)
-
-        migrate-team
-        (fn [team-id]
-          (try
-            (db/tx-run! (assoc main/system ::db/rollback rollback?)
-                        (fn [system]
-                          (db/exec-one! system ["SET LOCAL idle_in_transaction_session_timeout = 0"])
-                          (feat/migrate-team! system team-id
-                                              :label label
-                                              :validate? validate?
-                                              :skip-on-graphic-error? skip-on-graphic-error?)))
-
-            (catch Throwable cause
-              (l/wrn :hint "unexpected error on processing team (skiping)"
-                     :team-id (str team-id))
-
-              (events/tap :error
-                          (ex-info "unexpected error on processing team (skiping)"
-                                   {:team-id team-id}
-                                   cause))
-
-              (swap! stats update :errors (fnil inc 0)))
-
-            (finally
-              (ps/release! sjobs))))
-
-        process-team
-        (fn [team-id]
-          (ps/acquire! sjobs)
-          (let [ts (tpoint)]
-            (if (and mtime (neg? (compare mtime ts)))
-              (do
-                (l/inf :hint "max time constraint reached"
-                       :team-id (str team-id)
-                       :elapsed (dt/format-duration ts))
-                (ps/release! sjobs)
-                (reduced nil))
-
-              (px/run! executor (partial migrate-team team-id)))))]
-
-    (l/dbg :hint "migrate:start"
-           :label label
-           :rollback rollback?
-           :max-jobs max-jobs
-           :max-items max-items)
-
-    (add-watch stats :progress-report (report-progress-teams tpoint))
-
-    (binding [feat/*stats* stats
-              feat/*cache* cache
-              svgo/*semaphore* sprocs]
-      (try
-        (db/tx-run! main/system
-                    (fn [{:keys [::db/conn] :as system}]
-                      (db/exec! conn ["SET LOCAL statement_timeout = 0"])
-                      (db/exec! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"])
-
-                      (run! process-team
-                            (->> (get-teams conn query pred)
-                                 (filter (fn [{:keys [rown]}]
-                                           (if (int? partitions)
-                                             (= current-partition (inc (mod rown partitions)))
-                                             true)))
-                                 (map :id)
-                                 (take max-items)))
-
-                      ;; Close and await tasks
-                      (pu/close! executor)))
-
-        (-> (deref stats)
-            (assoc :elapsed (dt/format-duration (tpoint))))
-
-        (catch Throwable cause
-          (l/dbg :hint "migrate:error" :cause cause)
-          (events/tap :error cause))
-
-        (finally
-          (let [elapsed (dt/format-duration (tpoint))]
-            (l/dbg :hint "migrate:end"
-                   :rollback rollback?
-                   :elapsed elapsed)))))))
-
 (defn migrate-files!
   "A REPL helper for migrate all files.
 
@@ -399,8 +118,8 @@
   In order to get the report table populated, you will need to provide
   a correct `:label`. That label is also used for persist a file
   snaphot before continue with the migration."
-  [& {:keys [max-jobs max-items max-time rollback? validate? query
+  [& {:keys [max-jobs max-items rollback? validate?
-             pred max-procs cache skip-on-graphic-error?
+             cache skip-on-graphic-error?
              label partitions current-partition]
       :or {validate? false
            rollback? true
@@ -417,14 +136,10 @@
 
   (let [stats     (atom {})
         tpoint    (dt/tpoint)
-        mtime     (some-> max-time dt/duration)
-
         factory   (px/thread-factory :virtual false :prefix "penpot/migration/")
         executor  (px/cached-executor :factory factory)
 
-        max-procs (or max-procs max-jobs)
         sjobs     (ps/create :permits max-jobs)
-        sprocs    (ps/create :permits max-procs)
 
         migrate-file
         (fn [file-id rown]
@@ -455,16 +170,7 @@
         process-file
         (fn [{:keys [id rown]}]
           (ps/acquire! sjobs)
-          (let [ts (tpoint)]
+          (px/run! executor (partial migrate-file id rown)))]
-            (if (and mtime (neg? (compare mtime ts)))
-              (do
-                (l/inf :hint "max time constraint reached"
-                       :file-id (str id)
-                       :elapsed (dt/format-duration ts))
-                (ps/release! sjobs)
-                (reduced nil))
-
-              (px/run! executor (partial migrate-file id rown)))))]
 
     (l/dbg :hint "migrate:start"
            :label label
@@ -472,11 +178,8 @@
            :max-jobs max-jobs
            :max-items max-items)
 
-    (add-watch stats :progress-report (report-progress-files tpoint))
-
     (binding [feat/*stats* stats
-              feat/*cache* cache
+              feat/*cache* cache]
-              svgo/*semaphore* sprocs]
       (try
         (db/tx-run! main/system
                     (fn [{:keys [::db/conn] :as system}]
@@ -484,7 +187,7 @@
                       (db/exec! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"])
 
                       (run! process-file
-                            (->> (get-files conn query pred)
+                            (->> (get-files conn)
                                  (filter (fn [{:keys [rown] :as row}]
                                            (if (int? partitions)
                                              (= current-partition (inc (mod rown partitions)))
@@ -601,17 +304,3 @@
         (let [elapsed (dt/format-duration (tpoint))]
           (l/dbg :hint "populate:end"
                  :elapsed elapsed))))))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; FILE PROCESS HELPERS
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defn delete-broken-files
-  [{:keys [id data] :as file}]
-  (if (-> data :options :components-v2 true?)
-    (do
-      (l/wrn :hint "found old components-v2 format"
-             :file-id (str id)
-             :file-name (:name file))
-      (assoc file :deleted-at (dt/now)))
-    file))

backend/src/app/tasks/file_gc.clj

@@ -79,6 +79,7 @@
      FROM file AS f
     WHERE f.has_media_trimmed IS false
       AND f.modified_at < now() - ?::interval
+      AND f.deleted_at IS NULL
     ORDER BY f.modified_at DESC
       FOR UPDATE
      SKIP LOCKED")

backend/src/app/tasks/objects_gc.clj

@@ -210,6 +210,9 @@
                         :project-id (str project-id)
                         :deleted-at (dt/format-instant deleted-at))
 
+                 ;; NOTE: fragments not handled here because they have
+                 ;; cascade.
+
                  ;; And finally, permanently delete the file.
                  (db/delete! conn :file {:id id})
 
@@ -230,7 +233,6 @@
                  (inc total))
                0)))
 
-
 (def ^:private sql:get-file-thumbnails
   "SELECT file_id, revn, media_id, deleted_at
      FROM file_thumbnail

backend/test/backend_tests/rpc_management_test.clj

@@ -612,7 +612,7 @@
       (t/is (fn? result))
 
       (let [events (th/consume-sse result)]
-        (t/is (= 6 (count events)))
+        (t/is (= 5 (count events)))
         (t/is (= :end (first (last events))))))))
 
 (t/deftest get-list-of-buitin-templates

common/src/app/common/features.cljc

@@ -50,12 +50,8 @@
     "styles/v2"
     "layout/grid"})
 
-;; A set of features enabled by default for each file, they are
+;; A set of features enabled by default
-;; implicit and are enabled by default and can't be disabled. The
+(def default-features
-;; features listed in this set are mainly freatures addedby file
-;; migrations process, so all features referenced in migrations should
-;; be here.
-(def default-enabled-features
   #{"fdata/shape-data-type"
     "styles/v2"
     "layout/grid"
@@ -81,7 +77,8 @@
 (def no-migration-features
   (-> #{"fdata/objects-map"
         "fdata/pointer-map"
-        "layout/grid"}
+        "layout/grid"
+        "fdata/shape-data-type"}
       (into frontend-only-features)))
 
 (sm/def! ::features
@@ -132,7 +129,7 @@
 (defn get-enabled-features
   "Get the globally enabled fratures set."
   [flags]
-  (into default-enabled-features xf-flag-to-feature flags))
+  (into default-features xf-flag-to-feature flags))
 
 (defn get-team-enabled-features
   "Get the team enabled features.
@@ -144,7 +141,6 @@
         team-features    (into #{} xf-remove-ephimeral (:features team))]
     (-> enabled-features
         (set/intersection no-migration-features)
-        (set/union default-enabled-features)
         (set/union team-features))))
 
 (defn check-client-features!
@@ -247,7 +243,7 @@
   (let [not-supported (-> (or source-features #{})
                           (set/difference destination-features)
                           (set/difference no-migration-features)
-                          (set/difference default-enabled-features)
+                          (set/difference default-features)
                           (seq))]
     (when not-supported
       (ex/raise :type :restriction
@@ -259,7 +255,7 @@
   (let [not-supported (-> (or destination-features #{})
                           (set/difference source-features)
                           (set/difference no-migration-features)
-                          (set/difference default-enabled-features)
+                          (set/difference default-features)
                           (seq))]
     (when not-supported
       (ex/raise :type :restriction

common/src/app/common/files/migrations.cljc

@@ -45,7 +45,7 @@
          data data]
     (if-let [[to-version migrate-fn] (first migrations)]
       (let [migrate-fn (or migrate-fn identity)]
-        (l/inf :hint "migrate file"
+        (l/trc :hint "migrate file"
                :op (if (>= from-version to-version) "down" "up")
                :file-id (str (:id data))
                :version to-version)

docker/devenv/Dockerfile

@@ -143,23 +143,23 @@ RUN set -eux; \
     ARCH="$(dpkg --print-architecture)"; \
     case "${ARCH}" in \
        aarch64|arm64) \
-         BINARY_URL="https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-arm64.tar.xz"; \
+         BINARY_URL="https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-arm64.tar.gz"; \
          ;; \
        amd64|x86_64) \
-         BINARY_URL="https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-x64.tar.xz"; \
+         BINARY_URL="https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-x64.tar.gz"; \
          ;; \
        *) \
          echo "Unsupported arch: ${ARCH}"; \
          exit 1; \
          ;; \
     esac; \
-    curl -LfsSo /tmp/nodejs.tar.xz ${BINARY_URL}; \
+    curl -LfsSo /tmp/nodejs.tar.gz ${BINARY_URL}; \
     mkdir -p /usr/local/nodejs; \
     cd /usr/local/nodejs; \
-    tar -xf /tmp/nodejs.tar.xz --strip-components=1; \
+    tar -xf /tmp/nodejs.tar.gz --strip-components=1; \
     chown -R root /usr/local/nodejs; \
     corepack enable; \
-    rm -rf /tmp/nodejs.tar.xz;
+    rm -rf /tmp/nodejs.tar.gz;
 
 RUN set -ex; \
     ARCH="$(dpkg --print-architecture)"; \

docker/images/Dockerfile.backend

@@ -40,16 +40,12 @@ RUN set -eux; \
     ARCH="$(dpkg --print-architecture)"; \
     case "${ARCH}" in \
        aarch64|arm64) \
-         ESUM='1c4be9aa173cb0deb0d215643d9509c8900e5497290b29eee4bee335fa57984f'; \
+         ESUM='3ce6a2b357e2ef45fd6b53d6587aa05bfec7771e7fb982f2c964f6b771b7526a'; \
-         BINARY_URL='https://github.com/adoptium/temurin19-binaries/releases/download/jdk-19.0.2%2B7/OpenJDK19U-jdk_aarch64_linux_hotspot_19.0.2_7.tar.gz'; \
+         BINARY_URL='https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.2%2B13/OpenJDK21U-jdk_aarch64_linux_hotspot_21.0.2_13.tar.gz'; \
-         ;; \
-       armhf|armv7l) \
-         ESUM='6a51cb3868b5a3b81848a0d276267230ff3f8639f20ba9ae9ef1d386440bf1fd'; \
-         BINARY_URL='https://github.com/adoptium/temurin19-binaries/releases/download/jdk-19.0.2%2B7/OpenJDK19U-jdk_arm_linux_hotspot_19.0.2_7.tar.gz'; \
          ;; \
        amd64|x86_64) \
-         ESUM='3a3ba7a3f8c3a5999e2c91ea1dca843435a0d1c43737bd2f6822b2f02fc52165'; \
+         ESUM='454bebb2c9fe48d981341461ffb6bf1017c7b7c6e15c6b0c29b959194ba3aaa5'; \
-         BINARY_URL='https://github.com/adoptium/temurin19-binaries/releases/download/jdk-19.0.2%2B7/OpenJDK19U-jdk_x64_linux_hotspot_19.0.2_7.tar.gz'; \
+         BINARY_URL='https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.2%2B13/OpenJDK21U-jdk_x64_linux_hotspot_21.0.2_13.tar.gz'; \
          ;; \
        *) \
          echo "Unsupported arch: ${ARCH}"; \
@@ -63,7 +59,6 @@ RUN set -eux; \
     tar -xf /tmp/openjdk.tar.gz --strip-components=1; \
     rm -rf /tmp/openjdk.tar.gz;
 
-
 COPY --chown=penpot:penpot ./bundle-backend/ /opt/penpot/backend/
 
 USER penpot:penpot

docker/images/Dockerfile.exporter

@@ -3,7 +3,7 @@ LABEL maintainer="Andrey Antukh <niwi@niwi.nz>"
 
 ENV LANG=en_US.UTF-8 \
     LC_ALL=en_US.UTF-8 \
-    NODE_VERSION=v18.15.0 \
+    NODE_VERSION=v20.11.1 \
     DEBIAN_FRONTEND=noninteractive \
     PATH=/opt/node/bin:$PATH
 
@@ -75,26 +75,23 @@ RUN set -eux; \
     ARCH="$(dpkg --print-architecture)"; \
     case "${ARCH}" in \
        aarch64|arm64) \
-         BINARY_URL="https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-arm64.tar.xz"; \
+         BINARY_URL="https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-arm64.tar.gz"; \
-         ;; \
-       armhf|armv7l) \
-         BINARY_URL="https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-armv7l.tar.xz"; \
          ;; \
        amd64|x86_64) \
-         BINARY_URL="https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-x64.tar.xz"; \
+         BINARY_URL="https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-x64.tar.gz"; \
          ;; \
        *) \
          echo "Unsupported arch: ${ARCH}"; \
          exit 1; \
          ;; \
     esac; \
-    curl -LfsSo /tmp/nodejs.tar.xz ${BINARY_URL}; \
+    curl -LfsSo /tmp/nodejs.tar.gz ${BINARY_URL}; \
     mkdir -p /opt/node; \
     cd /opt/node; \
-    tar -xf /tmp/nodejs.tar.xz --strip-components=1; \
+    tar -xf /tmp/nodejs.tar.gz --strip-components=1; \
     chown -R root /opt/node; \
-    npm install -g yarn; \
+    corepack enable; \
-    rm -rf /tmp/nodejs.tar.xz; \
+    rm -rf /tmp/nodejs.tar.gz; \
     mkdir -p /opt/penpot; \
     chown -R penpot:penpot /opt/penpot;
 
@@ -104,7 +101,7 @@ WORKDIR /opt/penpot/exporter
 USER penpot:penpot
 
 RUN set -ex; \
-    yarn --network-timeout 1000000; \
+    yarn install; \
-    yarn --network-timeout 1000000 run playwright install chromium;
+    yarn run playwright install chromium;
 
 CMD ["node", "app.js"]

exporter/scripts/build

@@ -16,6 +16,7 @@ clojure -J-Xms100M -J-Xmx1000M -J-XX:+UseSerialGC -M:dev:shadow-cljs release mai
 rm -rf target/app;
 
 # Copy package*.json files
+cp ../.yarnrc.yml target/;
 cp yarn.lock target/;
 cp package.json target/;
 

frontend/src/app/main/features.cljs

@@ -35,7 +35,6 @@
   (-> global-enabled-features
       (set/union (:features/runtime state #{}))
       (set/intersection cfeat/no-migration-features)
-      (set/union cfeat/default-enabled-features)
       (set/union (:features/team state #{}))))
 
 (def features-ref