3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-06-06 16:01:39 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #3803 from penpot/niwinz-develop-components-migration-4

Browse source 
🐛 Several bugfixes
This commit is contained in:
Pablo Alba 2023-11-15 19:59:11 +01:00 • committed by GitHub
commit 83d786743b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 209 additions and 206 deletions
Download patch file Download diff file Expand all files Collapse all files

12
backend/dev/user.clj
View file

@ -94,22 +94,14 @@
(defn- start (defn- start
[] []
(try (try
(alter-var-root #'system (fn [sys] (main/start)
(when sys (ig/halt! sys))
(-> main/system-config
(cond-> (contains? cf/flags :backend-worker)
(merge main/worker-config))
(ig/prep)
(ig/init))))
:started :started
(catch Throwable cause (catch Throwable cause
(ex/print-throwable cause)))) (ex/print-throwable cause))))
(defn- stop (defn- stop
[] []
(alter-var-root #'system (fn [sys] (main/stop)
(when sys (ig/halt! sys))
nil))
:stopped) :stopped)
(defn restart (defn restart

2
backend/src/app/config.clj
View file

@ -203,6 +203,7 @@
(s/def ::storage-assets-s3-bucket ::us/string) (s/def ::storage-assets-s3-bucket ::us/string)
(s/def ::storage-assets-s3-region ::us/keyword) (s/def ::storage-assets-s3-region ::us/keyword)
(s/def ::storage-assets-s3-endpoint ::us/string) (s/def ::storage-assets-s3-endpoint ::us/string)
(s/def ::storage-assets-s3-io-threads ::us/integer)
(s/def ::telemetry-uri ::us/string) (s/def ::telemetry-uri ::us/string)
(s/def ::telemetry-with-taiga ::us/boolean) (s/def ::telemetry-with-taiga ::us/boolean)
(s/def ::tenant ::us/string) (s/def ::tenant ::us/string)
@ -320,6 +321,7 @@
::storage-assets-s3-bucket ::storage-assets-s3-bucket
::storage-assets-s3-region ::storage-assets-s3-region
::storage-assets-s3-endpoint ::storage-assets-s3-endpoint
::storage-assets-s3-io-threads
::telemetry-enabled ::telemetry-enabled
::telemetry-uri ::telemetry-uri
::telemetry-referer ::telemetry-referer

243
backend/src/app/features/components_v2.clj
View file

@ -45,6 +45,7 @@
[buddy.core.codecs :as bc] [buddy.core.codecs :as bc]
[cuerdas.core :as str] [cuerdas.core :as str]
[datoteka.io :as io] [datoteka.io :as io]
[promesa.exec :as px]
[promesa.exec.semaphore :as ps])) [promesa.exec.semaphore :as ps]))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@ -53,6 +54,8 @@
(def ^:dynamic *system* nil) (def ^:dynamic *system* nil)
(def ^:dynamic *stats* nil) (def ^:dynamic *stats* nil)
(def ^:dynamic *file-stats* nil)
(def ^:dynamic *team-stats* nil)
(def ^:dynamic *semaphore* nil) (def ^:dynamic *semaphore* nil)
(def ^:dynamic *skip-on-error* true) (def ^:dynamic *skip-on-error* true)
@ -380,12 +383,10 @@
fdata fdata
(d/zip components positions))))] (d/zip components positions))))]
(when (some? *stats*) (let [total (count components)]
(let [total (count components)] (some-> *stats* (swap! update :processed/components (fnil + 0) total))
(swap! *stats* (fn [stats] (some-> *team-stats* (swap! update :processed/components (fnil + 0) total))
(-> stats (some-> *file-stats* (swap! assoc :processed/components total)))
(update :processed/components (fnil + 0) total)
(assoc :current/components total))))))
(-> file-data (-> file-data
(prepare-file-data libraries) (prepare-file-data libraries)
@ -575,8 +576,7 @@
cfsh/prepare-create-artboard-from-selection) cfsh/prepare-create-artboard-from-selection)
changes (pcb/concat-changes changes changes2)] changes (pcb/concat-changes changes changes2)]
(cp/process-changes (assoc-in fdata [:options :components-v2] true) ; Process component creation in v2 way (:redo-changes changes)))
(:redo-changes changes) false)))
(defn- migrate-graphics (defn- migrate-graphics
[fdata] [fdata]
@ -591,30 +591,48 @@
(grc/rect->points))] (grc/rect->points))]
(assoc media :points points))))) (assoc media :points points)))))
;; FIXME: think about what to do with existing media entries ??
grid (ctst/generate-shape-grid media position grid-gap)] grid (ctst/generate-shape-grid media position grid-gap)]
(when (some? *stats*) (let [total (count media)]
(let [total (count media)] (some-> *stats* (swap! update :processed/graphics (fnil + 0) total))
(swap! *stats* (fn [stats] (some-> *team-stats* (swap! update :processed/graphics (fnil + 0) total))
(-> stats (some-> *file-stats* (swap! assoc :processed/graphics total)))
(update :processed/graphics (fnil + 0) total)
(assoc :current/graphics total))))))
(->> (d/zip media grid) (let [factory (px/thread-factory :virtual true)
(reduce (fn [fdata [mobj position]] executor (px/fixed-executor :parallelism 10 :factory factory)
(try process (fn [mobj position]
(process-media-object fdata page-id mobj position) (let [tp1 (dt/tpoint)]
(catch Throwable cause (try
(l/warn :hint "unable to process file media object (skiping)" (process-media-object fdata page-id mobj position)
:file-id (str (:id fdata)) (catch Throwable cause
:id (str (:id mobj)) (l/wrn :hint "unable to process file media object (skiping)"
:cause cause) :file-id (str (:id fdata))
:id (str (:id mobj))
:cause cause)
(if-not *skip-on-error* (if-not *skip-on-error*
(throw cause) (throw cause)
fdata)))) fdata))
fdata))))) (finally
(l/trc :hint "graphic processed"
:file-id (str (:id fdata))
:media-id (str (:id mobj))
:elapsed (dt/format-duration (tp1)))))))
process (px/wrap-bindings process)]
(try
(->> (d/zip media grid)
(map (fn [[mobj position]]
(l/trc :hint "submit graphic processing" :file-id (str (:id fdata)) :id (str (:id mobj)))
(px/submit! executor (partial process mobj position))))
(reduce (fn [fdata promise]
(if-let [changes (deref promise)]
(cp/process-changes fdata changes false)
fdata))
fdata))
(finally
(.close ^java.lang.AutoCloseable executor)))))))
(defn- migrate-file-data (defn- migrate-file-data
[fdata libs] [fdata libs]
@ -665,7 +683,7 @@
(when validate? (when validate?
(let [errors (cfv/validate-file file libs)] (let [errors (cfv/validate-file file libs)]
(when (seq errors) (when (seq errors)
(l/err :hint "migrate:file:validation-error" (l/wrn :hint "migrate:file:validation-error"
:file-id (str (:id file)) :file-id (str (:id file))
:file-name (:name file) :file-name (:name file)
:errors errors)))) :errors errors))))
@ -674,43 +692,38 @@
(defn migrate-file! (defn migrate-file!
[system file-id & {:keys [validate?]}] [system file-id & {:keys [validate?]}]
(let [tpoint (dt/tpoint) (let [tpoint (dt/tpoint)
file-id (if (string? file-id) file-id (if (string? file-id)
(parse-uuid file-id) (parse-uuid file-id)
file-id)] file-id)]
(try (binding [*file-stats* (atom {})]
(l/dbg :hint "migrate:file:start" :file-id (str file-id)) (try
(let [system (update system ::sto/storage media/configure-assets-storage)] (l/dbg :hint "migrate:file:start" :file-id (str file-id))
(db/tx-run! system
(fn [{:keys [::db/conn] :as system}]
(fsnap/take-file-snapshot! system {:file-id file-id
:label "migration/components-v2"})
(binding [*system* system] (let [system (update system ::sto/storage media/configure-assets-storage)]
(-> (db/get conn :file {:id file-id}) (db/tx-run! system
(update :features db/decode-pgarray #{}) (fn [{:keys [::db/conn] :as system}]
(process-file :validate? validate?)))))) (binding [*system* system]
(fsnap/take-file-snapshot! system {:file-id file-id
:label "migration/components-v2"})
(-> (db/get conn :file {:id file-id})
(update :features db/decode-pgarray #{})
(process-file :validate? validate?))))))
(finally (finally
(let [elapsed (tpoint) (let [elapsed (tpoint)
stats (some-> *stats* deref)] components (get @*file-stats* :processed/components 0)
(l/dbg :hint "migrate:file:end" graphics (get @*file-stats* :processed/graphics 0)]
:file-id (str file-id)
:components (:current/components stats 0)
:graphics (:current/graphics stats 0)
:elapsed (dt/format-duration elapsed))
(when (some? *stats*) (l/dbg :hint "migrate:file:end"
(swap! *stats* (fn [stats] :file-id (str file-id)
(let [elapsed (inst-ms elapsed) :graphics graphics
completed (inc (get stats :processed/files 0)) :components components
total (+ (get stats :elapsed/total-by-file 0) elapsed) :elapsed (dt/format-duration elapsed))
avg (/ (double elapsed) completed)]
(-> stats (some-> *stats* (swap! update :processed/files (fnil inc 0)))
(update :elapsed/max-by-file (fnil max 0) elapsed) (some-> *team-stats* (swap! update :processed/files (fnil inc 0)))))))))
(assoc :elapsed/avg-by-file avg)
(assoc :elapsed/total-by-file total)
(assoc :processed/files completed)))))))))))
(defn migrate-team! (defn migrate-team!
[system team-id & {:keys [validate?]}] [system team-id & {:keys [validate?]}]
@ -719,72 +732,66 @@
(parse-uuid team-id) (parse-uuid team-id)
team-id)] team-id)]
(l/dbg :hint "migrate:team:start" :team-id (dm/str team-id)) (l/dbg :hint "migrate:team:start" :team-id (dm/str team-id))
(try (binding [*team-stats* (atom {})]
;; We execute this out of transaction because we want this (try
;; change to be visible to all other sessions before starting ;; We execute this out of transaction because we want this
;; the migration ;; change to be visible to all other sessions before starting
(let [sql (str "UPDATE team SET features = " ;; the migration
" array_append(features, 'ephimeral/v2-migration') " (let [sql (str "UPDATE team SET features = "
" WHERE id = ?")] " array_append(features, 'ephimeral/v2-migration') "
(db/exec-one! system [sql team-id])) " WHERE id = ?")]
(db/exec-one! system [sql team-id]))
(db/tx-run! system (db/tx-run! system
(fn [{:keys [::db/conn] :as system}] (fn [{:keys [::db/conn] :as system}]
;; Lock the team ;; Lock the team
(db/exec-one! conn ["SET idle_in_transaction_session_timeout = 0"]) (db/exec-one! conn ["SET idle_in_transaction_session_timeout = 0"])
(let [{:keys [features] :as team} (-> (db/get conn :team {:id team-id}) (let [{:keys [features] :as team} (-> (db/get conn :team {:id team-id})
(update :features db/decode-pgarray #{}))] (update :features db/decode-pgarray #{}))]
(if (contains? features "components/v2") (if (contains? features "components/v2")
(l/dbg :hint "team already migrated") (l/dbg :hint "team already migrated")
(let [sql (str/concat (let [sql (str/concat
"SELECT f.id FROM file AS f " "SELECT f.id FROM file AS f "
" JOIN project AS p ON (p.id = f.project_id) " " JOIN project AS p ON (p.id = f.project_id) "
"WHERE p.team_id = ? AND f.deleted_at IS NULL AND p.deleted_at IS NULL " "WHERE p.team_id = ? AND f.deleted_at IS NULL AND p.deleted_at IS NULL "
"FOR UPDATE") "FOR UPDATE")
rows (->> (db/exec! conn [sql team-id]) rows (->> (db/exec! conn [sql team-id])
(map :id))] (map :id))]
(run! #(migrate-file! system % :validate? validate?) rows) (run! #(migrate-file! system % :validate? validate?) rows)
(some-> *stats* (swap! assoc :current/files (count rows)))
(let [features (-> features (let [features (-> features
(disj "ephimeral/v2-migration") (disj "ephimeral/v2-migration")
(conj "components/v2") (conj "components/v2")
(conj "layout/grid") (conj "layout/grid")
(conj "styles/v2"))] (conj "styles/v2"))]
(db/update! conn :team (db/update! conn :team
{:features (db/create-array conn "text" features)} {:features (db/create-array conn "text" features)}
{:id team-id}))))))) {:id team-id})))))))
(finally (finally
(some-> *semaphore* ps/release!) (some-> *semaphore* ps/release!)
(let [elapsed (tpoint) (let [elapsed (tpoint)]
stats (some-> *stats* deref)] (some-> *stats* (swap! update :processed/teams (fnil inc 0)))
(when (some? *stats*)
(swap! *stats* (fn [stats]
(let [elapsed (inst-ms elapsed)
completed (inc (get stats :processed/teams 0))
total (+ (get stats :elapsed/total-by-team 0) elapsed)
avg (/ (double elapsed) completed)]
(-> stats
(update :elapsed/max-by-team (fnil max 0) elapsed)
(assoc :elapsed/avg-by-team avg)
(assoc :elapsed/total-by-team total)
(assoc :processed/teams completed))))))
;; We execute this out of transaction because we want this ;; We execute this out of transaction because we want this
;; change to be visible to all other sessions before starting ;; change to be visible to all other sessions before starting
;; the migration ;; the migration
(let [sql (str "UPDATE team SET features = " (let [sql (str "UPDATE team SET features = "
" array_remove(features, 'ephimeral/v2-migration') " " array_remove(features, 'ephimeral/v2-migration') "
" WHERE id = ?")] " WHERE id = ?")]
(db/exec-one! system [sql team-id])) (db/exec-one! system [sql team-id]))
(l/dbg :hint "migrate:team:end" (let [components (get @*team-stats* :processed/components 0)
:team-id (dm/str team-id) graphics (get @*team-stats* :processed/graphics 0)
:files (:current/files stats 0) files (get @*team-stats* :processed/files 0)]
:elapsed (dt/format-duration elapsed))))))) (l/dbg :hint "migrate:team:end"
:team-id (dm/str team-id)
:files files
:components components
:graphics graphics
:elapsed (dt/format-duration elapsed)))))))))

8
backend/src/app/main.clj
View file

@ -450,10 +450,10 @@
:assets-fs (ig/ref [::assets :app.storage.fs/backend])}} :assets-fs (ig/ref [::assets :app.storage.fs/backend])}}
[::assets :app.storage.s3/backend] [::assets :app.storage.s3/backend]
{::sto.s3/region (cf/get :storage-assets-s3-region) {::sto.s3/region (cf/get :storage-assets-s3-region)
::sto.s3/endpoint (cf/get :storage-assets-s3-endpoint) ::sto.s3/endpoint (cf/get :storage-assets-s3-endpoint)
::sto.s3/bucket (cf/get :storage-assets-s3-bucket) ::sto.s3/bucket (cf/get :storage-assets-s3-bucket)
::wrk/executor (ig/ref ::wrk/executor)} ::sto.s3/io-threads (cf/get :storage-assets-s3-io-threads)}
[::assets :app.storage.fs/backend] [::assets :app.storage.fs/backend]
{::sto.fs/directory (cf/get :storage-assets-fs-directory)} {::sto.fs/directory (cf/get :storage-assets-fs-directory)}

1
backend/src/app/rpc/commands/binfile.clj
View file

@ -731,6 +731,7 @@
(assoc :modified-at timestamp) (assoc :modified-at timestamp)
(update :data (fn [data] (update :data (fn [data]
(-> data (-> data
(dissoc :recent-colors)
(assoc :id file-id') (assoc :id file-id')
(cond-> (> (:version data) cfd/version) (cond-> (> (:version data) cfd/version)
(assoc :version cfd/version)) (assoc :version cfd/version))

17
backend/src/app/rpc/commands/files.clj
View file

@ -170,11 +170,18 @@
(defn load-pointer (defn load-pointer
[conn file-id id] [conn file-id id]
(let [row (db/get conn :file-data-fragment (let [{:keys [content]} (db/get conn :file-data-fragment
{:id id :file-id file-id} {:id id :file-id file-id}
{:columns [:content] {:columns [:content]
::db/check-deleted? false})] ::db/check-deleted? false})]
(blob/decode (:content row)))) (when-not content
(ex/raise :type :internal
:code :fragment-not-found
:hint "fragment not found"
:file-id file-id
:fragment-id id))
(blob/decode content)))
(defn- load-all-pointers! (defn- load-all-pointers!
[{:keys [data] :as file}] [{:keys [data] :as file}]

93
backend/src/app/srepl/components_v2.clj
View file

@ -21,17 +21,9 @@
(defn- print-stats! (defn- print-stats!
[stats] [stats]
(let [stats (-> stats (->> stats
(d/update-when :elapsed/max-by-team (comp dt/format-duration dt/duration int)) (into (sorted-map))
(d/update-when :elapsed/avg-by-team (comp dt/format-duration dt/duration int)) (pp/pprint)))
(d/update-when :elapsed/total-by-team (comp dt/format-duration dt/duration int))
(d/update-when :elapsed/max-by-file (comp dt/format-duration dt/duration int))
(d/update-when :elapsed/avg-by-file (comp dt/format-duration dt/duration int))
(d/update-when :elapsed/total-by-file (comp dt/format-duration dt/duration int))
)]
(->> stats
(into (sorted-map))
(pp/pprint))))
(defn- report-progress-files (defn- report-progress-files
[tpoint] [tpoint]
@ -42,7 +34,7 @@
completed (:processed/files newv) completed (:processed/files newv)
progress (/ (* completed 100.0) total) progress (/ (* completed 100.0) total)
elapsed (tpoint)] elapsed (tpoint)]
(l/trc :hint "progress" (l/dbg :hint "progress"
:completed (:processed/files newv) :completed (:processed/files newv)
:total (:total/files newv) :total (:total/files newv)
:progress (str (int progress) "%") :progress (str (int progress) "%")
@ -57,8 +49,11 @@
completed (:processed/teams newv) completed (:processed/teams newv)
progress (/ (* completed 100.0) total) progress (/ (* completed 100.0) total)
elapsed (tpoint)] elapsed (tpoint)]
(l/trc :hint "progress" (l/dbg :hint "progress"
:completed (:processed/teams newv) :completed-teams (:processed/teams newv)
:completed-files (:processed/files newv)
:completed-graphics (:processed/graphics newv)
:completed-components (:processed/components newv)
:progress (str (int progress) "%") :progress (str (int progress) "%")
:elapsed (dt/format-duration elapsed)))))) :elapsed (dt/format-duration elapsed))))))
@ -88,36 +83,35 @@
(:count res))) (:count res)))
(defn migrate-file! (defn migrate-file!
[system file-id & {:keys [rollback] :or {rollback true}}] [system file-id & {:keys [rollback?] :or {rollback? true}}]
(l/dbg :hint "migrate:start") (l/dbg :hint "migrate:start")
(let [tpoint (dt/tpoint)] (let [tpoint (dt/tpoint)]
(try (try
(binding [feat/*stats* (atom {})] (binding [feat/*stats* (atom {})]
(-> (assoc system ::db/rollback rollback) (-> (assoc system ::db/rollback rollback?)
(feat/migrate-file! file-id)) (feat/migrate-file! file-id))
(-> (deref feat/*stats*) (-> (deref feat/*stats*)
(assoc :elapsed (dt/format-duration (tpoint))) (assoc :elapsed (dt/format-duration (tpoint)))))
(dissoc :current/graphics)
(dissoc :current/components)
(dissoc :current/files)))
(catch Throwable cause (catch Throwable cause
(l/dbg :hint "migrate:error" :cause cause)) (l/wrn :hint "migrate:error" :cause cause))
(finally (finally
(let [elapsed (dt/format-duration (tpoint))] (let [elapsed (dt/format-duration (tpoint))]
(l/dbg :hint "migrate:end" :elapsed elapsed)))))) (l/dbg :hint "migrate:end" :elapsed elapsed))))))
(defn migrate-files! (defn migrate-files!
[{:keys [::db/pool] :as system} & {:keys [chunk-size max-jobs max-items start-at preset rollback skip-on-error validate] [{:keys [::db/pool] :as system}
:or {chunk-size 10 & {:keys [chunk-size max-jobs max-items start-at preset rollback? skip-on-error validate?]
skip-on-error true :or {chunk-size 10
max-jobs 10 skip-on-error true
max-items Long/MAX_VALUE max-jobs 10
preset :shutdown-on-failure max-items Long/MAX_VALUE
rollback true preset :shutdown-on-failure
validate false}}] rollback? true
validate? false}}]
(letfn [(get-chunk [cursor] (letfn [(get-chunk [cursor]
(let [sql (str/concat (let [sql (str/concat
"SELECT id, created_at FROM file " "SELECT id, created_at FROM file "
@ -151,17 +145,14 @@
(run! (fn [file-id] (run! (fn [file-id]
(ps/acquire! feat/*semaphore*) (ps/acquire! feat/*semaphore*)
(px/submit! scope (fn [] (px/submit! scope (fn []
(-> (assoc system ::db/rollback rollback) (-> (assoc system ::db/rollback rollback?)
(feat/migrate-file! file-id :validate? validate))))) (feat/migrate-file! file-id :validate? validate?)))))
(get-candidates)) (get-candidates))
(p/await! scope)) (p/await! scope))
(-> (deref feat/*stats*) (-> (deref feat/*stats*)
(assoc :elapsed (dt/format-duration (tpoint))) (assoc :elapsed (dt/format-duration (tpoint))))
(dissoc :current/graphics)
(dissoc :current/components)
(dissoc :current/files))
(catch Throwable cause (catch Throwable cause
(l/dbg :hint "migrate:error" :cause cause)) (l/dbg :hint "migrate:error" :cause cause))
@ -172,8 +163,8 @@
(defn migrate-team! (defn migrate-team!
[{:keys [::db/pool] :as system} team-id [{:keys [::db/pool] :as system} team-id
& {:keys [rollback skip-on-error validate] & {:keys [rollback? skip-on-error validate?]
:or {rollback true skip-on-error true validate false}}] :or {rollback? true skip-on-error true validate? false}}]
(l/dbg :hint "migrate:start") (l/dbg :hint "migrate:start")
(let [total (get-total-files pool :team-id team-id) (let [total (get-total-files pool :team-id team-id)
@ -185,15 +176,13 @@
(try (try
(binding [feat/*stats* stats (binding [feat/*stats* stats
feat/*skip-on-error* skip-on-error] feat/*skip-on-error* skip-on-error]
(-> (assoc system ::db/rollback rollback) (-> (assoc system ::db/rollback rollback?)
(feat/migrate-team! team-id :validate? validate)) (feat/migrate-team! team-id :validate? validate?))
(print-stats! (print-stats!
(-> (deref feat/*stats*) (-> (deref feat/*stats*)
(dissoc :total/files) (dissoc :total/files)
(dissoc :current/graphics) (assoc :elapsed (dt/format-duration (tpoint))))))
(dissoc :current/components)
(dissoc :current/files))))
(catch Throwable cause (catch Throwable cause
(l/dbg :hint "migrate:error" :cause cause)) (l/dbg :hint "migrate:error" :cause cause))
@ -204,14 +193,14 @@
(defn migrate-teams! (defn migrate-teams!
[{:keys [::db/pool] :as system} [{:keys [::db/pool] :as system}
& {:keys [chunk-size max-jobs max-items start-at rollback preset skip-on-error max-time validate] & {:keys [chunk-size max-jobs max-items start-at rollback? preset skip-on-error max-time validate?]
:or {chunk-size 10000 :or {chunk-size 10000
rollback true validate? false
rollback? true
skip-on-error true skip-on-error true
preset :shutdown-on-failure preset :shutdown-on-failure
max-jobs Integer/MAX_VALUE max-jobs Integer/MAX_VALUE
max-items Long/MAX_VALUE max-items Long/MAX_VALUE}}]
validate false}}]
(letfn [(get-chunk [cursor] (letfn [(get-chunk [cursor]
(let [sql (str/concat (let [sql (str/concat
@ -233,8 +222,8 @@
(migrate-team [team-id] (migrate-team [team-id]
(try (try
(-> (assoc system ::db/rollback rollback) (-> (assoc system ::db/rollback rollback?)
(feat/migrate-team! team-id :validate? validate)) (feat/migrate-team! team-id :validate? validate?))
(catch Throwable cause (catch Throwable cause
(l/err :hint "unexpected error on processing team" :team-id (dm/str team-id) :cause cause)))) (l/err :hint "unexpected error on processing team" :team-id (dm/str team-id) :cause cause))))
@ -242,7 +231,7 @@
(ps/acquire! feat/*semaphore*) (ps/acquire! feat/*semaphore*)
(let [ts (tpoint)] (let [ts (tpoint)]
(if (and mtime (neg? (compare mtime ts))) (if (and mtime (neg? (compare mtime ts)))
(l/trc :hint "max time constraint reached" :elapsed (dt/format-duration ts)) (l/inf :hint "max time constraint reached" :elapsed (dt/format-duration ts))
(px/submit! scope (partial migrate-team team-id)))))] (px/submit! scope (partial migrate-team team-id)))))]
(l/dbg :hint "migrate:start") (l/dbg :hint "migrate:start")
@ -270,10 +259,8 @@
(print-stats! (print-stats!
(-> (deref feat/*stats*) (-> (deref feat/*stats*)
(dissoc :total/teams) (assoc :elapsed/total (dt/format-duration (tpoint)))
(dissoc :current/graphics) (dissoc :total/teams)))
(dissoc :current/components)
(dissoc :current/files)))
(catch Throwable cause (catch Throwable cause
(l/dbg :hint "migrate:error" :cause cause)) (l/dbg :hint "migrate:error" :cause cause))

15
backend/src/app/storage/s3.clj
View file

@ -17,7 +17,6 @@
[app.storage.impl :as impl] [app.storage.impl :as impl]
[app.storage.tmp :as tmp] [app.storage.tmp :as tmp]
[app.util.time :as dt] [app.util.time :as dt]
[app.worker :as wrk]
[clojure.java.io :as io] [clojure.java.io :as io]
[clojure.spec.alpha :as s] [clojure.spec.alpha :as s]
[datoteka.fs :as fs] [datoteka.fs :as fs]
@ -40,7 +39,6 @@
software.amazon.awssdk.core.async.AsyncRequestBody software.amazon.awssdk.core.async.AsyncRequestBody
software.amazon.awssdk.core.async.AsyncResponseTransformer software.amazon.awssdk.core.async.AsyncResponseTransformer
software.amazon.awssdk.core.client.config.ClientAsyncConfiguration software.amazon.awssdk.core.client.config.ClientAsyncConfiguration
software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption
software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient
software.amazon.awssdk.http.nio.netty.SdkEventLoopGroup software.amazon.awssdk.http.nio.netty.SdkEventLoopGroup
software.amazon.awssdk.regions.Region software.amazon.awssdk.regions.Region
@ -77,9 +75,10 @@
(s/def ::bucket ::us/string) (s/def ::bucket ::us/string)
(s/def ::prefix ::us/string) (s/def ::prefix ::us/string)
(s/def ::endpoint ::us/string) (s/def ::endpoint ::us/string)
(s/def ::io-threads ::us/integer)
(defmethod ig/pre-init-spec ::backend [_] (defmethod ig/pre-init-spec ::backend [_]
(s/keys :opt [::region ::bucket ::prefix ::endpoint ::wrk/executor])) (s/keys :opt [::region ::bucket ::prefix ::endpoint ::io-threads]))
(defmethod ig/prep-key ::backend (defmethod ig/prep-key ::backend
[_ {:keys [::prefix ::region] :as cfg}] [_ {:keys [::prefix ::region] :as cfg}]
@ -114,8 +113,7 @@
::client ::client
::presigner] ::presigner]
:opt [::prefix :opt [::prefix
::sto/id ::sto/id]))
::wrk/executor]))
;; --- API IMPL ;; --- API IMPL
@ -161,7 +159,6 @@
;; --- HELPERS ;; --- HELPERS
(def default-eventloop-threads 4)
(def default-timeout (def default-timeout
(dt/duration {:seconds 30})) (dt/duration {:seconds 30}))
@ -171,18 +168,18 @@
(Region/of (name region))) (Region/of (name region)))
(defn- build-s3-client (defn- build-s3-client
[{:keys [::region ::endpoint ::wrk/executor]}] [{:keys [::region ::endpoint ::io-threads]}]
(let [aconfig (-> (ClientAsyncConfiguration/builder) (let [aconfig (-> (ClientAsyncConfiguration/builder)
(.advancedOption SdkAdvancedAsyncClientOption/FUTURE_COMPLETION_EXECUTOR executor)
(.build)) (.build))
sconfig (-> (S3Configuration/builder) sconfig (-> (S3Configuration/builder)
(cond-> (some? endpoint) (.pathStyleAccessEnabled true)) (cond-> (some? endpoint) (.pathStyleAccessEnabled true))
(.build)) (.build))
thr-num (or io-threads (min 16 (px/get-available-processors)))
hclient (-> (NettyNioAsyncHttpClient/builder) hclient (-> (NettyNioAsyncHttpClient/builder)
(.eventLoopGroupBuilder (-> (SdkEventLoopGroup/builder) (.eventLoopGroupBuilder (-> (SdkEventLoopGroup/builder)
(.numberOfThreads (int default-eventloop-threads)))) (.numberOfThreads (int thr-num))))
(.connectionAcquisitionTimeout default-timeout) (.connectionAcquisitionTimeout default-timeout)
(.connectionTimeout default-timeout) (.connectionTimeout default-timeout)
(.readTimeout default-timeout) (.readTimeout default-timeout)

2
docker/devenv/docker-compose.yaml
View file

@ -70,7 +70,7 @@ services:
- PENPOT_LDAP_ATTRS_PHOTO=jpegPhoto - PENPOT_LDAP_ATTRS_PHOTO=jpegPhoto
minio: minio:
image: "minio/minio:latest" image: "minio/minio:RELEASE.2023-11-11T08-14-41Z"
command: minio server /mnt/data --console-address ":9001" command: minio server /mnt/data --console-address ":9001"
volumes: volumes:

2
frontend/src/app/main/data/dashboard.cljs
View file

@ -920,8 +920,8 @@
(rx/tap on-success) (rx/tap on-success)
(rx/catch on-error)))))) (rx/catch on-error))))))
;; --- EVENT: clone-template ;; --- EVENT: clone-template
(defn clone-template (defn clone-template
[{:keys [template-id project-id] :as params}] [{:keys [template-id project-id] :as params}]
(dm/assert! (uuid? project-id)) (dm/assert! (uuid? project-id))

1
frontend/src/app/main/data/workspace.cljs
View file

@ -298,6 +298,7 @@
(rx/map (fn [[file thumbnails team-users comments-users]] (rx/map (fn [[file thumbnails team-users comments-users]]
(let [bundle (-> bundle (let [bundle (-> bundle
(assoc :file file) (assoc :file file)
(assoc :features features)
(assoc :thumbnails thumbnails) (assoc :thumbnails thumbnails)
(assoc :team-users team-users) (assoc :team-users team-users)
(assoc :comments-users comments-users))] (assoc :comments-users comments-users))]

7
frontend/src/app/main/errors.cljs
View file

@ -50,6 +50,13 @@
(finally (finally
(js/console.groupEnd message)))) (js/console.groupEnd message))))
(defn print-cause!
[message cause]
(print-group! message (fn []
(print-data! cause)
(print-explain! cause)
(print-trace! cause))))
(defn on-error (defn on-error
"A general purpose error handler." "A general purpose error handler."
[error] [error]

12
frontend/src/app/main/ui/dashboard/import.cljs
View file

@ -13,6 +13,7 @@
[app.main.data.events :as ev] [app.main.data.events :as ev]
[app.main.data.messages :as msg] [app.main.data.messages :as msg]
[app.main.data.modal :as modal] [app.main.data.modal :as modal]
[app.main.errors :as errors]
[app.main.features :as features] [app.main.features :as features]
[app.main.store :as st] [app.main.store :as st]
[app.main.ui.components.file-uploader :refer [file-uploader]] [app.main.ui.components.file-uploader :refer [file-uploader]]
@ -296,10 +297,10 @@
(st/emit! (dd/fetch-recent-files))) (st/emit! (dd/fetch-recent-files)))
on-template-cloned-error on-template-cloned-error
(fn [] (fn [cause]
(st/emit! (errors/print-cause! "Template Clone Error" cause)
(modal/hide) (st/emit! (modal/hide)
(msg/error (tr "dashboard.libraries-and-templates.import-error")))) (msg/error (tr "dashboard.libraries-and-templates.import-error"))))
continue-files continue-files
(fn [] (fn []
@ -314,7 +315,8 @@
continue-template continue-template
(fn [] (fn []
(let [mdata {:on-success on-template-cloned-success :on-error on-template-cloned-error} (let [mdata {:on-success on-template-cloned-success
:on-error on-template-cloned-error}
params {:project-id project-id :template-id (:id template)}] params {:project-id project-id :template-id (:id template)}]
(swap! state (swap! state
(fn [state] (fn [state]