3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-06-06 17:01:38 +02:00
Code Issues Projects Releases Packages Wiki Activity

🎉 Add file-data offload mechanism

Browse source
This commit is contained in:
Andrey Antukh 2024-08-01 16:17:22 +02:00
parent f6bfe3931c
commit 0e92bcc0de
35 changed files with 502 additions and 216 deletions
Download patch file Download diff file Expand all files Collapse all files

1
backend/src/app/rpc/commands/auth.clj
View file

@ -522,7 +522,6 @@
(create-recovery-token)
(send-email-notification conn)))))))
(def schema:request-profile-recovery
[:map {:title "request-profile-recovery"}
[:email ::sm/email]])

23
backend/src/app/rpc/commands/files.clj
View file

@ -68,6 +68,9 @@
:max-version fmg/version))
file))
;; --- FILE DATA
;; --- FILE PERMISSIONS
(def ^:private sql:file-permissions
@ -258,11 +261,12 @@
(let [params (merge {:id id}
(when (some? project-id)
{:project-id project-id}))
file (-> (db/get conn :file params
{::db/check-deleted (not include-deleted?)
::db/remove-deleted (not include-deleted?)
::sql/for-update lock-for-update?})
(decode-row))]
file (->> (db/get conn :file params
{::db/check-deleted (not include-deleted?)
::db/remove-deleted (not include-deleted?)
::sql/for-update lock-for-update?})
(feat.fdata/resolve-file-data cfg)
(decode-row))]
(if (and migrate? (fmg/need-migration? file))
(migrate-file cfg file)
file)))
@ -328,8 +332,10 @@
(defn- get-file-fragment
[cfg file-id fragment-id]
(some-> (db/get cfg :file-data-fragment {:file-id file-id :id fragment-id})
(update :data blob/decode)))
(let [resolve-file-data (partial feat.fdata/resolve-file-data cfg)]
(some-> (db/get cfg :file-data-fragment {:file-id file-id :id fragment-id})
(resolve-file-data)
(update :data blob/decode))))
(sv/defmethod ::get-file-fragment
"Retrieve a file fragment by its ID. Only authenticated users."
@ -802,7 +808,8 @@
(db/update! cfg :file
{:revn (inc (:revn file))
:data (blob/encode (:data file))
:modified-at (dt/now)}
:modified-at (dt/now)
:has-media-trimmed false}
{:id file-id})
(feat.fdata/persist-pointers! cfg file-id))))

5
backend/src/app/rpc/commands/files_snapshot.clj
View file

@ -14,7 +14,6 @@
[app.db :as db]
[app.db.sql :as-alias sql]
[app.main :as-alias main]
[app.media :as media]
[app.rpc :as-alias rpc]
[app.rpc.commands.files :as files]
[app.rpc.commands.profile :as profile]
@ -63,8 +62,8 @@
(db/run! cfg get-file-snapshots params))
(defn restore-file-snapshot!
[{:keys [::db/conn ::sto/storage] :as cfg} {:keys [file-id id]}]
(let [storage (media/configure-assets-storage storage conn)
[{:keys [::db/conn] :as cfg} {:keys [file-id id]}]
(let [storage (sto/resolve cfg {::db/reuse-conn true})
file (files/get-minimal-file conn file-id {::db/for-update true})
snapshot (db/get* conn :file-change
{:file-id file-id

8
backend/src/app/rpc/commands/files_thumbnails.clj
View file

@ -295,8 +295,7 @@
(db/run! cfg files/check-edition-permissions! profile-id file-id)
(let [cfg (update cfg ::sto/storage media/configure-assets-storage)]
(create-file-object-thumbnail! cfg file-id object-id media (or tag "frame"))))
(create-file-object-thumbnail! cfg file-id object-id media (or tag "frame")))
;; --- MUTATION COMMAND: delete-file-object-thumbnail
@ -327,7 +326,7 @@
(files/check-edition-permissions! cfg profile-id file-id)
(db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
(-> cfg
(update ::sto/storage media/configure-assets-storage conn)
(update ::sto/storage sto/configure conn)
(delete-file-object-thumbnail! file-id object-id))
nil)))
@ -405,7 +404,6 @@
(db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
(files/check-edition-permissions! conn profile-id file-id)
(when-not (db/read-only? conn)
(let [cfg (update cfg ::sto/storage media/configure-assets-storage)
media (create-file-thumbnail! cfg params)]
(let [media (create-file-thumbnail! cfg params)]
{:uri (files/resolve-public-uri (:id media))
:id (:id media)})))))

7
backend/src/app/rpc/commands/files_update.clj
View file

@ -227,8 +227,12 @@
(defn- update-file*
[{:keys [::db/conn ::wrk/executor] :as cfg}
{:keys [profile-id file changes session-id ::created-at skip-validate] :as params}]
(let [;; Process the file data on separated thread for avoid to do
(let [;; Retrieve the file data
file (feat.fdata/resolve-file-data cfg file {:touch true})
;; Process the file data on separated thread for avoid to do
;; the CPU intensive operation on vthread.
file (px/invoke! executor (partial update-file-data cfg file changes skip-validate))
features (db/create-array conn "text" (:features file))]
@ -254,6 +258,7 @@
:version (:version file)
:features features
:data-backend nil
:data-ref-id nil
:modified-at created-at
:has-media-trimmed false}
{:id (:id file)})

30
backend/src/app/rpc/commands/fonts.clj
View file

@ -95,12 +95,11 @@
[cfg {:keys [::rpc/profile-id team-id] :as params}]
(db/tx-run! cfg
(fn [{:keys [::db/conn] :as cfg}]
(let [cfg (update cfg ::sto/storage media/configure-assets-storage)]
(teams/check-edition-permissions! conn profile-id team-id)
(quotes/check-quote! conn {::quotes/id ::quotes/font-variants-per-team
::quotes/profile-id profile-id
::quotes/team-id team-id})
(create-font-variant cfg (assoc params :profile-id profile-id))))))
(teams/check-edition-permissions! conn profile-id team-id)
(quotes/check-quote! conn {::quotes/id ::quotes/font-variants-per-team
::quotes/profile-id profile-id
::quotes/team-id team-id})
(create-font-variant cfg (assoc params :profile-id profile-id)))))
(defn create-font-variant
[{:keys [::sto/storage ::db/conn ::wrk/executor]} {:keys [data] :as params}]
@ -203,14 +202,13 @@
::sm/params schema:delete-font}
[cfg {:keys [::rpc/profile-id id team-id]}]
(db/tx-run! cfg
(fn [{:keys [::db/conn ::sto/storage] :as cfg}]
(fn [{:keys [::db/conn] :as cfg}]
(teams/check-edition-permissions! conn profile-id team-id)
(let [fonts (db/query conn :team-font-variant
{:team-id team-id
:font-id id
:deleted-at nil}
{::sql/for-update true})
storage (media/configure-assets-storage storage conn)
tnow (dt/now)]
(when-not (seq fonts)
@ -220,11 +218,7 @@
(doseq [font fonts]
(db/update! conn :team-font-variant
{:deleted-at tnow}
{:id (:id font)})
(some->> (:woff1-file-id font) (sto/touch-object! storage))
(some->> (:woff2-file-id font) (sto/touch-object! storage))
(some->> (:ttf-file-id font) (sto/touch-object! storage))
(some->> (:otf-file-id font) (sto/touch-object! storage)))
{:id (:id font)}))
(rph/with-meta (rph/wrap)
{::audit/props {:id id
@ -245,22 +239,16 @@
::sm/params schema:delete-font-variant}
[cfg {:keys [::rpc/profile-id id team-id]}]
(db/tx-run! cfg
(fn [{:keys [::db/conn ::sto/storage] :as cfg}]
(fn [{:keys [::db/conn] :as cfg}]
(teams/check-edition-permissions! conn profile-id team-id)
(let [variant (db/get conn :team-font-variant
{:id id :team-id team-id}
{::sql/for-update true})
storage (media/configure-assets-storage storage conn)]
{::sql/for-update true})]
(db/update! conn :team-font-variant
{:deleted-at (dt/now)}
{:id (:id variant)})
(some->> (:woff1-file-id variant) (sto/touch-object! storage))
(some->> (:woff2-file-id variant) (sto/touch-object! storage))
(some->> (:ttf-file-id variant) (sto/touch-object! storage))
(some->> (:otf-file-id variant) (sto/touch-object! storage))
(rph/with-meta (rph/wrap)
{::audit/props {:font-family (:font-family variant)
:font-id (:font-id variant)}})))))

31
backend/src/app/rpc/commands/media.clj
View file

@ -56,21 +56,19 @@
::climit/id [[:process-image/by-profile ::rpc/profile-id]
[:process-image/global]]}
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id content] :as params}]
(let [cfg (update cfg ::sto/storage media/configure-assets-storage)]
(files/check-edition-permissions! pool profile-id file-id)
(media/validate-media-type! content)
(media/validate-media-size! content)
(files/check-edition-permissions! pool profile-id file-id)
(media/validate-media-type! content)
(media/validate-media-size! content)
(db/run! cfg (fn [cfg]
(let [object (create-file-media-object cfg params)
props {:name (:name params)
:file-id file-id
:is-local (:is-local params)
:size (:size content)
:mtype (:mtype content)}]
(with-meta object
{::audit/replace-props props}))))))
(db/run! cfg (fn [cfg]
(let [object (create-file-media-object cfg params)
props {:name (:name params)
:file-id file-id
:is-local (:is-local params)
:size (:size content)
:mtype (:mtype content)}]
(with-meta object
{::audit/replace-props props})))))
(defn- big-enough-for-thumbnail?
"Checks if the provided image info is big enough for
@ -183,9 +181,8 @@
{::doc/added "1.17"
::sm/params schema:create-file-media-object-from-url}
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
(let [cfg (update cfg ::sto/storage media/configure-assets-storage)]
(files/check-edition-permissions! pool profile-id file-id)
(create-file-media-object-from-url cfg (assoc params :profile-id profile-id))))
(files/check-edition-permissions! pool profile-id file-id)
(create-file-media-object-from-url cfg (assoc params :profile-id profile-id)))
(defn download-image
[{:keys [::http/client]} uri]

3
backend/src/app/rpc/commands/profile.clj
View file

@ -210,8 +210,7 @@
[cfg {:keys [::rpc/profile-id file] :as params}]
;; Validate incoming mime type
(media/validate-media-type! file #{"image/jpeg" "image/png" "image/webp"})
(let [cfg (update cfg ::sto/storage media/configure-assets-storage)]
(update-profile-photo cfg (assoc params :profile-id profile-id))))
(update-profile-photo cfg (assoc params :profile-id profile-id)))
(defn update-profile-photo
[{:keys [::db/pool ::sto/storage] :as cfg} {:keys [profile-id file] :as params}]

3
backend/src/app/rpc/commands/teams.clj
View file

@ -674,8 +674,7 @@
[cfg {:keys [::rpc/profile-id file] :as params}]
;; Validate incoming mime type
(media/validate-media-type! file #{"image/jpeg" "image/png" "image/webp"})
(let [cfg (update cfg ::sto/storage media/configure-assets-storage)]
(update-team-photo cfg (assoc params :profile-id profile-id))))
(update-team-photo cfg (assoc params :profile-id profile-id)))
(defn update-team-photo
[{:keys [::db/pool ::sto/storage] :as cfg} {:keys [profile-id team-id] :as params}]