mirror of
https://github.com/penpot/penpot.git
synced 2025-06-06 21:04:53 +02:00
✨ Add safety mechanism for direct object deletion
The main objective is prevent deletion of objects that can leave unreachable orphan objects which we are unable to correctly track. Additionally, this commit includes: 1. Properly implement safe cascade deletion of all participating tables on soft deletion in the objects-gc task; 2. Make the file thumbnail related tables also participate in the touch/refcount mechanism applyign to the same safety checks; 3. Add helper for db query lazy iteration using PostgreSQL support for server side cursors; 4. Fix efficiency issues on gc related task using server side cursors instead of custom chunked iteration for processing data. The problem resided when a large chunk of rows that has identical value on the deleted_at column and the chunk size is small (the default); when the custom chunked iteration only reads a first N items and skip the rest of the set to the next run. This has caused many objects to remain pending to be eliminated, taking up space for longer than expected. The server side cursor based iteration does not has this problem and iterates correctly over all objects. 5. Fix refcount issues on font variant deletion RPC methods
This commit is contained in:
Andrey Antukh
parent
e6fb96c4c2
commit
addb392ecc
37 changed files with 1918 additions and 1026 deletions
|
|
@ -27,6 +27,7 @@
|
|||
[app.rpc.commands.teams :as teams]
|
||||
[app.rpc.cond :as-alias cond]
|
||||
[app.rpc.doc :as-alias doc]
|
||||
[app.rpc.retry :as rtry]
|
||||
[app.storage :as sto]
|
||||
[app.util.pointer-map :as pmap]
|
||||
[app.util.services :as sv]
|
||||
|
|
@ -46,7 +47,7 @@
|
|||
(let [sql (str/concat
|
||||
"select object_id, media_id, tag "
|
||||
" from file_tagged_object_thumbnail"
|
||||
" where file_id=? and tag=?")
|
||||
" where file_id=? and tag=? and deleted_at is null")
|
||||
res (db/exec! conn [sql file-id tag])]
|
||||
(->> res
|
||||
(d/index-by :object-id (fn [row]
|
||||
|
|
@ -58,7 +59,7 @@
|
|||
(let [sql (str/concat
|
||||
"select object_id, media_id, tag "
|
||||
" from file_tagged_object_thumbnail"
|
||||
" where file_id=?")
|
||||
" where file_id=? and deleted_at is null")
|
||||
res (db/exec! conn [sql file-id])]
|
||||
(->> res
|
||||
(d/index-by :object-id (fn [row]
|
||||
|
|
@ -69,7 +70,7 @@
|
|||
(let [sql (str/concat
|
||||
"select object_id, media_id, tag "
|
||||
" from file_tagged_object_thumbnail"
|
||||
" where file_id=? and object_id = ANY(?)")
|
||||
" where file_id=? and object_id = ANY(?) and deleted_at is null")
|
||||
ids (db/create-array conn "text" (seq object-ids))
|
||||
res (db/exec! conn [sql file-id ids])]
|
||||
|
||||
|
|
@ -226,34 +227,54 @@
|
|||
;; MUTATION COMMANDS
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
;; --- MUTATION COMMAND: create-file-object-thumbnail
|
||||
|
||||
(def ^:private sql:create-object-thumbnail
|
||||
"insert into file_tagged_object_thumbnail(file_id, object_id, media_id, tag)
|
||||
values (?, ?, ?, ?)
|
||||
on conflict(file_id, tag, object_id) do
|
||||
update set media_id = ?
|
||||
returning *;")
|
||||
;; MUTATION COMMAND: create-file-object-thumbnail
|
||||
|
||||
(defn- create-file-object-thumbnail!
|
||||
[{:keys [::db/conn ::sto/storage]} file-id object-id media tag]
|
||||
|
||||
(let [path (:path media)
|
||||
(let [thumb (db/get* conn :file-tagged-object-thumbnail
|
||||
{:file-id file-id
|
||||
:object-id object-id
|
||||
:tag tag}
|
||||
{::db/remove-deleted? false
|
||||
::db/for-update? true})
|
||||
|
||||
path (:path media)
|
||||
mtype (:mtype media)
|
||||
hash (sto/calculate-hash path)
|
||||
data (-> (sto/content path)
|
||||
(sto/wrap-with-hash hash))
|
||||
tnow (dt/now)
|
||||
|
||||
media (sto/put-object! storage
|
||||
{::sto/content data
|
||||
::sto/deduplicate? true
|
||||
::sto/touched-at (dt/now)
|
||||
::sto/touched-at tnow
|
||||
:content-type mtype
|
||||
:bucket "file-object-thumbnail"})]
|
||||
|
||||
(db/exec-one! conn [sql:create-object-thumbnail file-id object-id
|
||||
(:id media) tag (:id media)])))
|
||||
(if (some? thumb)
|
||||
(do
|
||||
;; We mark the old media id as touched if it does not matches
|
||||
(when (not= (:id media) (:media-id thumb))
|
||||
(sto/touch-object! storage (:media-id thumb)))
|
||||
(db/update! conn :file-tagged-object-thumbnail
|
||||
{:media-id (:id media)
|
||||
:deleted-at nil
|
||||
:updated-at tnow}
|
||||
{:file-id file-id
|
||||
:object-id object-id
|
||||
:tag tag}))
|
||||
(db/insert! conn :file-tagged-object-thumbnail
|
||||
{:file-id file-id
|
||||
:object-id object-id
|
||||
:created-at tnow
|
||||
:updated-at tnow
|
||||
:tag tag
|
||||
:media-id (:id media)}))))
|
||||
|
||||
(def schema:create-file-object-thumbnail
|
||||
(def ^:private
|
||||
schema:create-file-object-thumbnail
|
||||
[:map {:title "create-file-object-thumbnail"}
|
||||
[:file-id ::sm/uuid]
|
||||
[:object-id :string]
|
||||
|
|
@ -268,32 +289,37 @@
|
|||
::audit/skip true
|
||||
::sm/params schema:create-file-object-thumbnail}
|
||||
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id object-id media tag]}]
|
||||
(db/with-atomic [conn pool]
|
||||
(files/check-edition-permissions! conn profile-id file-id)
|
||||
(media/validate-media-type! media)
|
||||
(media/validate-media-size! media)
|
||||
[cfg {:keys [::rpc/profile-id file-id object-id media tag]}]
|
||||
(db/tx-run! cfg
|
||||
(fn [{:keys [::db/conn] :as cfg}]
|
||||
(files/check-edition-permissions! conn profile-id file-id)
|
||||
(media/validate-media-type! media)
|
||||
(media/validate-media-size! media)
|
||||
|
||||
(when-not (db/read-only? conn)
|
||||
(-> cfg
|
||||
(update ::sto/storage media/configure-assets-storage)
|
||||
(assoc ::db/conn conn)
|
||||
(create-file-object-thumbnail! file-id object-id media (or tag "frame"))))))
|
||||
(when-not (db/read-only? conn)
|
||||
(let [cfg (-> cfg
|
||||
(update ::sto/storage media/configure-assets-storage)
|
||||
(assoc ::rtry/when rtry/conflict-exception?)
|
||||
(assoc ::rtry/max-retries 5)
|
||||
(assoc ::rtry/label "create-file-object-thumbnail"))]
|
||||
(rtry/invoke cfg create-file-object-thumbnail!
|
||||
file-id object-id media (or tag "frame")))))))
|
||||
|
||||
;; --- MUTATION COMMAND: delete-file-object-thumbnail
|
||||
|
||||
(defn- delete-file-object-thumbnail!
|
||||
[{:keys [::db/conn ::sto/storage]} file-id object-id]
|
||||
(when-let [{:keys [media-id]} (db/get* conn :file-tagged-object-thumbnail
|
||||
{:file-id file-id
|
||||
:object-id object-id}
|
||||
{::db/for-update? true})]
|
||||
|
||||
(when-let [{:keys [media-id tag]} (db/get* conn :file-tagged-object-thumbnail
|
||||
{:file-id file-id
|
||||
:object-id object-id}
|
||||
{::db/for-update? true})]
|
||||
(sto/touch-object! storage media-id)
|
||||
(db/delete! conn :file-tagged-object-thumbnail
|
||||
(db/update! conn :file-tagged-object-thumbnail
|
||||
{:deleted-at (dt/now)}
|
||||
{:file-id file-id
|
||||
:object-id object-id})
|
||||
nil))
|
||||
:object-id object-id
|
||||
:tag tag}
|
||||
{::db/return-keys? false})))
|
||||
|
||||
(s/def ::delete-file-object-thumbnail
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
|
|
@ -302,29 +328,21 @@
|
|||
(sv/defmethod ::delete-file-object-thumbnail
|
||||
{::doc/added "1.19"
|
||||
::doc/module :files
|
||||
::doc/deprecated "1.20"
|
||||
::climit/id :file-thumbnail-ops
|
||||
::climit/key-fn ::rpc/profile-id
|
||||
::audit/skip true}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id object-id]}]
|
||||
|
||||
(db/with-atomic [conn pool]
|
||||
(files/check-edition-permissions! conn profile-id file-id)
|
||||
|
||||
(when-not (db/read-only? conn)
|
||||
(-> cfg
|
||||
(update ::sto/storage media/configure-assets-storage)
|
||||
(assoc ::db/conn conn)
|
||||
(delete-file-object-thumbnail! file-id object-id))
|
||||
nil)))
|
||||
[cfg {:keys [::rpc/profile-id file-id object-id]}]
|
||||
(db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
|
||||
(files/check-edition-permissions! conn profile-id file-id)
|
||||
(when-not (db/read-only? conn)
|
||||
(-> cfg
|
||||
(update ::sto/storage media/configure-assets-storage conn)
|
||||
(delete-file-object-thumbnail! file-id object-id))
|
||||
nil))))
|
||||
|
||||
;; --- MUTATION COMMAND: create-file-thumbnail
|
||||
|
||||
(def ^:private sql:create-file-thumbnail
|
||||
"insert into file_thumbnail (file_id, revn, media_id, props)
|
||||
values (?, ?, ?, ?::jsonb)
|
||||
on conflict(file_id, revn) do
|
||||
update set media_id=?, props=?, updated_at=now();")
|
||||
|
||||
(defn- create-file-thumbnail!
|
||||
[{:keys [::db/conn ::sto/storage]} {:keys [file-id revn props media] :as params}]
|
||||
(media/validate-media-type! media)
|
||||
|
|
@ -336,14 +354,42 @@
|
|||
hash (sto/calculate-hash path)
|
||||
data (-> (sto/content path)
|
||||
(sto/wrap-with-hash hash))
|
||||
tnow (dt/now)
|
||||
media (sto/put-object! storage
|
||||
{::sto/content data
|
||||
::sto/deduplicate? false
|
||||
::sto/deduplicate? true
|
||||
::sto/touched-at tnow
|
||||
:content-type mtype
|
||||
:bucket "file-thumbnail"})]
|
||||
(db/exec-one! conn [sql:create-file-thumbnail file-id revn
|
||||
(:id media) props
|
||||
(:id media) props])
|
||||
:bucket "file-thumbnail"})
|
||||
|
||||
thumb (db/get* conn :file-thumbnail
|
||||
{:file-id file-id
|
||||
:revn revn}
|
||||
{::db/remove-deleted? false
|
||||
::db/for-update? true})]
|
||||
|
||||
(if (some? thumb)
|
||||
(do
|
||||
;; We mark the old media id as touched if it does not match
|
||||
(when (not= (:id media) (:media-id thumb))
|
||||
(sto/touch-object! storage (:media-id thumb)))
|
||||
|
||||
(db/update! conn :file-thumbnail
|
||||
{:media-id (:id media)
|
||||
:deleted-at nil
|
||||
:updated-at tnow
|
||||
:props props}
|
||||
{:file-id file-id
|
||||
:revn revn}))
|
||||
|
||||
(db/insert! conn :file-thumbnail
|
||||
{:file-id file-id
|
||||
:revn revn
|
||||
:created-at tnow
|
||||
:updated-at tnow
|
||||
:props props
|
||||
:media-id (:id media)}))
|
||||
|
||||
media))
|
||||
|
||||
(sv/defmethod ::create-file-thumbnail
|
||||
|
|
@ -359,13 +405,14 @@
|
|||
[:revn :int]
|
||||
[:media ::media/upload]]}
|
||||
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(files/check-edition-permissions! conn profile-id file-id)
|
||||
(when-not (db/read-only? conn)
|
||||
(let [media (-> cfg
|
||||
(update ::sto/storage media/configure-assets-storage)
|
||||
(assoc ::db/conn conn)
|
||||
(create-file-thumbnail! params))]
|
||||
|
||||
{:uri (files/resolve-public-uri (:id media))}))))
|
||||
[cfg {:keys [::rpc/profile-id file-id] :as params}]
|
||||
(db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
|
||||
(files/check-edition-permissions! conn profile-id file-id)
|
||||
(when-not (db/read-only? conn)
|
||||
(let [cfg (-> cfg
|
||||
(update ::sto/storage media/configure-assets-storage)
|
||||
(assoc ::rtry/when rtry/conflict-exception?)
|
||||
(assoc ::rtry/max-retries 5)
|
||||
(assoc ::rtry/label "create-thumbnail"))
|
||||
media (rtry/invoke cfg create-file-thumbnail! params)]
|
||||
{:uri (files/resolve-public-uri (:id media))})))))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue