3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-05-31 21:31:38 +02:00
Code Issues Projects Releases Packages Wiki Activity

✨ Add safety mechanism for direct object deletion

Browse source 
The main objective is prevent deletion of objects that can leave
unreachable orphan objects which we are unable to correctly track.

Additionally, this commit includes:

1. Properly implement safe cascade deletion of all participating
   tables on soft deletion in the objects-gc task;

2. Make the file thumbnail related tables also participate in the
   touch/refcount mechanism applyign to the same safety checks;

3. Add helper for db query lazy iteration using PostgreSQL support
   for server side cursors;

4. Fix efficiency issues on gc related task using server side
   cursors instead of custom chunked iteration for processing data.

   The problem resided when a large chunk of rows that has identical
   value on the deleted_at column and the chunk size is small (the
   default); when the custom chunked iteration only reads a first N
   items and skip the rest of the set to the next run.

   This has caused many objects to remain pending to be eliminated,
   taking up space for longer than expected. The server side cursor
   based iteration does not has this problem and iterates correctly
   over all objects.

5. Fix refcount issues on font variant deletion RPC methods
This commit is contained in:
Andrey Antukh 2023-12-29 15:21:14 +01:00
parent e6fb96c4c2
commit addb392ecc
37 changed files with 1918 additions and 1026 deletions
Download patch file Download diff file Expand all files Collapse all files

64
backend/test/backend_tests/storage_test.clj
View file

@ -113,7 +113,7 @@
(let [res (th/run-task! :storage-gc-deleted {})]
(t/is (= 1 (:deleted res))))
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object;"])]
(let [res (th/db-exec-one! ["select count(*) from storage_object;"])]
(t/is (= 2 (:count res))))))
(t/deftest test-touched-gc-task-1
@ -156,29 +156,33 @@
(t/is (= (:media-id result-1) (:media-id result-2)))
;; now we proceed to manually delete one file-media-object
(db/exec-one! th/*pool* ["delete from file_media_object where id = ?" (:id result-1)])
(th/db-update! :file-media-object
{:deleted-at (dt/now)}
{:id (:id result-1)})
;; run the objects gc task for permanent deletion
(let [res (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
;; check that we still have all the storage objects
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object"])]
(let [res (th/db-exec-one! ["select count(*) from storage_object"])]
(t/is (= 2 (:count res))))
;; now check if the storage objects are touched
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where touched_at is not null"])]
(let [res (th/db-exec-one! ["select count(*) from storage_object where touched_at is not null"])]
(t/is (= 2 (:count res))))
;; run the touched gc task
(let [task (:app.storage/gc-touched-task th/*system*)
res (task {})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
;; now check that there are no touched objects
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where touched_at is not null"])]
(let [res (th/db-exec-one! ["select count(*) from storage_object where touched_at is not null"])]
(t/is (= 0 (:count res))))
;; now check that all objects are marked to be deleted
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where deleted_at is not null"])]
(let [res (th/db-exec-one! ["select count(*) from storage_object where deleted_at is not null"])]
(t/is (= 0 (:count res)))))))
@ -231,31 +235,35 @@
(t/is (nil? (:error out2)))
;; run the touched gc task
(let [task (:app.storage/gc-touched-task th/*system*)
res (task {})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 5 (:freeze res)))
(t/is (= 0 (:delete res)))
(let [result-1 (:result out1)
result-2 (:result out2)]
;; now we proceed to manually delete one team-font-variant
(db/exec-one! th/*pool* ["delete from team_font_variant where id = ?" (:id result-2)])
(th/db-update! :team-font-variant
{:deleted-at (dt/now)}
{:id (:id result-2)})
;; run the objects gc task for permanent deletion
(let [res (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
;; revert touched state to all storage objects
(db/exec-one! th/*pool* ["update storage_object set touched_at=now()"])
(th/db-exec-one! ["update storage_object set touched_at=now()"])
;; Run the task again
(let [res (task {})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 2 (:freeze res)))
(t/is (= 3 (:delete res))))
;; now check that there are no touched objects
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where touched_at is not null"])]
(let [res (th/db-exec-one! ["select count(*) from storage_object where touched_at is not null"])]
(t/is (= 0 (:count res))))
;; now check that all objects are marked to be deleted
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where deleted_at is not null"])]
(let [res (th/db-exec-one! ["select count(*) from storage_object where deleted_at is not null"])]
(t/is (= 3 (:count res))))))))
(t/deftest test-touched-gc-task-3
@ -289,28 +297,28 @@
result-2 (:result out2)]
;; now we proceed to manually mark all storage objects touched
(db/exec-one! th/*pool* ["update storage_object set touched_at=now()"])
(th/db-exec! ["update storage_object set touched_at=now()"])
;; run the touched gc task
(let [task (:app.storage/gc-touched-task th/*system*)
res (task {})]
(let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
;; check that we have all object in the db
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where deleted_at is null"])]
(t/is (= 2 (:count res)))))
(let [rows (th/db-exec! ["select * from storage_object"])]
(t/is (= 2 (count rows)))))
;; now we proceed to manually delete all file_media_object
(db/exec-one! th/*pool* ["delete from file_media_object"])
(th/db-exec! ["update file_media_object set deleted_at = now()"])
(let [res (th/run-task! "objects-gc" {:min-age 0})]
(t/is (= 2 (:processed res))))
;; run the touched gc task
(let [task (:app.storage/gc-touched-task th/*system*)
res (task {})]
(let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))
;; check that we have all no objects
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where deleted_at is null"])]
(t/is (= 0 (:count res))))))
(let [rows (th/db-exec! ["select * from storage_object where deleted_at is null"])]
(t/is (= 0 (count rows))))))