3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-05-17 18:36:11 +02:00
Code Issues Projects Releases Packages Wiki Activity

✨ Add safety mechanism for direct object deletion

Browse source 
The main objective is prevent deletion of objects that can leave
unreachable orphan objects which we are unable to correctly track.

Additionally, this commit includes:

1. Properly implement safe cascade deletion of all participating
   tables on soft deletion in the objects-gc task;

2. Make the file thumbnail related tables also participate in the
   touch/refcount mechanism applyign to the same safety checks;

3. Add helper for db query lazy iteration using PostgreSQL support
   for server side cursors;

4. Fix efficiency issues on gc related task using server side
   cursors instead of custom chunked iteration for processing data.

   The problem resided when a large chunk of rows that has identical
   value on the deleted_at column and the chunk size is small (the
   default); when the custom chunked iteration only reads a first N
   items and skip the rest of the set to the next run.

   This has caused many objects to remain pending to be eliminated,
   taking up space for longer than expected. The server side cursor
   based iteration does not has this problem and iterates correctly
   over all objects.

5. Fix refcount issues on font variant deletion RPC methods
This commit is contained in:
Andrey Antukh 2023-12-29 15:21:14 +01:00
parent e6fb96c4c2
commit addb392ecc
37 changed files with 1918 additions and 1026 deletions
Download patch file Download diff file Expand all files Collapse all files

15
backend/test/backend_tests/helpers.clj
View file

@ -175,12 +175,11 @@
" WHERE table_schema = 'public' "
" AND table_name != 'migrations';")]
(db/with-atomic [conn *pool*]
(db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])
(db/exec-one! conn ["SET LOCAL rules.deletion_protection TO off"])
(let [result (->> (db/exec! conn [sql])
(map :table-name)
(remove #(= "task" %)))
sql (str "TRUNCATE "
(apply str (interpose ", " result))
" CASCADE;")]
(remove #(= "task" %)))]
(doseq [table result]
(db/exec! conn [(str "delete from " table ";")]))))
@ -433,11 +432,11 @@
(us/pretty-explain data))
(= :params-validation (:code data))
(app.common.pprint/pprint
(println
(sm/humanize-explain (::sm/explain data)))
(= :data-validation (:code data))
(app.common.pprint/pprint
(println
(sm/humanize-explain (::sm/explain data)))
(= :service-error (:type data))
@ -512,6 +511,10 @@
[sql]
(db/exec! *pool* sql))
(defn db-exec-one!
[sql]
(db/exec-one! *pool* sql))
(defn db-delete!
[& params]
(apply db/delete! *pool* params))

152
backend/test/backend_tests/rpc_file_test.clj
View file

@ -149,7 +149,7 @@
shape-id (uuid/random)]
;; Preventive file-gc
(let [res (th/run-task! "file-gc" {:min-age 0})]
(let [res (th/run-task! :file-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
;; Check the number of fragments before adding the page
@ -175,7 +175,7 @@
(t/is (= 2 (count rows))))
;; The file-gc should remove unused fragments
(let [res (th/run-task! "file-gc" {:min-age 0})]
(let [res (th/run-task! :file-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
@ -203,7 +203,7 @@
(t/is (= 3 (count rows))))
;; The file-gc should remove unused fragments
(let [res (th/run-task! "file-gc" {:min-age 0})]
(let [res (th/run-task! :file-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
;; Check the number of fragments; should be 3 because changes
@ -220,12 +220,23 @@
;; The file-gc should remove fragments related to changes
;; snapshots previously deleted.
(let [res (th/run-task! "file-gc" {:min-age 0})]
(let [res (th/run-task! :file-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
;; Check the number of fragments;
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)})]
(t/is (= 2 (count rows)))))))
;; (pp/pprint rows)
(t/is (= 3 (count rows)))
(t/is (= 2 (count (remove (comp some? :deleted-at) rows)))))
(let [res (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)})]
(t/is (= 2 (count rows))))
)))
(t/deftest file-gc-task-with-thumbnails
(letfn [(add-file-media-object [& {:keys [profile-id file-id]}]
@ -301,17 +312,16 @@
;; freeze because of the deduplication (we have uploaded 2 times
;; the same files).
(let [task (:app.storage/gc-touched-task th/*system*)
res (task {:min-age (dt/duration 0)})]
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
;; run the file-gc task immediately without forced min-age
(let [res (th/run-task! "file-gc")]
(let [res (th/run-task! :file-gc)]
(t/is (= 0 (:processed res))))
;; run the task again
(let [res (th/run-task! "file-gc" {:min-age 0})]
(let [res (th/run-task! :file-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
;; retrieve file and check trimmed attribute
@ -319,8 +329,17 @@
(t/is (true? (:has-media-trimmed row))))
;; check file media objects
(let [rows (th/db-exec! ["select * from file_media_object where file_id = ?" (:id file)])]
(t/is (= 1 (count rows))))
(let [rows (th/db-query :file-media-object {:file-id (:id file)})]
(t/is (= 2 (count rows)))
(t/is (= 1 (count (remove (comp some? :deleted-at) rows)))))
(let [res (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 2 (:processed res))))
;; check file media objects
(let [rows (th/db-query :file-media-object {:file-id (:id file)})]
(t/is (= 1 (count rows)))
(t/is (= 1 (count (remove (comp some? :deleted-at) rows)))))
;; The underlying storage objects are still available.
(t/is (some? (sto/get-object storage (:media-id fmo2))))
@ -340,15 +359,16 @@
;; Now, we have deleted the usage of pointers to the
;; file-media-objects, if we paste file-gc, they should be marked
;; as deleted.
(let [task (:app.tasks.file-gc/handler th/*system*)
res (task {:min-age (dt/duration 0)})]
(let [res (th/run-task! :file-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
(let [res (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
;; Now that file-gc have deleted the file-media-object usage,
;; lets execute the touched-gc task, we should see that two of
;; them are marked to be deleted.
(let [task (:app.storage/gc-touched-task th/*system*)
res (task {:min-age (dt/duration 0)})]
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))
@ -457,11 +477,14 @@
:strokes [{:opacity 1 :stroke-image {:id (:id fmo5) :width 100 :height 100 :mtype "image/jpeg"}}]})}])
;; run the file-gc task immediately without forced min-age
(let [res (th/run-task! "file-gc")]
(let [res (th/run-task! :file-gc)]
(t/is (= 0 (:processed res))))
;; run the task again
(let [res (th/run-task! "file-gc" {:min-age 0})]
(let [res (th/run-task! :file-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
(let [res (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
;; retrieve file and check trimmed attribute
@ -494,15 +517,16 @@
;; Now, we have deleted the usage of pointers to the
;; file-media-objects, if we paste file-gc, they should be marked
;; as deleted.
(let [task (:app.tasks.file-gc/handler th/*system*)
res (task {:min-age (dt/duration 0)})]
(let [res (th/run-task! :file-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
(let [res (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 5 (:processed res))))
;; Now that file-gc have deleted the file-media-object usage,
;; lets execute the touched-gc task, we should see that two of
;; them are marked to be deleted.
(let [task (:app.storage/gc-touched-task th/*system*)
res (task {:min-age (dt/duration 0)})]
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))
@ -515,7 +539,6 @@
(t/is (nil? (sto/get-object storage (:media-id fmo2))))
(t/is (nil? (sto/get-object storage (:media-id fmo1)))))))
(t/deftest file-gc-task-with-object-thumbnails
(letfn [(insert-file-object-thumbnail! [& {:keys [profile-id file-id page-id frame-id]}]
(let [object-id (thc/fmt-object-id file-id page-id frame-id "frame")
@ -609,16 +632,16 @@
;; because of the deduplication (we have uploaded 2 times the
;; same files).
(let [res (th/run-task! "storage-gc-touched" {:min-age (dt/duration 0)})]
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(t/is (= 1 (:freeze res)))
(t/is (= 0 (:delete res))))
;; run the file-gc task immediately without forced min-age
(let [res (th/run-task! "file-gc")]
(let [res (th/run-task! :file-gc)]
(t/is (= 0 (:processed res))))
;; run the task again
(let [res (th/run-task! "file-gc" {:min-age 0})]
(let [res (th/run-task! :file-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
;; retrieve file and check trimmed attribute
@ -648,22 +671,29 @@
:page-id page-id
:id frame-id-2}])
(let [res (th/run-task! "file-gc" {:min-age (dt/duration 0)})]
(let [res (th/run-task! :file-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
(let [rows (th/db-exec! ["select * from file_tagged_object_thumbnail where file_id = ?" file-id])]
;; (pp/pprint rows)
(t/is (= 1 (count rows)))
(let [rows (th/db-query :file-tagged-object-thumbnail {:file-id file-id})]
(t/is (= 2 (count rows)))
(t/is (= 1 (count (remove (comp some? :deleted-at) rows))))
(t/is (= (thc/fmt-object-id file-id page-id frame-id-1 "frame")
(-> rows first :object-id))))
;; Now that file-gc have deleted the object thumbnail lets
;; Now that file-gc have marked for deletion the object
;; thumbnail lets execute the objects-gc task which remove
;; the rows and mark as touched the storage object rows
(let [res (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 2 (:processed res))))
;; Now that objects-gc have deleted the object thumbnail lets
;; execute the touched-gc task
(let [res (th/run-task! "storage-gc-touched" {:min-age (dt/duration 0)})]
(let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
(t/is (= 1 (:freeze res))))
;; check file media objects
(let [rows (th/db-exec! ["select * from storage_object where deleted_at is null"])]
(let [rows (th/db-query :storage-object {:deleted-at nil})]
;; (pp/pprint rows)
(t/is (= 1 (count rows))))
@ -676,31 +706,32 @@
:page-id page-id
:id frame-id-1}])
(let [res (th/run-task! "file-gc" {:min-age (dt/duration 0)})]
(let [res (th/run-task! :file-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
(let [rows (th/db-exec! ["select * from file_tagged_object_thumbnail where file_id = ?" file-id])]
(t/is (= 0 (count rows))))
(let [rows (th/db-query :file-tagged-object-thumbnail {:file-id file-id})]
(t/is (= 1 (count rows)))
(t/is (= 0 (count (remove (comp some? :deleted-at) rows)))))
(let [res (th/run-task! :objects-gc {:min-age 0})]
;; (pp/pprint res)
(t/is (= 1 (:processed res))))
;; We still have th storage objects in the table
(let [rows (th/db-exec! ["select * from storage_object where deleted_at is null"])]
(let [rows (th/db-query :storage-object {:deleted-at nil})]
;; (pp/pprint rows)
(t/is (= 1 (count rows))))
;; Now that file-gc have deleted the object thumbnail lets
;; execute the touched-gc task
(let [res (th/run-task! "storage-gc-touched" {:min-age (dt/duration 0)})]
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(t/is (= 1 (:delete res))))
;; check file media objects
(let [rows (th/db-exec! ["select * from storage_object where deleted_at is null"])]
(let [rows (th/db-query :storage-object {:deleted-at nil})]
;; (pp/pprint rows)
(t/is (= 0 (count rows)))))))
(t/deftest permissions-checks-creating-file
(let [profile1 (th/create-profile* 1)
profile2 (th/create-profile* 2)
@ -811,13 +842,12 @@
(t/is (th/ex-of-type? error :not-found))))
(t/deftest deletion
(let [task (:app.tasks.objects-gc/handler th/*system*)
profile1 (th/create-profile* 1)
(let [profile1 (th/create-profile* 1)
file (th/create-file* 1 {:project-id (:default-project-id profile1)
:profile-id (:id profile1)})]
;; file is not deleted because it does not meet all
;; conditions to be deleted.
(let [result (task {:min-age (dt/duration 0)})]
(let [result (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 0 (:processed result))))
;; query the list of files
@ -848,7 +878,7 @@
(t/is (= 0 (count result)))))
;; run permanent deletion (should be noop)
(let [result (task {:min-age (dt/duration {:minutes 1})})]
(let [result (th/run-task! :objects-gc {:min-age (dt/duration {:minutes 1})})]
(t/is (= 0 (:processed result))))
;; query the list of file libraries of a after hard deletion
@ -862,7 +892,7 @@
(t/is (= 0 (count result)))))
;; run permanent deletion
(let [result (task {:min-age (dt/duration 0)})]
(let [result (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 1 (:processed result))))
;; query the list of file libraries of a after hard deletion
@ -874,7 +904,8 @@
(let [error (:error out)
error-data (ex-data error)]
(t/is (th/ex-info? error))
(t/is (= (:type error-data) :not-found))))))
(t/is (= (:type error-data) :not-found))))
))
(t/deftest object-thumbnails-ops
@ -1075,7 +1106,7 @@
(th/sleep 300)
;; run the task
(let [res (th/run-task! "file-gc" {:min-age 0})]
(let [res (th/run-task! :file-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
;; check that object thumbnails are still here
@ -1104,13 +1135,19 @@
(t/is (= 2 (count res))))
;; run the task again
(let [res (th/run-task! "file-gc" {:min-age 0})]
(let [res (th/run-task! :file-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
;; check that the unknown frame thumbnail is deleted
(let [res (th/db-exec! ["select * from file_tagged_object_thumbnail"])]
(t/is (= 1 (count res)))))))
(let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})]
(t/is (= 2 (count rows)))
(t/is (= 1 (count (remove (comp some? :deleted-at) rows)))))
(let [res (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 2 (:processed res))))
(let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})]
(t/is (= 1 (count rows)))))))
(t/deftest file-thumbnail-ops
(let [prof (th/create-profile* 1 {:is-active true})
@ -1155,12 +1192,19 @@
(t/testing "gc task"
;; make the file eligible for GC waiting 300ms (configured
;; timeout for testing)
(th/sleep 300)
(let [res (th/run-task! :file-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
(let [res (th/run-task! "file-gc" {:min-age 0})]
(let [rows (th/db-query :file-thumbnail {:file-id (:id file)})]
(t/is (= 2 (count rows)))
(t/is (= 1 (count (remove (comp some? :deleted-at) rows)))))
(let [res (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
(let [rows (th/db-query :file-thumbnail {:file-id (:id file)})]
(t/is (= 1 (count rows)))))))

55
backend/test/backend_tests/rpc_file_thumbnails_test.clj
View file

@ -6,6 +6,7 @@
(ns backend-tests.rpc-file-thumbnails-test
(:require
[app.common.pprint :as pp]
[app.common.thumbnails :as thc]
[app.common.types.shape :as cts]
[app.common.uuid :as uuid]
@ -114,9 +115,12 @@
;; Run the File GC task that should remove unused file object
;; thumbnails
(let [result (th/run-task! :file-gc {:min-age (dt/duration 0)})]
(let [result (th/run-task! :file-gc {:min-age 0})]
(t/is (= 1 (:processed result))))
(let [result (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 2 (:processed result))))
;; check if row2 related thumbnail row still exists
(let [[row :as rows] (th/db-query :file-tagged-object-thumbnail
{:file-id (:id file)}
@ -141,7 +145,7 @@
;; Run the storage gc deleted task, it should permanently delete
;; all storage objects related to the deleted thumbnails
(let [result (th/run-task! :storage-gc-deleted {:min-age (dt/duration 0)})]
(let [result (th/run-task! :storage-gc-deleted {:min-age 0})]
(t/is (= 1 (:deleted result))))
(t/is (nil? (sto/get-object storage (:media-id row1))))
@ -188,13 +192,12 @@
(let [[row1 row2 :as rows] (th/db-query :file-thumbnail
{:file-id (:id file)}
{:order-by [[:created-at :asc]]})]
{:order-by [[:revn :asc]]})]
(t/is (= 2 (count rows)))
(t/is (= (:file-id data1) (:file-id row1)))
(t/is (= (:revn data1) (:revn row1)))
(t/is (uuid? (:media-id row1)))
(t/is (= (:file-id data2) (:file-id row2)))
(t/is (= (:revn data2) (:revn row2)))
(t/is (uuid? (:media-id row2)))
@ -215,7 +218,10 @@
;; Run the File GC task that should remove unused file object
;; thumbnails
(let [result (th/run-task! :file-gc {:min-age (dt/duration 0)})]
(let [result (th/run-task! :file-gc {:min-age 0})]
(t/is (= 1 (:processed result))))
(let [result (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 1 (:processed result))))
;; check if row1 related thumbnail row still exists
@ -227,6 +233,9 @@
(t/is (= (:object-id data1) (:object-id row)))
(t/is (uuid? (:media-id row1))))
(let [result (th/run-task! :storage-gc-touched {:min-age 0})]
(t/is (= 1 (:delete result))))
;; Check if storage objects still exists after file-gc
(t/is (nil? (sto/get-object storage (:media-id row1))))
(t/is (some? (sto/get-object storage (:media-id row2))))
@ -236,10 +245,42 @@
;; Run the storage gc deleted task, it should permanently delete
;; all storage objects related to the deleted thumbnails
(let [result (th/run-task! :storage-gc-deleted {:min-age (dt/duration 0)})]
(let [result (th/run-task! :storage-gc-deleted {:min-age 0})]
(t/is (= 1 (:deleted result))))
(t/is (some? (sto/get-object storage (:media-id row2)))))))
(t/is (some? (sto/get-object storage (:media-id row2))))
)))
(t/deftest error-on-direct-storage-obj-deletion
(let [storage (::sto/storage th/*system*)
profile (th/create-profile* 1)
file (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)
:is-shared false
:revn 3})
data1 {::th/type :create-file-thumbnail
::rpc/profile-id (:id profile)
:file-id (:id file)
:revn 2
:media {:filename "sample.jpg"
:size 7923
:path (th/tempfile "backend_tests/test_files/sample2.jpg")
:mtype "image/jpeg"}}]
(let [out (th/command! data1)]
;; (th/print-result! out)
(t/is (nil? (:error out)))
(t/is (contains? (:result out) :uri)))
(let [[row1 :as rows] (th/db-query :file-thumbnail {:file-id (:id file)})]
(t/is (= 1 (count rows)))
(t/is (thrown? org.postgresql.util.PSQLException
(th/db-delete! :storage-object {:id (:media-id row1)}))))))
(t/deftest get-file-object-thumbnail
(let [storage (::sto/storage th/*system*)

189
backend/test/backend_tests/rpc_font_test.clj
View file

@ -92,3 +92,192 @@
:font-family
:font-weight
:font-style))))
(t/deftest font-deletion-1
(let [prof (th/create-profile* 1 {:is-active true})
team-id (:default-team-id prof)
proj-id (:default-project-id prof)
font-id (uuid/custom 10 1)
data1 (-> (io/resource "backend_tests/test_files/font-1.woff")
io/input-stream
io/read-as-bytes)
data2 (-> (io/resource "backend_tests/test_files/font-2.woff")
io/input-stream
io/read-as-bytes)]
;; Create front variant
(let [params {::th/type :create-font-variant
::rpc/profile-id (:id prof)
:team-id team-id
:font-id font-id
:font-family "somefont"
:font-weight 400
:font-style "normal"
:data {"font/woff" data1}}
out (th/command! params)]
;; (th/print-result! out)
(t/is (nil? (:error out))))
(let [params {::th/type :create-font-variant
::rpc/profile-id (:id prof)
:team-id team-id
:font-id font-id
:font-family "somefont"
:font-weight 500
:font-style "normal"
:data {"font/woff" data2}}
out (th/command! params)]
;; (th/print-result! out)
(t/is (nil? (:error out))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(t/is (= 6 (:freeze res))))
(let [params {::th/type :delete-font
::rpc/profile-id (:id prof)
:team-id team-id
:id font-id}
out (th/command! params)]
;; (th/print-result! out)
(t/is (nil? (:error out)))
(t/is (nil? (:result out))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(t/is (= 6 (:freeze res)))
(t/is (= 0 (:delete res))))
(let [res (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 2 (:processed res))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(t/is (= 0 (:freeze res)))
(t/is (= 6 (:delete res))))
))
(t/deftest font-deletion-2
(let [prof (th/create-profile* 1 {:is-active true})
team-id (:default-team-id prof)
proj-id (:default-project-id prof)
font-id (uuid/custom 10 1)
data1 (-> (io/resource "backend_tests/test_files/font-1.woff")
io/input-stream
io/read-as-bytes)
data2 (-> (io/resource "backend_tests/test_files/font-2.woff")
io/input-stream
io/read-as-bytes)]
;; Create front variant
(let [params {::th/type :create-font-variant
::rpc/profile-id (:id prof)
:team-id team-id
:font-id font-id
:font-family "somefont"
:font-weight 400
:font-style "normal"
:data {"font/woff" data1}}
out (th/command! params)]
;; (th/print-result! out)
(t/is (nil? (:error out))))
(let [params {::th/type :create-font-variant
::rpc/profile-id (:id prof)
:team-id team-id
:font-id (uuid/custom 10 2)
:font-family "somefont"
:font-weight 400
:font-style "normal"
:data {"font/woff" data2}}
out (th/command! params)]
;; (th/print-result! out)
(t/is (nil? (:error out))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(t/is (= 6 (:freeze res))))
(let [params {::th/type :delete-font
::rpc/profile-id (:id prof)
:team-id team-id
:id font-id}
out (th/command! params)]
;; (th/print-result! out)
(t/is (nil? (:error out)))
(t/is (nil? (:result out))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(t/is (= 3 (:freeze res)))
(t/is (= 0 (:delete res))))
(let [res (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(t/is (= 0 (:freeze res)))
(t/is (= 3 (:delete res))))
))
(t/deftest font-deletion-3
(let [prof (th/create-profile* 1 {:is-active true})
team-id (:default-team-id prof)
proj-id (:default-project-id prof)
font-id (uuid/custom 10 1)
data1 (-> (io/resource "backend_tests/test_files/font-1.woff")
io/input-stream
io/read-as-bytes)
data2 (-> (io/resource "backend_tests/test_files/font-2.woff")
io/input-stream
io/read-as-bytes)
params1 {::th/type :create-font-variant
::rpc/profile-id (:id prof)
:team-id team-id
:font-id font-id
:font-family "somefont"
:font-weight 400
:font-style "normal"
:data {"font/woff" data1}}
params2 {::th/type :create-font-variant
::rpc/profile-id (:id prof)
:team-id team-id
:font-id font-id
:font-family "somefont"
:font-weight 500
:font-style "normal"
:data {"font/woff" data2}}
out1 (th/command! params1)
out2 (th/command! params2)]
;; (th/print-result! out1)
(t/is (nil? (:error out1)))
(t/is (nil? (:error out2)))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(t/is (= 6 (:freeze res))))
(let [params {::th/type :delete-font-variant
::rpc/profile-id (:id prof)
:team-id team-id
:id (-> out1 :result :id)}
out (th/command! params)]
;; (th/print-result! out)
(t/is (nil? (:error out)))
(t/is (nil? (:result out))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(t/is (= 3 (:freeze res)))
(t/is (= 0 (:delete res))))
(let [res (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(t/is (= 0 (:freeze res)))
(t/is (= 3 (:delete res))))
))

76
backend/test/backend_tests/rpc_profile_test.clj
View file

@ -125,7 +125,7 @@
;; profile is not deleted because it does not meet all
;; conditions to be deleted.
(let [result (th/run-task! :objects-gc {:min-age (dt/duration 0)})]
(let [result (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 0 (:processed result))))
;; Request profile to be deleted
@ -144,8 +144,16 @@
(t/is (= 1 (count (:result out)))))
;; execute permanent deletion task
(let [result (th/run-task! :objects-gc {:min-age (dt/duration "-1m")})]
(t/is (= 2 (:processed result))))
(let [result (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 1 (:processed result))))
(let [row (th/db-get :team
{:id (:default-team-id prof)}
{::db/remove-deleted? false})]
(t/is (nil? (:deleted-at row))))
(let [result (th/run-task! :orphan-teams-gc {:min-age 0})]
(t/is (= 1 (:processed result))))
(let [row (th/db-get :team
{:id (:default-team-id prof)}
@ -158,67 +166,9 @@
out (th/command! params)]
;; (th/print-result! out)
(let [result (:result out)]
(t/is (= uuid/zero (:id result)))))))
(t/is (= uuid/zero (:id result)))))
(t/deftest profile-immediate-deletion
(let [prof1 (th/create-profile* 1)
prof2 (th/create-profile* 2)
file (th/create-file* 1 {:profile-id (:id prof1)
:project-id (:default-project-id prof1)
:is-shared false})
team (th/create-team* 1 {:profile-id (:id prof1)})
_ (th/create-team-role* {:team-id (:id team)
:profile-id (:id prof2)
:role :admin})]
;; profile is not deleted because it does not meet all
;; conditions to be deleted.
(let [result (th/run-task! :objects-gc {:min-age (dt/duration 0)})]
(t/is (= 0 (:orphans result)))
(t/is (= 0 (:processed result))))
;; just delete the profile
(th/db-delete! :profile {:id (:id prof1)})
;; query files after profile deletion, expecting not found
(let [params {::th/type :get-project-files
::rpc/profile-id (:id prof1)
:project-id (:default-project-id prof1)}
out (th/command! params)]
;; (th/print-result! out)
(t/is (not (th/success? out)))
(let [edata (-> out :error ex-data)]
(t/is (= :not-found (:type edata)))))
;; the files and projects still exists on the database
(let [files (th/db-query :file {:project-id (:default-project-id prof1)})
projects (th/db-query :project {:team-id (:default-team-id prof1)})]
(t/is (= 1 (count files)))
(t/is (= 1 (count projects))))
;; execute the gc task
(let [result (th/run-task! :objects-gc {:min-age (dt/duration "-1m")})]
(t/is (= 1 (:processed result)))
(t/is (= 1 (:orphans result))))
;; Check the deletion flag on the default profile team
(let [row (th/db-get :team
{:id (:default-team-id prof1)}
{::db/remove-deleted? false})]
(t/is (dt/instant? (:deleted-at row))))
;; Check the deletion flag on the shared team
(let [row (th/db-get :team
{:id (:id team)}
{::db/remove-deleted? false})]
(t/is (nil? (:deleted-at row))))
;; Check the roles on the shared team
(let [rows (th/db-query :team-profile-rel {:team-id (:id team)})]
(t/is (= 1 (count rows)))
(t/is (= (:id prof2) (get-in rows [0 :profile-id])))
(t/is (= false (get-in rows [0 :is-owner]))))))
))
(t/deftest registration-domain-whitelist
(let [whitelist #{"gmail.com" "hey.com" "ya.ru"}]

10
backend/test/backend_tests/rpc_project_test.clj
View file

@ -172,14 +172,13 @@
(t/deftest test-deletion
(let [task (:app.tasks.objects-gc/handler th/*system*)
profile1 (th/create-profile* 1)
(let [profile1 (th/create-profile* 1)
project (th/create-project* 1 {:team-id (:default-team-id profile1)
:profile-id (:id profile1)})]
;; project is not deleted because it does not meet all
;; conditions to be deleted.
(let [result (task {:min-age (dt/duration 0)})]
(let [result (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 0 (:processed result))))
;; query the list of projects
@ -187,6 +186,7 @@
::rpc/profile-id (:id profile1)
:team-id (:default-team-id profile1)}
out (th/command! data)]
;; (th/print-result! out)
(t/is (nil? (:error out)))
(let [result (:result out)]
@ -210,7 +210,7 @@
(t/is (= 1 (count result)))))
;; run permanent deletion (should be noop)
(let [result (task {:min-age (dt/duration {:minutes 1})})]
(let [result (th/run-task! :objects-gc {:min-age (dt/duration {:minutes 1})})]
(t/is (= 0 (:processed result))))
;; query the list of files of a after soft deletion
@ -224,7 +224,7 @@
(t/is (= 0 (count result)))))
;; run permanent deletion
(let [result (task {:min-age (dt/duration 0)})]
(let [result (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 1 (:processed result))))
;; query the list of files of a after hard deletion

186
backend/test/backend_tests/rpc_team_test.clj
View file

@ -269,76 +269,6 @@
(t/is (= 1 (count members)))
(t/is (true? (-> members first :can-edit))))))))
(t/deftest team-deletion
(let [profile1 (th/create-profile* 1 {:is-active true})
team (th/create-team* 1 {:profile-id (:id profile1)})
pool (:app.db/pool th/*system*)
data {::th/type :delete-team
::rpc/profile-id (:id profile1)
:team-id (:id team)}]
;; team is not deleted because it does not meet all
;; conditions to be deleted.
(let [result (th/run-task! :objects-gc {:min-age (dt/duration 0)})]
(t/is (= 0 (:processed result))))
;; query the list of teams
(let [data {::th/type :get-teams
::rpc/profile-id (:id profile1)}
out (th/command! data)]
;; (th/print-result! out)
(t/is (th/success? out))
(let [result (:result out)]
(t/is (= 2 (count result)))
(t/is (= (:id team) (get-in result [1 :id])))
(t/is (= (:default-team-id profile1) (get-in result [0 :id])))))
;; Request team to be deleted
(let [params {::th/type :delete-team
::rpc/profile-id (:id profile1)
:id (:id team)}
out (th/command! params)]
(t/is (th/success? out)))
;; query the list of teams after soft deletion
(let [data {::th/type :get-teams
::rpc/profile-id (:id profile1)}
out (th/command! data)]
;; (th/print-result! out)
(t/is (th/success? out))
(let [result (:result out)]
(t/is (= 1 (count result)))
(t/is (= (:default-team-id profile1) (get-in result [0 :id])))))
;; run permanent deletion (should be noop)
(let [result (th/run-task! :objects-gc {:min-age (dt/duration {:minutes 1})})]
(t/is (= 0 (:processed result))))
;; query the list of projects after hard deletion
(let [data {::th/type :get-projects
::rpc/profile-id (:id profile1)
:team-id (:id team)}
out (th/command! data)]
;; (th/print-result! out)
(t/is (not (th/success? out)))
(let [edata (-> out :error ex-data)]
(t/is (= :not-found (:type edata)))))
;; run permanent deletion
(let [result (th/run-task! :objects-gc {:min-age (dt/duration 0)})]
(t/is (= 1 (:processed result))))
;; query the list of projects of a after hard deletion
(let [data {::th/type :get-projects
::rpc/profile-id (:id profile1)
:team-id (:id team)}
out (th/command! data)]
;; (th/print-result! out)
(t/is (not (th/success? out)))
(let [edata (-> out :error ex-data)]
(t/is (= :not-found (:type edata)))))))
(t/deftest query-team-invitations
(let [prof (th/create-profile* 1 {:is-active true})
team (th/create-team* 1 {:profile-id (:id prof)})
@ -418,3 +348,119 @@
(t/is (th/success? out))
(t/is (nil? (:result out)))
(t/is (nil? res)))))
(t/deftest team-deletion-1
(let [profile1 (th/create-profile* 1 {:is-active true})
team (th/create-team* 1 {:profile-id (:id profile1)})
pool (:app.db/pool th/*system*)
data {::th/type :delete-team
::rpc/profile-id (:id profile1)
:team-id (:id team)}]
;; team is not deleted because it does not meet all
;; conditions to be deleted.
(let [result (th/run-task! :objects-gc {:min-age (dt/duration 0)})]
(t/is (= 0 (:processed result))))
;; query the list of teams
(let [data {::th/type :get-teams
::rpc/profile-id (:id profile1)}
out (th/command! data)]
;; (th/print-result! out)
(t/is (th/success? out))
(let [result (:result out)]
(t/is (= 2 (count result)))
(t/is (= (:id team) (get-in result [1 :id])))
(t/is (= (:default-team-id profile1) (get-in result [0 :id])))))
;; Request team to be deleted
(let [params {::th/type :delete-team
::rpc/profile-id (:id profile1)
:id (:id team)}
out (th/command! params)]
(t/is (th/success? out)))
;; query the list of teams after soft deletion
(let [data {::th/type :get-teams
::rpc/profile-id (:id profile1)}
out (th/command! data)]
;; (th/print-result! out)
(t/is (th/success? out))
(let [result (:result out)]
(t/is (= 1 (count result)))
(t/is (= (:default-team-id profile1) (get-in result [0 :id])))))
;; run permanent deletion (should be noop)
(let [result (th/run-task! :objects-gc {:min-age (dt/duration {:minutes 1})})]
(t/is (= 0 (:processed result))))
;; query the list of projects after hard deletion
(let [data {::th/type :get-projects
::rpc/profile-id (:id profile1)
:team-id (:id team)}
out (th/command! data)]
;; (th/print-result! out)
(t/is (not (th/success? out)))
(let [edata (-> out :error ex-data)]
(t/is (= :not-found (:type edata)))))
;; run permanent deletion
(let [result (th/run-task! :objects-gc {:min-age (dt/duration 0)})]
(t/is (= 2 (:processed result))))
;; query the list of projects of a after hard deletion
(let [data {::th/type :get-projects
::rpc/profile-id (:id profile1)
:team-id (:id team)}
out (th/command! data)]
;; (th/print-result! out)
(t/is (not (th/success? out)))
(let [edata (-> out :error ex-data)]
(t/is (= :not-found (:type edata)))))))
(t/deftest team-deletion-2
(let [storage (-> (:app.storage/storage th/*system*)
(assoc ::sto/backend :assets-fs))
prof (th/create-profile* 1)
team (th/create-team* 1 {:profile-id (:id prof)})
proj (th/create-project* 1 {:profile-id (:id prof)
:team-id (:id team)})
file (th/create-file* 1 {:profile-id (:id prof)
:project-id (:default-project-id team)
:is-shared false})
mfile {:filename "sample.jpg"
:path (th/tempfile "backend_tests/test_files/sample.jpg")
:mtype "image/jpeg"
:size 312043}]
(let [params {::th/type :upload-file-media-object
::rpc/profile-id (:id prof)
:file-id (:id file)
:is-local true
:name "testfile"
:content mfile}
out (th/command! params)]
(t/is (nil? (:error out))))
(let [params {::th/type :delete-team
::rpc/profile-id (:id prof)
:id (:id team)}
out (th/command! params)]
#_(th/print-result! out)
(t/is (nil? (:error out))))
(let [rows (th/db-exec! ["select * from team where id = ?" (:id team)])]
(t/is (= 1 (count rows)))
(t/is (dt/instant? (:deleted-at (first rows)))))
(let [result (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 5 (:processed result))))
))

64
backend/test/backend_tests/storage_test.clj
View file

@ -113,7 +113,7 @@
(let [res (th/run-task! :storage-gc-deleted {})]
(t/is (= 1 (:deleted res))))
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object;"])]
(let [res (th/db-exec-one! ["select count(*) from storage_object;"])]
(t/is (= 2 (:count res))))))
(t/deftest test-touched-gc-task-1
@ -156,29 +156,33 @@
(t/is (= (:media-id result-1) (:media-id result-2)))
;; now we proceed to manually delete one file-media-object
(db/exec-one! th/*pool* ["delete from file_media_object where id = ?" (:id result-1)])
(th/db-update! :file-media-object
{:deleted-at (dt/now)}
{:id (:id result-1)})
;; run the objects gc task for permanent deletion
(let [res (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
;; check that we still have all the storage objects
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object"])]
(let [res (th/db-exec-one! ["select count(*) from storage_object"])]
(t/is (= 2 (:count res))))
;; now check if the storage objects are touched
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where touched_at is not null"])]
(let [res (th/db-exec-one! ["select count(*) from storage_object where touched_at is not null"])]
(t/is (= 2 (:count res))))
;; run the touched gc task
(let [task (:app.storage/gc-touched-task th/*system*)
res (task {})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
;; now check that there are no touched objects
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where touched_at is not null"])]
(let [res (th/db-exec-one! ["select count(*) from storage_object where touched_at is not null"])]
(t/is (= 0 (:count res))))
;; now check that all objects are marked to be deleted
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where deleted_at is not null"])]
(let [res (th/db-exec-one! ["select count(*) from storage_object where deleted_at is not null"])]
(t/is (= 0 (:count res)))))))
@ -231,31 +235,35 @@
(t/is (nil? (:error out2)))
;; run the touched gc task
(let [task (:app.storage/gc-touched-task th/*system*)
res (task {})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 5 (:freeze res)))
(t/is (= 0 (:delete res)))
(let [result-1 (:result out1)
result-2 (:result out2)]
;; now we proceed to manually delete one team-font-variant
(db/exec-one! th/*pool* ["delete from team_font_variant where id = ?" (:id result-2)])
(th/db-update! :team-font-variant
{:deleted-at (dt/now)}
{:id (:id result-2)})
;; run the objects gc task for permanent deletion
(let [res (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 1 (:processed res))))
;; revert touched state to all storage objects
(db/exec-one! th/*pool* ["update storage_object set touched_at=now()"])
(th/db-exec-one! ["update storage_object set touched_at=now()"])
;; Run the task again
(let [res (task {})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 2 (:freeze res)))
(t/is (= 3 (:delete res))))
;; now check that there are no touched objects
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where touched_at is not null"])]
(let [res (th/db-exec-one! ["select count(*) from storage_object where touched_at is not null"])]
(t/is (= 0 (:count res))))
;; now check that all objects are marked to be deleted
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where deleted_at is not null"])]
(let [res (th/db-exec-one! ["select count(*) from storage_object where deleted_at is not null"])]
(t/is (= 3 (:count res))))))))
(t/deftest test-touched-gc-task-3
@ -289,28 +297,28 @@
result-2 (:result out2)]
;; now we proceed to manually mark all storage objects touched
(db/exec-one! th/*pool* ["update storage_object set touched_at=now()"])
(th/db-exec! ["update storage_object set touched_at=now()"])
;; run the touched gc task
(let [task (:app.storage/gc-touched-task th/*system*)
res (task {})]
(let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
;; check that we have all object in the db
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where deleted_at is null"])]
(t/is (= 2 (:count res)))))
(let [rows (th/db-exec! ["select * from storage_object"])]
(t/is (= 2 (count rows)))))
;; now we proceed to manually delete all file_media_object
(db/exec-one! th/*pool* ["delete from file_media_object"])
(th/db-exec! ["update file_media_object set deleted_at = now()"])
(let [res (th/run-task! "objects-gc" {:min-age 0})]
(t/is (= 2 (:processed res))))
;; run the touched gc task
(let [task (:app.storage/gc-touched-task th/*system*)
res (task {})]
(let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))
;; check that we have all no objects
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where deleted_at is null"])]
(t/is (= 0 (:count res))))))
(let [rows (th/db-exec! ["select * from storage_object where deleted_at is null"])]
(t/is (= 0 (count rows))))))