3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-08-03 15:48:23 +02:00
Code Issues Projects Releases Packages Wiki Activity

♻️ Refactor backend to be more async friendly

Browse source
This commit is contained in:
Andrey Antukh 2022-02-28 17:15:58 +01:00 • committed by Alonso Torres
parent 087d896569
commit 9e4a50fb15
49 changed files with 1503 additions and 1378 deletions
Download patch file Download diff file Expand all files Collapse all files

33
backend/test/app/services_files_test.clj
View file

@ -174,12 +174,18 @@
:type :image
:metadata {:id (:id fmo1)}}}]})]
;; Check that reference storage objets on filemediaobjects
;; are the same because of deduplication feature.
(t/is (= (:media-id fmo1) (:media-id fmo2)))
(t/is (= (:thumbnail-id fmo1) (:thumbnail-id fmo2)))
;; If we launch gc-touched-task, we should have 4 items to freeze.
;; If we launch gc-touched-task, we should have 2 items to
;; freeze because of the deduplication (we have uploaded 2 times
;; 2 two same files).
(let [task (:app.storage/gc-touched-task th/*system*)
res (task {})]
(t/is (= 4 (:freeze res)))
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
;; run the task immediately
@ -205,27 +211,26 @@
(t/is (= 1 (count rows))))
;; The underlying storage objects are still available.
(t/is (some? (sto/get-object storage (:media-id fmo2))))
(t/is (some? (sto/get-object storage (:thumbnail-id fmo2))))
(t/is (some? (sto/get-object storage (:media-id fmo1))))
(t/is (some? (sto/get-object storage (:thumbnail-id fmo1))))
(t/is (some? @(sto/get-object storage (:media-id fmo2))))
(t/is (some? @(sto/get-object storage (:thumbnail-id fmo2))))
(t/is (some? @(sto/get-object storage (:media-id fmo1))))
(t/is (some? @(sto/get-object storage (:thumbnail-id fmo1))))
;; now, we have deleted the unused file-media-object, if we
;; execute the touched-gc task, we should see that two of them
;; are marked to be deleted.
(let [task (:app.storage/gc-touched-task th/*system*)
res (task {})]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
;; Finally, check that some of the objects that are marked as
;; deleted we are unable to retrieve them using standard storage
;; public api.
(t/is (nil? (sto/get-object storage (:media-id fmo2))))
(t/is (nil? (sto/get-object storage (:thumbnail-id fmo2))))
(t/is (some? (sto/get-object storage (:media-id fmo1))))
(t/is (some? (sto/get-object storage (:thumbnail-id fmo1))))
(t/is (some? @(sto/get-object storage (:media-id fmo2))))
(t/is (some? @(sto/get-object storage (:thumbnail-id fmo2))))
(t/is (some? @(sto/get-object storage (:media-id fmo1))))
(t/is (some? @(sto/get-object storage (:thumbnail-id fmo1))))
)))

55
backend/test/app/services_management_test.clj
View file

@ -23,9 +23,9 @@
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
sobject (sto/put-object storage {:content (sto/content "content")
:content-type "text/plain"
:other "data"})
sobject @(sto/put-object! storage {::sto/content (sto/content "content")
:content-type "text/plain"
:other "data"})
profile (th/create-profile* 1 {:is-active true})
project (th/create-project* 1 {:team-id (:default-team-id profile)
:profile-id (:id profile)})
@ -92,15 +92,17 @@
))))
(t/deftest duplicate-file-with-deleted-rels
(t/deftest duplicate-file-with-deleted-relations
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
sobject (sto/put-object storage {:content (sto/content "content")
:content-type "text/plain"
:other "data"})
sobject @(sto/put-object! storage {::sto/content (sto/content "content")
:content-type "text/plain"
:other "data"})
profile (th/create-profile* 1 {:is-active true})
project (th/create-project* 1 {:team-id (:default-team-id profile)
:profile-id (:id profile)})
file1 (th/create-file* 1 {:profile-id (:id profile)
:project-id (:id project)})
file2 (th/create-file* 2 {:profile-id (:id profile)
@ -112,16 +114,10 @@
mobj (th/create-file-media-object* {:file-id (:id file1)
:is-local false
:media-id (:id sobject)})
:media-id (:id sobject)})]
_ (th/mark-file-deleted* {:id (:id file2)})
_ (sto/del-object storage (:id sobject))]
(th/update-file*
{:file-id (:id file1)
:profile-id (:id profile)
:changes [{:type :add-media
:object (select-keys mobj [:id :width :height :mtype :name])}]})
(th/mark-file-deleted* {:id (:id file2)})
@(sto/del-object! storage sobject)
(let [data {::th/type :duplicate-file
:profile-id (:id profile)
@ -140,7 +136,7 @@
(t/is (= "file 1 (copy)" (:name result)))
(t/is (not= (:id file1) (:id result)))
;; Check that the deleted library is not duplicated
;; Check that there are no relation to a deleted library
(let [[item :as rows] (db/query th/*pool* :file-library-rel {:file-id (:id result)})]
(t/is (= 0 (count rows))))
@ -158,9 +154,10 @@
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
sobject (sto/put-object storage {:content (sto/content "content")
:content-type "text/plain"
:other "data"})
sobject @(sto/put-object! storage {::sto/content (sto/content "content")
:content-type "text/plain"
:other "data"})
profile (th/create-profile* 1 {:is-active true})
project (th/create-project* 1 {:team-id (:default-team-id profile)
:profile-id (:id profile)})
@ -176,6 +173,7 @@
:is-local false
:media-id (:id sobject)})]
(th/update-file*
{:file-id (:id file1)
:profile-id (:id profile)
@ -229,9 +227,9 @@
(t/deftest duplicate-project-with-deleted-files
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
sobject (sto/put-object storage {:content (sto/content "content")
:content-type "text/plain"
:other "data"})
sobject @(sto/put-object! storage {::sto/content (sto/content "content")
:content-type "text/plain"
:other "data"})
profile (th/create-profile* 1 {:is-active true})
project (th/create-project* 1 {:team-id (:default-team-id profile)
:profile-id (:id profile)})
@ -247,12 +245,6 @@
:is-local false
:media-id (:id sobject)})]
(th/update-file*
{:file-id (:id file1)
:profile-id (:id profile)
:changes [{:type :add-media
:object (select-keys mobj [:id :width :height :mtype :name])}]})
(th/mark-file-deleted* {:id (:id file1)})
(let [data {::th/type :duplicate-project
@ -432,7 +424,7 @@
;; project1 now should have 2 file
(let [[item1 item2 :as rows] (db/query th/*pool* :file {:project-id (:id project1)}
{:order-by [:created-at]})]
{:order-by [:created-at]})]
;; (clojure.pprint/pprint rows)
(t/is (= 2 (count rows)))
(t/is (= (:id item1) (:id file2))))
@ -610,6 +602,3 @@
(t/is (= (:library-file-id item1) (:id file2))))
)))

8
backend/test/app/services_media_test.clj
View file

@ -41,8 +41,8 @@
(t/is (uuid? media-id))
(t/is (uuid? thumbnail-id))
(let [storage (:app.storage/storage th/*system*)
mobj1 (sto/get-object storage media-id)
mobj2 (sto/get-object storage thumbnail-id)]
mobj1 @(sto/get-object storage media-id)
mobj2 @(sto/get-object storage thumbnail-id)]
(t/is (sto/storage-object? mobj1))
(t/is (sto/storage-object? mobj2))
(t/is (= 122785 (:size mobj1)))
@ -79,8 +79,8 @@
(t/is (uuid? media-id))
(t/is (uuid? thumbnail-id))
(let [storage (:app.storage/storage th/*system*)
mobj1 (sto/get-object storage media-id)
mobj2 (sto/get-object storage thumbnail-id)]
mobj1 @(sto/get-object storage media-id)
mobj2 @(sto/get-object storage thumbnail-id)]
(t/is (sto/storage-object? mobj1))
(t/is (sto/storage-object? mobj2))
(t/is (= 312043 (:size mobj1)))

95
backend/test/app/storage_test.clj
View file

@ -37,69 +37,74 @@
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
content (sto/content "content")
object (sto/put-object storage {:content content
:content-type "text/plain"
:other "data"})]
object @(sto/put-object! storage {::sto/content content
:content-type "text/plain"
:other "data"})]
(t/is (sto/storage-object? object))
(t/is (fs/path? (sto/get-object-path storage object)))
(t/is (fs/path? @(sto/get-object-path storage object)))
(t/is (nil? (:expired-at object)))
(t/is (= :tmp (:backend object)))
(t/is (= "data" (:other (meta object))))
(t/is (= "text/plain" (:content-type (meta object))))
(t/is (= "content" (slurp (sto/get-object-data storage object))))
(t/is (= "content" (slurp (sto/get-object-path storage object))))
(t/is (= "content" (slurp @(sto/get-object-data storage object))))
(t/is (= "content" (slurp @(sto/get-object-path storage object))))
))
(t/deftest put-and-retrieve-expired-object
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
content (sto/content "content")
object (sto/put-object storage {:content content
:content-type "text/plain"
:expired-at (dt/in-future {:seconds 1})})]
object @(sto/put-object! storage {::sto/content content
::sto/expired-at (dt/in-future {:seconds 1})
:content-type "text/plain"
})]
(t/is (sto/storage-object? object))
(t/is (dt/instant? (:expired-at object)))
(t/is (dt/is-after? (:expired-at object) (dt/now)))
(t/is (= object (sto/get-object storage (:id object))))
(t/is (= object @(sto/get-object storage (:id object))))
(th/sleep 1000)
(t/is (nil? (sto/get-object storage (:id object))))
(t/is (nil? (sto/get-object-data storage object)))
(t/is (nil? (sto/get-object-url storage object)))
(t/is (nil? (sto/get-object-path storage object)))
(t/is (nil? @(sto/get-object storage (:id object))))
(t/is (nil? @(sto/get-object-data storage object)))
(t/is (nil? @(sto/get-object-url storage object)))
(t/is (nil? @(sto/get-object-path storage object)))
))
(t/deftest put-and-delete-object
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
content (sto/content "content")
object (sto/put-object storage {:content content
:content-type "text/plain"
:expired-at (dt/in-future {:seconds 1})})]
object @(sto/put-object! storage {::sto/content content
:content-type "text/plain"
:expired-at (dt/in-future {:seconds 1})})]
(t/is (sto/storage-object? object))
(t/is (true? (sto/del-object storage object)))
(t/is (true? @(sto/del-object! storage object)))
;; retrieving the same object should be not nil because the
;; deletion is not immediate
(t/is (some? (sto/get-object-data storage object)))
(t/is (some? (sto/get-object-url storage object)))
(t/is (some? (sto/get-object-path storage object)))
(t/is (some? @(sto/get-object-data storage object)))
(t/is (some? @(sto/get-object-url storage object)))
(t/is (some? @(sto/get-object-path storage object)))
;; But you can't retrieve the object again because in database is
;; marked as deleted/expired.
(t/is (nil? (sto/get-object storage (:id object))))
(t/is (nil? @(sto/get-object storage (:id object))))
))
(t/deftest test-deleted-gc-task
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
content (sto/content "content")
object1 (sto/put-object storage {:content content
:content-type "text/plain"
:expired-at (dt/now)})
object2 (sto/put-object storage {:content content
:content-type "text/plain"
:expired-at (dt/in-past {:hours 2})})]
content1 (sto/content "content1")
content2 (sto/content "content2")
object1 @(sto/put-object! storage {::sto/content content1
::sto/expired-at (dt/now)
:content-type "text/plain"
})
object2 @(sto/put-object! storage {::sto/content content2
::sto/expired-at (dt/in-past {:hours 2})
:content-type "text/plain"
})]
(th/sleep 200)
(let [task (:app.storage/gc-deleted-task th/*system*)
@ -147,22 +152,24 @@
(t/is (uuid? (:media-id result-1)))
(t/is (uuid? (:media-id result-2)))
(t/is (= (:media-id result-1) (:media-id result-2)))
;; now we proceed to manually delete one file-media-object
(db/exec-one! th/*pool* ["delete from file_media_object where id = ?" (:id result-1)])
;; check that we still have all the storage objects
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object"])]
(t/is (= 4 (:count res))))
(t/is (= 2 (:count res))))
;; now check if the storage objects are touched
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where touched_at is not null"])]
(t/is (= 4 (:count res))))
(t/is (= 2 (:count res))))
;; run the touched gc task
(let [task (:app.storage/gc-touched-task th/*system*)
res (task {})]
(t/is (= 2 (:freeze res)))
(t/is (= 2 (:delete res))))
(t/is (= 0 (:delete res))))
;; now check that there are no touched objects
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where touched_at is not null"])]
@ -170,8 +177,8 @@
;; now check that all objects are marked to be deleted
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where deleted_at is not null"])]
(t/is (= 2 (:count res))))
)))
(t/is (= 0 (:count res))))
)))
(t/deftest test-touched-gc-task-2
@ -249,7 +256,7 @@
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where deleted_at is not null"])]
(t/is (= 4 (:count res))))))))
(t/deftest test-touched-gc-task-without-delete
(t/deftest test-touched-gc-task-3
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
prof (th/create-profile* 1)
@ -285,9 +292,23 @@
;; run the touched gc task
(let [task (:app.storage/gc-touched-task th/*system*)
res (task {})]
(t/is (= 4 (:freeze res)))
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
;; check that we have all object in the db
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where deleted_at is null"])]
(t/is (= 4 (:count res)))))))
(t/is (= 2 (:count res)))))
;; now we proceed to manually delete all team_font_variant
(db/exec-one! th/*pool* ["delete from file_media_object"])
;; run the touched gc task
(let [task (:app.storage/gc-touched-task th/*system*)
res (task {})]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))
;; check that we have all no objects
(let [res (db/exec-one! th/*pool* ["select count(*) from storage_object where deleted_at is null"])]
(t/is (= 0 (:count res))))))

2
backend/test/app/tasks_telemetry_test.clj
View file

@ -27,7 +27,7 @@
(task-fn nil)
(t/is (:called? @mock))
(let [[data] (-> @mock :call-args)]
(let [[_ data] (-> @mock :call-args)]
(t/is (contains? data :total-fonts))
(t/is (contains? data :total-users))
(t/is (contains? data :total-projects))

2
backend/test/app/test_helpers.clj
View file

@ -61,7 +61,7 @@
:app.http.oauth/gitlab
:app.http.oauth/github
:app.http.oauth/all
:app.worker/scheduler
:app.worker/cron
:app.worker/worker)
(d/deep-merge
{:app.tasks.file-media-gc/handler {:max-age (dt/duration 300)}}))