Add safety mechanism for direct object deletion

The main objective is prevent deletion of objects that can leave
unreachable orphan objects which we are unable to correctly track.

Additionally, this commit includes:

1. Properly implement safe cascade deletion of all participating
   tables on soft deletion in the objects-gc task;

2. Make the file thumbnail related tables also participate in the
   touch/refcount mechanism applyign to the same safety checks;

3. Add helper for db query lazy iteration using PostgreSQL support
   for server side cursors;

4. Fix efficiency issues on gc related task using server side
   cursors instead of custom chunked iteration for processing data.

   The problem resided when a large chunk of rows that has identical
   value on the deleted_at column and the chunk size is small (the
   default); when the custom chunked iteration only reads a first N
   items and skip the rest of the set to the next run.

   This has caused many objects to remain pending to be eliminated,
   taking up space for longer than expected. The server side cursor
   based iteration does not has this problem and iterates correctly
   over all objects.

5. Fix refcount issues on font variant deletion RPC methods
This commit is contained in:
Andrey Antukh 2023-12-29 15:21:14 +01:00
parent e6fb96c4c2
commit addb392ecc
37 changed files with 1918 additions and 1026 deletions

View file

@ -125,7 +125,7 @@
;; profile is not deleted because it does not meet all
;; conditions to be deleted.
(let [result (th/run-task! :objects-gc {:min-age (dt/duration 0)})]
(let [result (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 0 (:processed result))))
;; Request profile to be deleted
@ -144,8 +144,16 @@
(t/is (= 1 (count (:result out)))))
;; execute permanent deletion task
(let [result (th/run-task! :objects-gc {:min-age (dt/duration "-1m")})]
(t/is (= 2 (:processed result))))
(let [result (th/run-task! :objects-gc {:min-age 0})]
(t/is (= 1 (:processed result))))
(let [row (th/db-get :team
{:id (:default-team-id prof)}
{::db/remove-deleted? false})]
(t/is (nil? (:deleted-at row))))
(let [result (th/run-task! :orphan-teams-gc {:min-age 0})]
(t/is (= 1 (:processed result))))
(let [row (th/db-get :team
{:id (:default-team-id prof)}
@ -158,67 +166,9 @@
out (th/command! params)]
;; (th/print-result! out)
(let [result (:result out)]
(t/is (= uuid/zero (:id result)))))))
(t/is (= uuid/zero (:id result)))))
(t/deftest profile-immediate-deletion
(let [prof1 (th/create-profile* 1)
prof2 (th/create-profile* 2)
file (th/create-file* 1 {:profile-id (:id prof1)
:project-id (:default-project-id prof1)
:is-shared false})
team (th/create-team* 1 {:profile-id (:id prof1)})
_ (th/create-team-role* {:team-id (:id team)
:profile-id (:id prof2)
:role :admin})]
;; profile is not deleted because it does not meet all
;; conditions to be deleted.
(let [result (th/run-task! :objects-gc {:min-age (dt/duration 0)})]
(t/is (= 0 (:orphans result)))
(t/is (= 0 (:processed result))))
;; just delete the profile
(th/db-delete! :profile {:id (:id prof1)})
;; query files after profile deletion, expecting not found
(let [params {::th/type :get-project-files
::rpc/profile-id (:id prof1)
:project-id (:default-project-id prof1)}
out (th/command! params)]
;; (th/print-result! out)
(t/is (not (th/success? out)))
(let [edata (-> out :error ex-data)]
(t/is (= :not-found (:type edata)))))
;; the files and projects still exists on the database
(let [files (th/db-query :file {:project-id (:default-project-id prof1)})
projects (th/db-query :project {:team-id (:default-team-id prof1)})]
(t/is (= 1 (count files)))
(t/is (= 1 (count projects))))
;; execute the gc task
(let [result (th/run-task! :objects-gc {:min-age (dt/duration "-1m")})]
(t/is (= 1 (:processed result)))
(t/is (= 1 (:orphans result))))
;; Check the deletion flag on the default profile team
(let [row (th/db-get :team
{:id (:default-team-id prof1)}
{::db/remove-deleted? false})]
(t/is (dt/instant? (:deleted-at row))))
;; Check the deletion flag on the shared team
(let [row (th/db-get :team
{:id (:id team)}
{::db/remove-deleted? false})]
(t/is (nil? (:deleted-at row))))
;; Check the roles on the shared team
(let [rows (th/db-query :team-profile-rel {:team-id (:id team)})]
(t/is (= 1 (count rows)))
(t/is (= (:id prof2) (get-in rows [0 :profile-id])))
(t/is (= false (get-in rows [0 :is-owner]))))))
))
(t/deftest registration-domain-whitelist
(let [whitelist #{"gmail.com" "hey.com" "ya.ru"}]