✨ Add safety mechanism for direct object deletion

The main objective is prevent deletion of objects that can leave
unreachable orphan objects which we are unable to correctly track.

Additionally, this commit includes:

1. Properly implement safe cascade deletion of all participating
   tables on soft deletion in the objects-gc task;

2. Make the file thumbnail related tables also participate in the
   touch/refcount mechanism applyign to the same safety checks;

3. Add helper for db query lazy iteration using PostgreSQL support
   for server side cursors;

4. Fix efficiency issues on gc related task using server side
   cursors instead of custom chunked iteration for processing data.

   The problem resided when a large chunk of rows that has identical
   value on the deleted_at column and the chunk size is small (the
   default); when the custom chunked iteration only reads a first N
   items and skip the rest of the set to the next run.

   This has caused many objects to remain pending to be eliminated,
   taking up space for longer than expected. The server side cursor
   based iteration does not has this problem and iterates correctly
   over all objects.

5. Fix refcount issues on font variant deletion RPC methods

backend/src/app/migrations/sql/0107-add-deletion-protection-trigger-function.sql

@@ -0,0 +1,8 @@
+CREATE OR REPLACE FUNCTION raise_deletion_protection()
+  RETURNS TRIGGER AS $$
+  BEGIN
+    RAISE EXCEPTION 'unable to proceed to delete row on "%"', TG_TABLE_NAME
+          USING HINT = 'disable deletion protection with "SET rules.deletion_protection TO off"';
+    RETURN NULL;
+  END;
+$$ LANGUAGE plpgsql;

backend/src/app/migrations/sql/0108-mod-file-thumbnail-table.sql

@@ -0,0 +1,25 @@
+--- Add missing index for deleted_at column, we include all related
+--- columns because we expect the index to be small and expect use
+--- index-only scans.
+CREATE INDEX IF NOT EXISTS file_thumbnail__deleted_at__idx
+    ON file_thumbnail (deleted_at, file_id, revn, media_id)
+ WHERE deleted_at IS NOT NULL;
+
+--- Add missing for media_id column, used mainly for refs checking
+CREATE INDEX IF NOT EXISTS file_thumbnail__media_id__idx ON file_thumbnail (media_id);
+
+--- Remove CASCADE from media_id and file_id foreign constraint
+ALTER TABLE file_thumbnail
+ DROP CONSTRAINT file_thumbnail_file_id_fkey,
+  ADD FOREIGN KEY (file_id) REFERENCES file(id) DEFERRABLE;
+
+ALTER TABLE file_thumbnail
+ DROP CONSTRAINT file_thumbnail_media_id_fkey,
+  ADD FOREIGN KEY (media_id) REFERENCES storage_object(id) DEFERRABLE;
+
+--- Add deletion protection
+CREATE OR REPLACE TRIGGER deletion_protection__tgr
+BEFORE DELETE ON file_thumbnail FOR EACH STATEMENT
+  WHEN ((current_setting('rules.deletion_protection', true) IN ('on', '')) OR
+        (current_setting('rules.deletion_protection', true) IS NULL))
+  EXECUTE PROCEDURE raise_deletion_protection();

backend/src/app/migrations/sql/0109-mod-file-tagged-object-thumbnail-table.sql

@@ -0,0 +1,26 @@
+ALTER TABLE file_tagged_object_thumbnail
+  ADD COLUMN updated_at timestamptz NULL,
+  ADD COLUMN deleted_at timestamptz NULL;
+
+--- Add index for deleted_at column, we include all related columns
+--- because we expect the index to be small and expect use index-only
+--- scans.
+CREATE INDEX IF NOT EXISTS file_tagged_object_thumbnail__deleted_at__idx
+    ON file_tagged_object_thumbnail (deleted_at, file_id, object_id, media_id)
+ WHERE deleted_at IS NOT NULL;
+
+--- Remove CASCADE from media_id and file_id foreign constraint
+ALTER TABLE file_tagged_object_thumbnail
+ DROP CONSTRAINT file_tagged_object_thumbnail_media_id_fkey,
+  ADD FOREIGN KEY (media_id) REFERENCES storage_object(id) DEFERRABLE;
+
+ALTER TABLE file_tagged_object_thumbnail
+ DROP CONSTRAINT file_tagged_object_thumbnail_file_id_fkey,
+  ADD FOREIGN KEY (file_id) REFERENCES file(id) DEFERRABLE;
+
+--- Add deletion protection
+CREATE OR REPLACE TRIGGER deletion_protection__tgr
+BEFORE DELETE ON file_tagged_object_thumbnail FOR EACH STATEMENT
+  WHEN ((current_setting('rules.deletion_protection', true) IN ('on', '')) OR
+        (current_setting('rules.deletion_protection', true) IS NULL))
+  EXECUTE PROCEDURE raise_deletion_protection();

backend/src/app/migrations/sql/0110-mod-file-media-object-table.sql

@@ -0,0 +1,27 @@
+--- Fix legacy naming
+ALTER INDEX media_object_pkey RENAME TO file_media_object_pkey;
+ALTER INDEX media_object__file_id__idx RENAME TO file_media_object__file_id__idx;
+
+--- Create index for the deleted_at column
+CREATE INDEX IF NOT EXISTS file_media_object__deleted_at__idx
+    ON file_media_object (deleted_at, id, media_id)
+ WHERE deleted_at IS NOT NULL;
+
+--- Drop now unnecesary trigger because this will be handled by the
+--- application code
+DROP TRIGGER file_media_object__on_delete__tgr ON file_media_object;
+DROP FUNCTION on_delete_file_media_object ( ) CASCADE;
+DROP TRIGGER file_media_object__on_insert__tgr ON file_media_object;
+DROP FUNCTION on_media_object_insert () CASCADE;
+
+--- Remove CASCADE from file FOREIGN KEY
+ALTER TABLE file_media_object
+ DROP CONSTRAINT file_media_object_file_id_fkey,
+  ADD FOREIGN KEY (file_id) REFERENCES file(id) DEFERRABLE;
+
+--- Add deletion protection
+CREATE OR REPLACE TRIGGER deletion_protection__tgr
+BEFORE DELETE ON file_media_object FOR EACH STATEMENT
+  WHEN ((current_setting('rules.deletion_protection', true) IN ('on', '')) OR
+        (current_setting('rules.deletion_protection', true) IS NULL))
+  EXECUTE PROCEDURE raise_deletion_protection();

backend/src/app/migrations/sql/0111-mod-file-data-fragment-table.sql

@@ -0,0 +1,9 @@
+ALTER TABLE file_data_fragment
+  ADD COLUMN deleted_at timestamptz NULL;
+
+--- Add index for deleted_at column, we include all related columns
+--- because we expect the index to be small and expect use index-only
+--- scans.
+CREATE INDEX IF NOT EXISTS file_data_fragment__deleted_at__idx
+    ON file_data_fragment (deleted_at, file_id, id)
+ WHERE deleted_at IS NOT NULL;

backend/src/app/migrations/sql/0112-mod-profile-table.sql

@@ -0,0 +1,15 @@
+ALTER TABLE profile
+ DROP CONSTRAINT profile_photo_id_fkey,
+  ADD FOREIGN KEY (photo_id) REFERENCES storage_object(id) DEFERRABLE,
+ DROP CONSTRAINT profile_default_project_id_fkey,
+  ADD FOREIGN KEY (default_project_id) REFERENCES project(id) DEFERRABLE,
+ DROP CONSTRAINT profile_default_team_id_fkey,
+  ADD FOREIGN KEY (default_team_id) REFERENCES team(id) DEFERRABLE;
+
+--- Add deletion protection
+CREATE OR REPLACE TRIGGER deletion_protection__tgr
+BEFORE DELETE ON profile FOR EACH STATEMENT
+  WHEN ((current_setting('rules.deletion_protection', true) IN ('on', '')) OR
+        (current_setting('rules.deletion_protection', true) IS NULL))
+  EXECUTE PROCEDURE raise_deletion_protection();
+

backend/src/app/migrations/sql/0113-mod-team-font-variant-table.sql

@@ -0,0 +1,20 @@
+--- Remove ON DELETE SET NULL from foreign constraint on
+--- storage_object table
+ALTER TABLE team_font_variant
+ DROP CONSTRAINT team_font_variant_otf_file_id_fkey,
+  ADD FOREIGN KEY (otf_file_id) REFERENCES storage_object(id) DEFERRABLE,
+ DROP CONSTRAINT team_font_variant_ttf_file_id_fkey,
+  ADD FOREIGN KEY (ttf_file_id) REFERENCES storage_object(id) DEFERRABLE,
+ DROP CONSTRAINT team_font_variant_woff1_file_id_fkey,
+  ADD FOREIGN KEY (woff1_file_id) REFERENCES storage_object(id) DEFERRABLE,
+ DROP CONSTRAINT team_font_variant_woff2_file_id_fkey,
+  ADD FOREIGN KEY (woff2_file_id) REFERENCES storage_object(id) DEFERRABLE,
+ DROP CONSTRAINT team_font_variant_team_id_fkey,
+  ADD FOREIGN KEY (team_id) REFERENCES team(id) DEFERRABLE;
+
+--- Add deletion protection
+CREATE OR REPLACE TRIGGER deletion_protection__tgr
+BEFORE DELETE ON team_font_variant FOR EACH STATEMENT
+  WHEN ((current_setting('rules.deletion_protection', true) IN ('on', '')) OR
+        (current_setting('rules.deletion_protection', true) IS NULL))
+  EXECUTE PROCEDURE raise_deletion_protection();

backend/src/app/migrations/sql/0114-mod-team-table.sql

@@ -0,0 +1,10 @@
+--- Add deletion protection
+CREATE OR REPLACE TRIGGER deletion_protection__tgr
+BEFORE DELETE ON team FOR EACH STATEMENT
+  WHEN ((current_setting('rules.deletion_protection', true) IN ('on', '')) OR
+        (current_setting('rules.deletion_protection', true) IS NULL))
+  EXECUTE PROCEDURE raise_deletion_protection();
+
+ALTER TABLE team
+ DROP CONSTRAINT team_photo_id_fkey,
+  ADD FOREIGN KEY (photo_id) REFERENCES storage_object(id) DEFERRABLE;

backend/src/app/migrations/sql/0115-mod-project-table.sql

@@ -0,0 +1,3 @@
+ALTER TABLE project
+ DROP CONSTRAINT project_team_id_fkey,
+  ADD FOREIGN KEY (team_id) REFERENCES team(id) DEFERRABLE;

backend/src/app/migrations/sql/0116-mod-file-table.sql

@@ -0,0 +1,3 @@
+ALTER TABLE file
+ DROP CONSTRAINT file_project_id_fkey,
+  ADD FOREIGN KEY (project_id) REFERENCES project(id) DEFERRABLE;

backend/src/app/migrations/sql/0117-mod-file-object-thumbnail-table.sql

@@ -0,0 +1,12 @@
+ALTER TABLE file_object_thumbnail
+ DROP CONSTRAINT file_object_thumbnail_file_id_fkey,
+  ADD FOREIGN KEY (file_id) REFERENCES file(id) DEFERRABLE,
+ DROP CONSTRAINT file_object_thumbnail_media_id_fkey,
+  ADD FOREIGN KEY (media_id) REFERENCES storage_object(id) DEFERRABLE;
+
+--- Mark all related storage_object row as touched
+-- UPDATE storage_object SET touched_at = now()
+--  WHERE id IN (SELECT DISTINCT media_id
+--                 FROM file_object_thumbnail
+--                WHERE media_id IS NOT NULL)
+--    AND touched_at IS NULL;