diff --git a/backend/src/app/features/logical_deletion.clj b/backend/src/app/features/logical_deletion.clj new file mode 100644 index 000000000..8a06f3f30 --- /dev/null +++ b/backend/src/app/features/logical_deletion.clj @@ -0,0 +1,31 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns app.features.logical-deletion + "A code related to handle logical deletion mechanism" + (:require + [app.config :as cf] + [app.util.time :as dt])) + +(defn get-deletion-delay + "Calculate the next deleted-at for a resource (file, team, etc) in function + of team settings" + [team] + (if-let [subscription (get team :subscription)] + (cond + (and (= (:type subscription) "unlimited") + (= (:status subscription) "active")) + (dt/duration {:days 30}) + + (and (= (:type subscription) "enterprise") + (= (:status subscription) "active")) + (dt/duration {:days 90}) + + :else + (cf/get-deletion-delay)) + + (cf/get-deletion-delay))) + diff --git a/backend/src/app/rpc/commands/files.clj b/backend/src/app/rpc/commands/files.clj index bdb2fcbc5..7d418f7e3 100644 --- a/backend/src/app/rpc/commands/files.clj +++ b/backend/src/app/rpc/commands/files.clj @@ -23,6 +23,7 @@ [app.db.sql :as-alias sql] [app.features.fdata :as feat.fdata] [app.features.file-migrations :as feat.fmigr] + [app.features.logical-deletion :as ldel] [app.loggers.audit :as-alias audit] [app.loggers.webhooks :as-alias webhooks] [app.rpc :as-alias rpc] @@ -970,12 +971,13 @@ ;; --- MUTATION COMMAND: delete-file (defn- mark-file-deleted - [conn file-id] - (let [file (db/update! conn :file - {:deleted-at (dt/now)} - {:id file-id} - {::db/return-keys [:id :name :is-shared :deleted-at - :project-id :created-at :modified-at]})] + [conn team file-id] + (let [delay (ldel/get-deletion-delay team) + file (db/update! conn :file + {:deleted-at (dt/in-future delay)} + {:id file-id} + {::db/return-keys [:id :name :is-shared :deleted-at + :project-id :created-at :modified-at]})] (wrk/submit! {::db/conn conn ::wrk/task :delete-object ::wrk/params {:object :file @@ -991,7 +993,11 @@ (defn- delete-file [{:keys [::db/conn] :as cfg} {:keys [profile-id id] :as params}] (check-edition-permissions! conn profile-id id) - (let [file (mark-file-deleted conn id)] + (let [team (teams/get-team conn + :profile-id profile-id + :file-id id) + file (mark-file-deleted conn team id)] + (rph/with-meta (rph/wrap) {::audit/props {:project-id (:project-id file) :name (:name file) diff --git a/backend/src/app/rpc/commands/files_update.clj b/backend/src/app/rpc/commands/files_update.clj index 866e67b1f..3908c39cc 100644 --- a/backend/src/app/rpc/commands/files_update.clj +++ b/backend/src/app/rpc/commands/files_update.clj @@ -20,6 +20,7 @@ [app.db :as db] [app.features.fdata :as feat.fdata] [app.features.file-migrations :as feat.fmigr] + [app.features.logical-deletion :as ldel] [app.http.errors :as errors] [app.loggers.audit :as audit] [app.loggers.webhooks :as webhooks] @@ -209,7 +210,7 @@ Only intended for internal use on this module." [{:keys [::db/conn ::wrk/executor ::timestamp] :as cfg} - {:keys [profile-id file features changes session-id skip-validate] :as params}] + {:keys [profile-id file team features changes session-id skip-validate] :as params}] (let [;; Retrieve the file data file (feat.fmigr/resolve-applied-migrations cfg file) @@ -243,7 +244,7 @@ :created-at timestamp :updated-at timestamp :deleted-at (if (::snapshot-data file) - (dt/plus timestamp (cf/get-deletion-delay)) + (dt/plus timestamp (ldel/get-deletion-delay team)) (dt/plus timestamp (dt/duration {:hours 1}))) :file-id (:id file) :revn (:revn file) diff --git a/backend/src/app/rpc/commands/fonts.clj b/backend/src/app/rpc/commands/fonts.clj index 43b90305e..ed85ed4ec 100644 --- a/backend/src/app/rpc/commands/fonts.clj +++ b/backend/src/app/rpc/commands/fonts.clj @@ -12,6 +12,7 @@ [app.common.uuid :as uuid] [app.db :as db] [app.db.sql :as-alias sql] + [app.features.logical-deletion :as ldel] [app.loggers.audit :as-alias audit] [app.loggers.webhooks :as-alias webhooks] [app.media :as media] @@ -202,32 +203,40 @@ (sv/defmethod ::delete-font {::doc/added "1.18" ::webhooks/event? true - ::sm/params schema:delete-font} - [cfg {:keys [::rpc/profile-id id team-id]}] - (db/tx-run! cfg - (fn [{:keys [::db/conn] :as cfg}] - (teams/check-edition-permissions! conn profile-id team-id) - (let [fonts (db/query conn :team-font-variant - {:team-id team-id - :font-id id - :deleted-at nil} - {::sql/for-update true}) - tnow (dt/now)] + ::sm/params schema:delete-font + ::db/transaction true} + [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id team-id]}] + (let [team (teams/get-team conn + :profile-id profile-id + :team-id team-id) - (when-not (seq fonts) - (ex/raise :type :not-found - :code :object-not-found)) + fonts (db/query conn :team-font-variant + {:team-id team-id + :font-id id + :deleted-at nil} + {::sql/for-update true}) - (doseq [font fonts] - (db/update! conn :team-font-variant - {:deleted-at tnow} - {:id (:id font)})) + delay (ldel/get-deletion-delay team) + tnow (dt/in-future delay)] - (rph/with-meta (rph/wrap) - {::audit/props {:id id - :team-id team-id - :name (:font-family (peek fonts)) - :profile-id profile-id}}))))) + (teams/check-edition-permissions! (:permissions team)) + + (when-not (seq fonts) + (ex/raise :type :not-found + :code :object-not-found)) + + + (doseq [font fonts] + (db/update! conn :team-font-variant + {:deleted-at tnow} + {:id (:id font)} + {::db/return-keys false})) + + (rph/with-meta (rph/wrap) + {::audit/props {:id id + :team-id team-id + :name (:font-family (peek fonts)) + :profile-id profile-id}}))) ;; --- DELETE FONT VARIANT @@ -239,19 +248,23 @@ (sv/defmethod ::delete-font-variant {::doc/added "1.18" ::webhooks/event? true - ::sm/params schema:delete-font-variant} - [cfg {:keys [::rpc/profile-id id team-id]}] - (db/tx-run! cfg - (fn [{:keys [::db/conn] :as cfg}] - (teams/check-edition-permissions! conn profile-id team-id) - (let [variant (db/get conn :team-font-variant - {:id id :team-id team-id} - {::sql/for-update true})] + ::sm/params schema:delete-font-variant + ::db/transaction true} + [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id team-id]}] + (let [team (teams/get-team conn + :profile-id profile-id + :team-id team-id) + variant (db/get conn :team-font-variant + {:id id :team-id team-id} + {::sql/for-update true}) + delay (ldel/get-deletion-delay team)] - (db/update! conn :team-font-variant - {:deleted-at (dt/now)} - {:id (:id variant)}) + (teams/check-edition-permissions! (:permissions team)) + (db/update! conn :team-font-variant + {:deleted-at (dt/in-future delay)} + {:id (:id variant)} + {::db/return-keys false}) - (rph/with-meta (rph/wrap) - {::audit/props {:font-family (:font-family variant) - :font-id (:font-id variant)}}))))) + (rph/with-meta (rph/wrap) + {::audit/props {:font-family (:font-family variant) + :font-id (:font-id variant)}}))) diff --git a/backend/src/app/rpc/commands/profile.clj b/backend/src/app/rpc/commands/profile.clj index 28c9bf626..6449bef09 100644 --- a/backend/src/app/rpc/commands/profile.clj +++ b/backend/src/app/rpc/commands/profile.clj @@ -480,8 +480,7 @@ JOIN team AS t ON (t.id = tpr.team_id) WHERE tpr.is_owner IS TRUE AND tpr.profile_id = ? - AND (t.deleted_at IS NULL OR - t.deleted_at > now()) + AND t.deleted_at IS NULL ) SELECT tpr.team_id AS id, count(tpr.profile_id) - 1 AS participants diff --git a/backend/src/app/rpc/commands/projects.clj b/backend/src/app/rpc/commands/projects.clj index 5b46dbcf0..23cdff51a 100644 --- a/backend/src/app/rpc/commands/projects.clj +++ b/backend/src/app/rpc/commands/projects.clj @@ -11,6 +11,7 @@ [app.common.schema :as sm] [app.db :as db] [app.db.sql :as-alias sql] + [app.features.logical-deletion :as ldel] [app.loggers.audit :as-alias audit] [app.loggers.webhooks :as webhooks] [app.rpc :as-alias rpc] @@ -253,9 +254,10 @@ ;; --- MUTATION: Delete Project (defn- delete-project - [conn project-id] - (let [project (db/update! conn :project - {:deleted-at (dt/now)} + [conn team project-id] + (let [delay (ldel/get-deletion-delay team) + project (db/update! conn :project + {:deleted-at (dt/in-future delay)} {:id project-id} {::db/return-keys true})] @@ -272,7 +274,6 @@ project)) - (def ^:private schema:delete-project [:map {:title "delete-project"} [:id ::sm/uuid]]) @@ -284,7 +285,10 @@ ::db/transaction true} [{:keys [::db/conn]} {:keys [::rpc/profile-id id] :as params}] (check-edition-permissions! conn profile-id id) - (let [project (delete-project conn id)] + (let [team (teams/get-team conn + :profile-id profile-id + :project-id id) + project (delete-project conn team id)] (rph/with-meta (rph/wrap) {::audit/props {:team-id (:team-id project) :name (:name project) diff --git a/backend/src/app/rpc/commands/teams.clj b/backend/src/app/rpc/commands/teams.clj index 2cf06564c..97328174e 100644 --- a/backend/src/app/rpc/commands/teams.clj +++ b/backend/src/app/rpc/commands/teams.clj @@ -17,6 +17,7 @@ [app.db :as db] [app.db.sql :as sql] [app.email :as eml] + [app.features.logical-deletion :as ldel] [app.loggers.audit :as audit] [app.main :as-alias main] [app.media :as media] @@ -214,39 +215,43 @@ (defn get-team [conn & {:keys [profile-id team-id project-id file-id] :as params}] - (dm/assert! - "connection or pool is mandatory" - (or (db/connection? conn) - (db/pool? conn))) + (assert (uuid? profile-id) "profile-id is mandatory") + (assert (or (db/connection? conn) + (db/pool? conn)) + "connection or pool is mandatory") - (dm/assert! - "profile-id is mandatory" - (uuid? profile-id)) + (let [{:keys [default-team-id] :as profile} + (profile/get-profile conn profile-id) - (let [{:keys [default-team-id] :as profile} (profile/get-profile conn profile-id) - result (cond - (some? team-id) - (let [sql (str "WITH teams AS (" sql:get-teams-with-permissions - ") SELECT * FROM teams WHERE id=?")] - (db/exec-one! conn [sql default-team-id profile-id team-id])) + sql + (if (contains? cf/flags :subscriptions) + sql:get-teams-with-permissions-and-subscription + sql:get-teams-with-permissions) - (some? project-id) - (let [sql (str "WITH teams AS (" sql:get-teams-with-permissions ") " - "SELECT t.* FROM teams AS t " - " JOIN project AS p ON (p.team_id = t.id) " - " WHERE p.id=?")] - (db/exec-one! conn [sql default-team-id profile-id project-id])) + result + (cond + (some? team-id) + (let [sql (str "WITH teams AS (" sql ") " + "SELECT * FROM teams WHERE id=?")] + (db/exec-one! conn [sql default-team-id profile-id team-id])) - (some? file-id) - (let [sql (str "WITH teams AS (" sql:get-teams-with-permissions ") " - "SELECT t.* FROM teams AS t " - " JOIN project AS p ON (p.team_id = t.id) " - " JOIN file AS f ON (f.project_id = p.id) " - " WHERE f.id=?")] - (db/exec-one! conn [sql default-team-id profile-id file-id])) + (some? project-id) + (let [sql (str "WITH teams AS (" sql ") " + "SELECT t.* FROM teams AS t " + " JOIN project AS p ON (p.team_id = t.id) " + " WHERE p.id=?")] + (db/exec-one! conn [sql default-team-id profile-id project-id])) - :else - (throw (IllegalArgumentException. "invalid arguments")))] + (some? file-id) + (let [sql (str "WITH teams AS (" sql ") " + "SELECT t.* FROM teams AS t " + " JOIN project AS p ON (p.team_id = t.id) " + " JOIN file AS f ON (f.project_id = p.id) " + " WHERE f.id=?")] + (db/exec-one! conn [sql default-team-id profile-id file-id])) + + :else + (throw (IllegalArgumentException. "invalid arguments")))] (when-not result (ex/raise :type :not-found @@ -634,13 +639,13 @@ (defn- delete-team "Mark a team for deletion" - [conn team-id] + [conn {:keys [id] :as team}] - (let [deleted-at (dt/now) - team (db/update! conn :team - {:deleted-at deleted-at} - {:id team-id} - {::db/return-keys true})] + (let [delay (ldel/get-deletion-delay team) + team (db/update! conn :team + {:deleted-at (dt/in-future delay)} + {:id id} + {::db/return-keys true})] (when (:is-default team) (ex/raise :type :validation @@ -650,8 +655,8 @@ (wrk/submit! {::db/conn conn ::wrk/task :delete-object ::wrk/params {:object :team - :deleted-at deleted-at - :id team-id}}) + :deleted-at (:deleted-at team) + :id id}}) team)) (def ^:private schema:delete-team @@ -663,12 +668,14 @@ ::sm/params schema:delete-team ::db/transaction true} [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id] :as params}] - (let [perms (get-permissions conn profile-id id)] + (let [team (get-team conn :profile-id profile-id :team-id id) + perms (get team :permissions)] + (when-not (:is-owner perms) (ex/raise :type :validation :code :only-owner-can-delete-team)) - (delete-team conn id) + (delete-team conn team) nil)) ;; --- Mutation: Team Update Role diff --git a/backend/src/app/tasks/objects_gc.clj b/backend/src/app/tasks/objects_gc.clj index 843feb1c0..548324f98 100644 --- a/backend/src/app/tasks/objects_gc.clj +++ b/backend/src/app/tasks/objects_gc.clj @@ -9,7 +9,6 @@ of deleted or unreachable objects." (:require [app.common.logging :as l] - [app.config :as cf] [app.db :as db] [app.storage :as sto] [app.util.time :as dt] @@ -18,15 +17,15 @@ (def ^:private sql:get-profiles "SELECT id, photo_id FROM profile WHERE deleted_at IS NOT NULL - AND deleted_at < now() - ?::interval + AND deleted_at < now() + ?::interval ORDER BY deleted_at ASC LIMIT ? FOR UPDATE SKIP LOCKED") (defn- delete-profiles! - [{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}] - (->> (db/plan conn [sql:get-profiles min-age chunk-size] {:fetch-size 5}) + [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}] + (->> (db/plan conn [sql:get-profiles deletion-threshold chunk-size] {:fetch-size 5}) (reduce (fn [total {:keys [id photo-id]}] (l/trc :hint "permanently delete" :rel "profile" :id (str id)) @@ -41,15 +40,15 @@ (def ^:private sql:get-teams "SELECT deleted_at, id, photo_id FROM team WHERE deleted_at IS NOT NULL - AND deleted_at < now() - ?::interval + AND deleted_at < now() + ?::interval ORDER BY deleted_at ASC LIMIT ? FOR UPDATE SKIP LOCKED") (defn- delete-teams! - [{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}] - (->> (db/plan conn [sql:get-teams min-age chunk-size] {:fetch-size 5}) + [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}] + (->> (db/plan conn [sql:get-teams deletion-threshold chunk-size] {:fetch-size 5}) (reduce (fn [total {:keys [id photo-id deleted-at]}] (l/trc :hint "permanently delete" :rel "team" @@ -69,15 +68,15 @@ "SELECT id, team_id, deleted_at, woff1_file_id, woff2_file_id, otf_file_id, ttf_file_id FROM team_font_variant WHERE deleted_at IS NOT NULL - AND deleted_at < now() - ?::interval + AND deleted_at < now() + ?::interval ORDER BY deleted_at ASC LIMIT ? FOR UPDATE SKIP LOCKED") (defn- delete-fonts! - [{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}] - (->> (db/plan conn [sql:get-fonts min-age chunk-size] {:fetch-size 5}) + [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}] + (->> (db/plan conn [sql:get-fonts deletion-threshold chunk-size] {:fetch-size 5}) (reduce (fn [total {:keys [id team-id deleted-at] :as font}] (l/trc :hint "permanently delete" :rel "team-font-variant" @@ -101,15 +100,15 @@ "SELECT id, deleted_at, team_id FROM project WHERE deleted_at IS NOT NULL - AND deleted_at < now() - ?::interval + AND deleted_at < now() + ?::interval ORDER BY deleted_at ASC LIMIT ? FOR UPDATE SKIP LOCKED") (defn- delete-projects! - [{:keys [::db/conn ::min-age ::chunk-size] :as cfg}] - (->> (db/plan conn [sql:get-projects min-age chunk-size] {:fetch-size 5}) + [{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}] + (->> (db/plan conn [sql:get-projects deletion-threshold chunk-size] {:fetch-size 5}) (reduce (fn [total {:keys [id team-id deleted-at]}] (l/trc :hint "permanently delete" :rel "project" @@ -127,15 +126,15 @@ "SELECT id, deleted_at, project_id, data_backend, data_ref_id FROM file WHERE deleted_at IS NOT NULL - AND deleted_at < now() - ?::interval + AND deleted_at < now() + ?::interval ORDER BY deleted_at ASC LIMIT ? FOR UPDATE SKIP LOCKED") (defn- delete-files! - [{:keys [::db/conn ::sto/storage ::min-age ::chunk-size] :as cfg}] - (->> (db/plan conn [sql:get-files min-age chunk-size] {:fetch-size 5}) + [{:keys [::db/conn ::sto/storage ::deletion-threshold ::chunk-size] :as cfg}] + (->> (db/plan conn [sql:get-files deletion-threshold chunk-size] {:fetch-size 5}) (reduce (fn [total {:keys [id deleted-at project-id] :as file}] (l/trc :hint "permanently delete" :rel "file" @@ -156,15 +155,15 @@ "SELECT file_id, revn, media_id, deleted_at FROM file_thumbnail WHERE deleted_at IS NOT NULL - AND deleted_at < now() - ?::interval + AND deleted_at < now() + ?::interval ORDER BY deleted_at ASC LIMIT ? FOR UPDATE SKIP LOCKED") (defn delete-file-thumbnails! - [{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}] - (->> (db/plan conn [sql:get-file-thumbnails min-age chunk-size] {:fetch-size 5}) + [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}] + (->> (db/plan conn [sql:get-file-thumbnails deletion-threshold chunk-size] {:fetch-size 5}) (reduce (fn [total {:keys [file-id revn media-id deleted-at]}] (l/trc :hint "permanently delete" :rel "file-thumbnail" @@ -185,15 +184,15 @@ "SELECT file_id, object_id, media_id, deleted_at FROM file_tagged_object_thumbnail WHERE deleted_at IS NOT NULL - AND deleted_at < now() - ?::interval + AND deleted_at < now() + ?::interval ORDER BY deleted_at ASC LIMIT ? FOR UPDATE SKIP LOCKED") (defn delete-file-object-thumbnails! - [{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}] - (->> (db/plan conn [sql:get-file-object-thumbnails min-age chunk-size] {:fetch-size 5}) + [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}] + (->> (db/plan conn [sql:get-file-object-thumbnails deletion-threshold chunk-size] {:fetch-size 5}) (reduce (fn [total {:keys [file-id object-id media-id deleted-at]}] (l/trc :hint "permanently delete" :rel "file-tagged-object-thumbnail" @@ -214,15 +213,15 @@ "SELECT file_id, id, deleted_at, data_ref_id FROM file_data_fragment WHERE deleted_at IS NOT NULL - AND deleted_at < now() - ?::interval + AND deleted_at < now() + ?::interval ORDER BY deleted_at ASC LIMIT ? FOR UPDATE SKIP LOCKED") (defn- delete-file-data-fragments! - [{:keys [::db/conn ::sto/storage ::min-age ::chunk-size] :as cfg}] - (->> (db/plan conn [sql:get-file-data-fragments min-age chunk-size] {:fetch-size 5}) + [{:keys [::db/conn ::sto/storage ::deletion-threshold ::chunk-size] :as cfg}] + (->> (db/plan conn [sql:get-file-data-fragments deletion-threshold chunk-size] {:fetch-size 5}) (reduce (fn [total {:keys [file-id id deleted-at data-ref-id]}] (l/trc :hint "permanently delete" :rel "file-data-fragment" @@ -240,15 +239,15 @@ "SELECT id, file_id, media_id, thumbnail_id, deleted_at FROM file_media_object WHERE deleted_at IS NOT NULL - AND deleted_at < now() - ?::interval + AND deleted_at < now() + ?::interval ORDER BY deleted_at ASC LIMIT ? FOR UPDATE SKIP LOCKED") (defn- delete-file-media-objects! - [{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}] - (->> (db/plan conn [sql:get-file-media-objects min-age chunk-size] {:fetch-size 5}) + [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}] + (->> (db/plan conn [sql:get-file-media-objects deletion-threshold chunk-size] {:fetch-size 5}) (reduce (fn [total {:keys [id file-id deleted-at] :as fmo}] (l/trc :hint "permanently delete" :rel "file-media-object" @@ -269,15 +268,15 @@ "SELECT id, file_id, deleted_at, data_backend, data_ref_id FROM file_change WHERE deleted_at IS NOT NULL - AND deleted_at < now() - ?::interval + AND deleted_at < now() + ?::interval ORDER BY deleted_at ASC LIMIT ? FOR UPDATE SKIP LOCKED") (defn- delete-file-changes! - [{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}] - (->> (db/plan conn [sql:get-file-change min-age chunk-size] {:fetch-size 5}) + [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}] + (->> (db/plan conn [sql:get-file-change deletion-threshold chunk-size] {:fetch-size 5}) (reduce (fn [total {:keys [id file-id deleted-at] :as xlog}] (l/trc :hint "permanently delete" :rel "file-change" @@ -324,16 +323,13 @@ (defmethod ig/expand-key ::handler [k v] - {k (assoc v - ::min-age (cf/get-deletion-delay) - ::chunk-size 100)}) + {k (assoc v ::chunk-size 100)}) (defmethod ig/init-key ::handler [_ cfg] (fn [{:keys [props] :as task}] - (let [min-age (dt/duration (or (:min-age props) (::min-age cfg))) - cfg (assoc cfg ::min-age (db/interval min-age))] - + (let [threshold (dt/duration (get props :deletion-threshold 0)) + cfg (assoc cfg ::deletion-threshold (db/interval threshold))] (loop [procs (map deref deletion-proc-vars) total 0] (if-let [proc-fn (first procs)] diff --git a/backend/test/backend_tests/helpers.clj b/backend/test/backend_tests/helpers.clj index cc442d5a0..332a93789 100644 --- a/backend/test/backend_tests/helpers.clj +++ b/backend/test/backend_tests/helpers.clj @@ -222,7 +222,7 @@ ([params] (mark-file-deleted* *system* params)) ([conn {:keys [id] :as params}] - (#'files/mark-file-deleted conn id))) + (#'files/mark-file-deleted conn {} id))) (defn create-team* ([i params] (create-team* *system* i params)) diff --git a/backend/test/backend_tests/rpc_file_test.clj b/backend/test/backend_tests/rpc_file_test.clj index 951ee96a8..538cc82f0 100644 --- a/backend/test/backend_tests/rpc_file_test.clj +++ b/backend/test/backend_tests/rpc_file_test.clj @@ -8,10 +8,10 @@ (:require [app.common.features :as cfeat] [app.common.pprint :as pp] - [app.common.pprint :as pp] [app.common.thumbnails :as thc] [app.common.types.shape :as cts] [app.common.uuid :as uuid] + [app.config :as cf] [app.db :as db] [app.db.sql :as sql] [app.http :as http] @@ -123,8 +123,27 @@ :components-v2 true} out (th/command! data)] - ;; (th/print-result! out) + ;; (th/print-result! out) + (t/is (nil? (:error out))) + + (let [result (:result out)] + (t/is (some? (:deleted-at result))) + (t/is (= file-id (:id result))) + (t/is (= "new name" (:name result))) + (t/is (= 1 (count (get-in result [:data :pages])))) + (t/is (nil? (:users result)))))) + + (th/db-update! :file + {:deleted-at (dt/now)} + {:id file-id}) + + (t/testing "query single file after delete and wait" + (let [data {::th/type :get-file + ::rpc/profile-id (:id prof) + :id file-id + :components-v2 true} + out (th/command! data)] (let [error (:error out) error-data (ex-data error)] (t/is (th/ex-info? error)) @@ -195,7 +214,7 @@ (t/is (= 5 (count rows)))) ;; The objects-gc should remove unused fragments - (let [res (th/run-task! :objects-gc {:min-age 0})] + (let [res (th/run-task! :objects-gc {})] (t/is (= 3 (:processed res)))) ;; Check the number of fragments @@ -230,7 +249,7 @@ (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)}))) ;; The objects-gc should remove unused fragments - (let [res (th/run-task! :objects-gc {:min-age 0})] + (let [res (th/run-task! :objects-gc {})] (t/is (= 3 (:processed res)))) ;; Check the number of fragments; @@ -254,7 +273,7 @@ (t/is (= 4 (count rows))) (t/is (= 2 (count (remove (comp some? :deleted-at) rows))))) - (let [res (th/run-task! :objects-gc {:min-age 0})] + (let [res (th/run-task! :objects-gc {})] (t/is (= 2 (:processed res)))) (let [rows (th/db-query :file-data-fragment {:file-id (:id file)})] @@ -355,7 +374,7 @@ (t/is (= 2 (count rows))) (t/is (= 1 (count (remove (comp some? :deleted-at) rows))))) - (let [res (th/run-task! :objects-gc {:min-age 0})] + (let [res (th/run-task! :objects-gc {})] (t/is (= 3 (:processed res)))) ;; check file media objects @@ -386,7 +405,7 @@ ;; This only clears fragments, the file media objects still referenced because ;; snapshots are preserved - (let [res (th/run-task! :objects-gc {:min-age 0})] + (let [res (th/run-task! :objects-gc {})] (t/is (= 2 (:processed res)))) ;; Mark all snapshots to be a non-snapshot file change @@ -395,7 +414,7 @@ ;; Rerun the file-gc and objects-gc (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)}))) - (let [res (th/run-task! :objects-gc {:min-age 0})] + (let [res (th/run-task! :objects-gc {})] (t/is (= 2 (:processed res)))) ;; Now that file-gc have deleted the file-media-object usage, @@ -508,7 +527,7 @@ ;; run the task again (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)}))) - (let [res (th/run-task! :objects-gc {:min-age 0})] + (let [res (th/run-task! :objects-gc {})] (t/is (= 2 (:processed res)))) (let [rows (th/db-query :file-data-fragment {:file-id (:id file) @@ -550,7 +569,7 @@ ;; This only removes unused fragments, file media are still ;; referenced on snapshots. - (let [res (th/run-task! :objects-gc {:min-age 0})] + (let [res (th/run-task! :objects-gc {})] (t/is (= 2 (:processed res)))) ;; Mark all snapshots to be a non-snapshot file change @@ -560,7 +579,7 @@ ;; Rerun file-gc and objects-gc task for the same file once all snapshots are ;; "expired/deleted" (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)}))) - (let [res (th/run-task! :objects-gc {:min-age 0})] + (let [res (th/run-task! :objects-gc {})] (t/is (= 6 (:processed res)))) (let [rows (th/db-query :file-data-fragment {:file-id (:id file) @@ -712,7 +731,7 @@ ;; Now that file-gc have marked for deletion the object ;; thumbnail lets execute the objects-gc task which remove ;; the rows and mark as touched the storage object rows - (let [res (th/run-task! :objects-gc {:min-age 0})] + (let [res (th/run-task! :objects-gc {})] (t/is (= 5 (:processed res)))) ;; Now that objects-gc have deleted the object thumbnail lets @@ -741,7 +760,7 @@ (t/is (= 1 (count rows))) (t/is (= 0 (count (remove (comp some? :deleted-at) rows))))) - (let [res (th/run-task! :objects-gc {:min-age 0})] + (let [res (th/run-task! :objects-gc {})] ;; (pp/pprint res) (t/is (= 3 (:processed res)))) @@ -876,7 +895,7 @@ :profile-id (:id profile1)})] ;; file is not deleted because it does not meet all ;; conditions to be deleted. - (let [result (th/run-task! :objects-gc {:min-age 0})] + (let [result (th/run-task! :objects-gc {})] (t/is (= 0 (:processed result)))) ;; query the list of files @@ -907,7 +926,7 @@ (t/is (= 0 (count result))))) ;; run permanent deletion (should be noop) - (let [result (th/run-task! :objects-gc {:min-age (dt/duration {:minutes 1})})] + (let [result (th/run-task! :objects-gc {})] (t/is (= 0 (:processed result)))) ;; query the list of file libraries of a after hard deletion @@ -921,7 +940,7 @@ (t/is (= 0 (count result))))) ;; run permanent deletion - (let [result (th/run-task! :objects-gc {:min-age 0})] + (let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})] (t/is (= 1 (:processed result)))) ;; query the list of file libraries of a after hard deletion @@ -1176,7 +1195,7 @@ (t/is (= 2 (count rows))) (t/is (= 1 (count (remove :deleted-at rows))))) - (let [res (th/run-task! :objects-gc {:min-age 0})] + (let [res (th/run-task! :objects-gc {})] (t/is (= 4 (:processed res)))) (let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})] @@ -1232,7 +1251,7 @@ (t/is (= 2 (count rows))) (t/is (= 1 (count (remove (comp some? :deleted-at) rows))))) - (let [res (th/run-task! :objects-gc {:min-age 0})] + (let [res (th/run-task! :objects-gc {})] (t/is (= 2 (:processed res)))) (let [rows (th/db-query :file-thumbnail {:file-id (:id file)})] @@ -1251,7 +1270,7 @@ (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)}))) ;; Preventive objects-gc - (let [result (th/run-task! :objects-gc {:min-age 0})] + (let [result (th/run-task! :objects-gc {})] (t/is (= 1 (:processed result)))) ;; Check the number of fragments before adding the page @@ -1272,7 +1291,7 @@ (th/run-pending-tasks!)) ;; Clean objects after file-gc - (let [result (th/run-task! :objects-gc {:min-age 0})] + (let [result (th/run-task! :objects-gc {})] (t/is (= 1 (:processed result)))) ;; Check the number of fragments before adding the page @@ -1324,7 +1343,7 @@ (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)}))) ;; The objects-gc should remove unused fragments - (let [res (th/run-task! :objects-gc {:min-age 0})] + (let [res (th/run-task! :objects-gc {})] (t/is (= 2 (:processed res)))) ;; Check the number of fragments before adding the page @@ -1821,8 +1840,7 @@ (t/is (= (:id file-2) (:file-id (get rows 1)))) (t/is (nil? (:deleted-at (get rows 1))))) - (th/run-task! :objects-gc - {:min-age 0}) + (th/run-task! :objects-gc {}) (let [rows (th/db-exec! ["SELECT * FROM file_media_object ORDER BY created_at ASC"])] (t/is (= 1 (count rows))) diff --git a/backend/test/backend_tests/rpc_font_test.clj b/backend/test/backend_tests/rpc_font_test.clj index 91ee15f1f..08611426d 100644 --- a/backend/test/backend_tests/rpc_font_test.clj +++ b/backend/test/backend_tests/rpc_font_test.clj @@ -7,6 +7,7 @@ (ns backend-tests.rpc-font-test (:require [app.common.uuid :as uuid] + [app.config :as cf] [app.db :as db] [app.http :as http] [app.rpc :as-alias rpc] @@ -144,7 +145,7 @@ (t/is (= 0 (:freeze res))) (t/is (= 0 (:delete res)))) - (let [res (th/run-task! :objects-gc {:min-age 0})] + (let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})] (t/is (= 2 (:processed res)))) (let [res (th/run-task! :storage-gc-touched {:min-age 0})] @@ -204,7 +205,7 @@ (t/is (= 0 (:freeze res))) (t/is (= 0 (:delete res)))) - (let [res (th/run-task! :objects-gc {:min-age 0})] + (let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})] (t/is (= 1 (:processed res)))) (let [res (th/run-task! :storage-gc-touched {:min-age 0})] @@ -263,7 +264,7 @@ (t/is (= 0 (:freeze res))) (t/is (= 0 (:delete res)))) - (let [res (th/run-task! :objects-gc {:min-age 0})] + (let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})] (t/is (= 1 (:processed res)))) (let [res (th/run-task! :storage-gc-touched {:min-age 0})] diff --git a/backend/test/backend_tests/rpc_profile_test.clj b/backend/test/backend_tests/rpc_profile_test.clj index 47e58adba..7e4645f67 100644 --- a/backend/test/backend_tests/rpc_profile_test.clj +++ b/backend/test/backend_tests/rpc_profile_test.clj @@ -209,16 +209,16 @@ ::rpc/profile-id (:id prof1) :id (:id team1)} out (th/command! params)] - ;; (th/print-result! out) + ;; (th/print-result! out) (let [team (th/db-get :team {:id (:id team1)} {::db/remove-deleted false})] (t/is (dt/instant? (:deleted-at team))))) - ;; Request profile to be deleted + ;; Request profile to be deleted (let [params {::th/type :delete-profile ::rpc/profile-id (:id prof1)} out (th/command! params)] - ;; (th/print-result! out) + ;; (th/print-result! out) (t/is (nil? (:result out))) (t/is (nil? (:error out))))))) diff --git a/backend/test/backend_tests/rpc_project_test.clj b/backend/test/backend_tests/rpc_project_test.clj index 4613e6257..728cfd6e4 100644 --- a/backend/test/backend_tests/rpc_project_test.clj +++ b/backend/test/backend_tests/rpc_project_test.clj @@ -7,6 +7,7 @@ (ns backend-tests.rpc-project-test (:require [app.common.uuid :as uuid] + [app.config :as cf] [app.db :as db] [app.http :as http] [app.rpc :as-alias rpc] @@ -178,7 +179,7 @@ ;; project is not deleted because it does not meet all ;; conditions to be deleted. - (let [result (th/run-task! :objects-gc {:min-age 0})] + (let [result (th/run-task! :objects-gc {})] (t/is (= 0 (:processed result)))) ;; query the list of projects @@ -210,7 +211,7 @@ (t/is (= 1 (count result))))) ;; run permanent deletion (should be noop) - (let [result (th/run-task! :objects-gc {:min-age (dt/duration {:minutes 1})})] + (let [result (th/run-task! :objects-gc {})] (t/is (= 0 (:processed result)))) ;; query the list of files of a after soft deletion @@ -224,7 +225,7 @@ (t/is (= 0 (count result))))) ;; run permanent deletion - (let [result (th/run-task! :objects-gc {:min-age 0})] + (let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})] (t/is (= 1 (:processed result)))) ;; query the list of files of a after hard deletion diff --git a/backend/test/backend_tests/rpc_team_test.clj b/backend/test/backend_tests/rpc_team_test.clj index 66fa5d74c..a88d116dc 100644 --- a/backend/test/backend_tests/rpc_team_test.clj +++ b/backend/test/backend_tests/rpc_team_test.clj @@ -8,6 +8,7 @@ (:require [app.common.logging :as l] [app.common.uuid :as uuid] + [app.config :as cf] [app.db :as db] [app.http :as http] [app.rpc :as-alias rpc] @@ -459,7 +460,7 @@ ;; team is not deleted because it does not meet all ;; conditions to be deleted. - (let [result (th/run-task! :objects-gc {:min-age (dt/duration 0)})] + (let [result (th/run-task! :objects-gc {})] (t/is (= 0 (:processed result)))) ;; query the list of teams @@ -493,7 +494,7 @@ (th/run-pending-tasks!) ;; run permanent deletion (should be noop) - (let [result (th/run-task! :objects-gc {:min-age (dt/duration {:minutes 1})})] + (let [result (th/run-task! :objects-gc {})] (t/is (= 0 (:processed result)))) ;; query the list of projects after hard deletion @@ -507,7 +508,7 @@ (t/is (= :not-found (:type edata))))) ;; run permanent deletion - (let [result (th/run-task! :objects-gc {:min-age (dt/duration 0)})] + (let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})] (t/is (= 2 (:processed result)))) ;; query the list of projects of a after hard deletion @@ -521,7 +522,6 @@ (let [edata (-> out :error ex-data)] (t/is (= :not-found (:type edata))))))) - (t/deftest team-deletion-2 (let [storage (-> (:app.storage/storage th/*system*) (assoc ::sto/backend :assets-fs)) @@ -564,7 +564,7 @@ (t/is (= 1 (count rows))) (t/is (dt/instant? (:deleted-at (first rows))))) - (let [result (th/run-task! :objects-gc {:min-age 0})] + (let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})] (t/is (= 5 (:processed result)))))) (t/deftest create-team-access-request