Merge pull request #6512 from penpot/niwinz-develop-custom-deletion-rules

♻️ Normalize logical deletion delay handling
This commit is contained in:
Alejandro Alonso 2025-05-30 12:53:37 +02:00 committed by GitHub
commit 885231e9a1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 243 additions and 181 deletions

View file

@ -0,0 +1,31 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC
(ns app.features.logical-deletion
"A code related to handle logical deletion mechanism"
(:require
[app.config :as cf]
[app.util.time :as dt]))
(defn get-deletion-delay
"Calculate the next deleted-at for a resource (file, team, etc) in function
of team settings"
[team]
(if-let [subscription (get team :subscription)]
(cond
(and (= (:type subscription) "unlimited")
(= (:status subscription) "active"))
(dt/duration {:days 30})
(and (= (:type subscription) "enterprise")
(= (:status subscription) "active"))
(dt/duration {:days 90})
:else
(cf/get-deletion-delay))
(cf/get-deletion-delay)))

View file

@ -24,6 +24,7 @@
[app.db.sql :as-alias sql]
[app.features.fdata :as feat.fdata]
[app.features.file-migrations :as feat.fmigr]
[app.features.logical-deletion :as ldel]
[app.loggers.audit :as-alias audit]
[app.loggers.webhooks :as-alias webhooks]
[app.rpc :as-alias rpc]
@ -934,12 +935,13 @@
;; --- MUTATION COMMAND: delete-file
(defn- mark-file-deleted
[conn file-id]
(let [file (db/update! conn :file
{:deleted-at (dt/now)}
{:id file-id}
{::db/return-keys [:id :name :is-shared :deleted-at
:project-id :created-at :modified-at]})]
[conn team file-id]
(let [delay (ldel/get-deletion-delay team)
file (db/update! conn :file
{:deleted-at (dt/in-future delay)}
{:id file-id}
{::db/return-keys [:id :name :is-shared :deleted-at
:project-id :created-at :modified-at]})]
(wrk/submit! {::db/conn conn
::wrk/task :delete-object
::wrk/params {:object :file
@ -955,7 +957,11 @@
(defn- delete-file
[{:keys [::db/conn] :as cfg} {:keys [profile-id id] :as params}]
(check-edition-permissions! conn profile-id id)
(let [file (mark-file-deleted conn id)]
(let [team (teams/get-team conn
:profile-id profile-id
:file-id id)
file (mark-file-deleted conn team id)]
(rph/with-meta (rph/wrap)
{::audit/props {:project-id (:project-id file)
:name (:name file)

View file

@ -20,6 +20,7 @@
[app.db :as db]
[app.features.fdata :as feat.fdata]
[app.features.file-migrations :as feat.fmigr]
[app.features.logical-deletion :as ldel]
[app.http.errors :as errors]
[app.loggers.audit :as audit]
[app.loggers.webhooks :as webhooks]
@ -209,7 +210,7 @@
Only intended for internal use on this module."
[{:keys [::db/conn ::wrk/executor ::timestamp] :as cfg}
{:keys [profile-id file features changes session-id skip-validate] :as params}]
{:keys [profile-id file team features changes session-id skip-validate] :as params}]
(let [;; Retrieve the file data
file (feat.fmigr/resolve-applied-migrations cfg file)
@ -243,7 +244,7 @@
:created-at timestamp
:updated-at timestamp
:deleted-at (if (::snapshot-data file)
(dt/plus timestamp (cf/get-deletion-delay))
(dt/plus timestamp (ldel/get-deletion-delay team))
(dt/plus timestamp (dt/duration {:hours 1})))
:file-id (:id file)
:revn (:revn file)

View file

@ -12,6 +12,7 @@
[app.common.uuid :as uuid]
[app.db :as db]
[app.db.sql :as-alias sql]
[app.features.logical-deletion :as ldel]
[app.loggers.audit :as-alias audit]
[app.loggers.webhooks :as-alias webhooks]
[app.media :as media]
@ -202,32 +203,40 @@
(sv/defmethod ::delete-font
{::doc/added "1.18"
::webhooks/event? true
::sm/params schema:delete-font}
[cfg {:keys [::rpc/profile-id id team-id]}]
(db/tx-run! cfg
(fn [{:keys [::db/conn] :as cfg}]
(teams/check-edition-permissions! conn profile-id team-id)
(let [fonts (db/query conn :team-font-variant
{:team-id team-id
:font-id id
:deleted-at nil}
{::sql/for-update true})
tnow (dt/now)]
::sm/params schema:delete-font
::db/transaction true}
[{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id team-id]}]
(let [team (teams/get-team conn
:profile-id profile-id
:team-id team-id)
(when-not (seq fonts)
(ex/raise :type :not-found
:code :object-not-found))
fonts (db/query conn :team-font-variant
{:team-id team-id
:font-id id
:deleted-at nil}
{::sql/for-update true})
(doseq [font fonts]
(db/update! conn :team-font-variant
{:deleted-at tnow}
{:id (:id font)}))
delay (ldel/get-deletion-delay team)
tnow (dt/in-future delay)]
(rph/with-meta (rph/wrap)
{::audit/props {:id id
:team-id team-id
:name (:font-family (peek fonts))
:profile-id profile-id}})))))
(teams/check-edition-permissions! (:permissions team))
(when-not (seq fonts)
(ex/raise :type :not-found
:code :object-not-found))
(doseq [font fonts]
(db/update! conn :team-font-variant
{:deleted-at tnow}
{:id (:id font)}
{::db/return-keys false}))
(rph/with-meta (rph/wrap)
{::audit/props {:id id
:team-id team-id
:name (:font-family (peek fonts))
:profile-id profile-id}})))
;; --- DELETE FONT VARIANT
@ -239,19 +248,23 @@
(sv/defmethod ::delete-font-variant
{::doc/added "1.18"
::webhooks/event? true
::sm/params schema:delete-font-variant}
[cfg {:keys [::rpc/profile-id id team-id]}]
(db/tx-run! cfg
(fn [{:keys [::db/conn] :as cfg}]
(teams/check-edition-permissions! conn profile-id team-id)
(let [variant (db/get conn :team-font-variant
{:id id :team-id team-id}
{::sql/for-update true})]
::sm/params schema:delete-font-variant
::db/transaction true}
[{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id team-id]}]
(let [team (teams/get-team conn
:profile-id profile-id
:team-id team-id)
variant (db/get conn :team-font-variant
{:id id :team-id team-id}
{::sql/for-update true})
delay (ldel/get-deletion-delay team)]
(db/update! conn :team-font-variant
{:deleted-at (dt/now)}
{:id (:id variant)})
(teams/check-edition-permissions! (:permissions team))
(db/update! conn :team-font-variant
{:deleted-at (dt/in-future delay)}
{:id (:id variant)}
{::db/return-keys false})
(rph/with-meta (rph/wrap)
{::audit/props {:font-family (:font-family variant)
:font-id (:font-id variant)}})))))
(rph/with-meta (rph/wrap)
{::audit/props {:font-family (:font-family variant)
:font-id (:font-id variant)}})))

View file

@ -480,8 +480,7 @@
JOIN team AS t ON (t.id = tpr.team_id)
WHERE tpr.is_owner IS TRUE
AND tpr.profile_id = ?
AND (t.deleted_at IS NULL OR
t.deleted_at > now())
AND t.deleted_at IS NULL
)
SELECT tpr.team_id AS id,
count(tpr.profile_id) - 1 AS participants

View file

@ -11,6 +11,7 @@
[app.common.schema :as sm]
[app.db :as db]
[app.db.sql :as-alias sql]
[app.features.logical-deletion :as ldel]
[app.loggers.audit :as-alias audit]
[app.loggers.webhooks :as webhooks]
[app.rpc :as-alias rpc]
@ -253,9 +254,10 @@
;; --- MUTATION: Delete Project
(defn- delete-project
[conn project-id]
(let [project (db/update! conn :project
{:deleted-at (dt/now)}
[conn team project-id]
(let [delay (ldel/get-deletion-delay team)
project (db/update! conn :project
{:deleted-at (dt/in-future delay)}
{:id project-id}
{::db/return-keys true})]
@ -272,7 +274,6 @@
project))
(def ^:private schema:delete-project
[:map {:title "delete-project"}
[:id ::sm/uuid]])
@ -284,7 +285,10 @@
::db/transaction true}
[{:keys [::db/conn]} {:keys [::rpc/profile-id id] :as params}]
(check-edition-permissions! conn profile-id id)
(let [project (delete-project conn id)]
(let [team (teams/get-team conn
:profile-id profile-id
:project-id id)
project (delete-project conn team id)]
(rph/with-meta (rph/wrap)
{::audit/props {:team-id (:team-id project)
:name (:name project)

View file

@ -17,6 +17,7 @@
[app.db :as db]
[app.db.sql :as sql]
[app.email :as eml]
[app.features.logical-deletion :as ldel]
[app.loggers.audit :as audit]
[app.main :as-alias main]
[app.media :as media]
@ -233,39 +234,43 @@
(defn get-team
[conn & {:keys [profile-id team-id project-id file-id] :as params}]
(dm/assert!
"connection or pool is mandatory"
(or (db/connection? conn)
(db/pool? conn)))
(assert (uuid? profile-id) "profile-id is mandatory")
(assert (or (db/connection? conn)
(db/pool? conn))
"connection or pool is mandatory")
(dm/assert!
"profile-id is mandatory"
(uuid? profile-id))
(let [{:keys [default-team-id] :as profile}
(profile/get-profile conn profile-id)
(let [{:keys [default-team-id] :as profile} (profile/get-profile conn profile-id)
result (cond
(some? team-id)
(let [sql (str "WITH teams AS (" sql:get-teams-with-permissions
") SELECT * FROM teams WHERE id=?")]
(db/exec-one! conn [sql default-team-id profile-id team-id]))
sql
(if (contains? cf/flags :subscriptions)
sql:get-teams-with-permissions-and-subscription
sql:get-teams-with-permissions)
(some? project-id)
(let [sql (str "WITH teams AS (" sql:get-teams-with-permissions ") "
"SELECT t.* FROM teams AS t "
" JOIN project AS p ON (p.team_id = t.id) "
" WHERE p.id=?")]
(db/exec-one! conn [sql default-team-id profile-id project-id]))
result
(cond
(some? team-id)
(let [sql (str "WITH teams AS (" sql ") "
"SELECT * FROM teams WHERE id=?")]
(db/exec-one! conn [sql default-team-id profile-id team-id]))
(some? file-id)
(let [sql (str "WITH teams AS (" sql:get-teams-with-permissions ") "
"SELECT t.* FROM teams AS t "
" JOIN project AS p ON (p.team_id = t.id) "
" JOIN file AS f ON (f.project_id = p.id) "
" WHERE f.id=?")]
(db/exec-one! conn [sql default-team-id profile-id file-id]))
(some? project-id)
(let [sql (str "WITH teams AS (" sql ") "
"SELECT t.* FROM teams AS t "
" JOIN project AS p ON (p.team_id = t.id) "
" WHERE p.id=?")]
(db/exec-one! conn [sql default-team-id profile-id project-id]))
:else
(throw (IllegalArgumentException. "invalid arguments")))]
(some? file-id)
(let [sql (str "WITH teams AS (" sql ") "
"SELECT t.* FROM teams AS t "
" JOIN project AS p ON (p.team_id = t.id) "
" JOIN file AS f ON (f.project_id = p.id) "
" WHERE f.id=?")]
(db/exec-one! conn [sql default-team-id profile-id file-id]))
:else
(throw (IllegalArgumentException. "invalid arguments")))]
(when-not result
(ex/raise :type :not-found
@ -653,13 +658,13 @@
(defn- delete-team
"Mark a team for deletion"
[conn team-id]
[conn {:keys [id] :as team}]
(let [deleted-at (dt/now)
team (db/update! conn :team
{:deleted-at deleted-at}
{:id team-id}
{::db/return-keys true})]
(let [delay (ldel/get-deletion-delay team)
team (db/update! conn :team
{:deleted-at (dt/in-future delay)}
{:id id}
{::db/return-keys true})]
(when (:is-default team)
(ex/raise :type :validation
@ -669,8 +674,8 @@
(wrk/submit! {::db/conn conn
::wrk/task :delete-object
::wrk/params {:object :team
:deleted-at deleted-at
:id team-id}})
:deleted-at (:deleted-at team)
:id id}})
team))
(def ^:private schema:delete-team
@ -682,12 +687,14 @@
::sm/params schema:delete-team
::db/transaction true}
[{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id] :as params}]
(let [perms (get-permissions conn profile-id id)]
(let [team (get-team conn :profile-id profile-id :team-id id)
perms (get team :permissions)]
(when-not (:is-owner perms)
(ex/raise :type :validation
:code :only-owner-can-delete-team))
(delete-team conn id)
(delete-team conn team)
nil))
;; --- Mutation: Team Update Role

View file

@ -9,7 +9,6 @@
of deleted or unreachable objects."
(:require
[app.common.logging :as l]
[app.config :as cf]
[app.db :as db]
[app.storage :as sto]
[app.util.time :as dt]
@ -18,15 +17,15 @@
(def ^:private sql:get-profiles
"SELECT id, photo_id FROM profile
WHERE deleted_at IS NOT NULL
AND deleted_at < now() - ?::interval
AND deleted_at < now() + ?::interval
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-profiles!
[{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-profiles min-age chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-profiles deletion-threshold chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id photo-id]}]
(l/trc :hint "permanently delete" :rel "profile" :id (str id))
@ -41,15 +40,15 @@
(def ^:private sql:get-teams
"SELECT deleted_at, id, photo_id FROM team
WHERE deleted_at IS NOT NULL
AND deleted_at < now() - ?::interval
AND deleted_at < now() + ?::interval
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-teams!
[{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-teams min-age chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-teams deletion-threshold chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id photo-id deleted-at]}]
(l/trc :hint "permanently delete"
:rel "team"
@ -69,15 +68,15 @@
"SELECT id, team_id, deleted_at, woff1_file_id, woff2_file_id, otf_file_id, ttf_file_id
FROM team_font_variant
WHERE deleted_at IS NOT NULL
AND deleted_at < now() - ?::interval
AND deleted_at < now() + ?::interval
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-fonts!
[{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-fonts min-age chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-fonts deletion-threshold chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id team-id deleted-at] :as font}]
(l/trc :hint "permanently delete"
:rel "team-font-variant"
@ -101,15 +100,15 @@
"SELECT id, deleted_at, team_id
FROM project
WHERE deleted_at IS NOT NULL
AND deleted_at < now() - ?::interval
AND deleted_at < now() + ?::interval
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-projects!
[{:keys [::db/conn ::min-age ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-projects min-age chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-projects deletion-threshold chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id team-id deleted-at]}]
(l/trc :hint "permanently delete"
:rel "project"
@ -127,15 +126,15 @@
"SELECT id, deleted_at, project_id, data_backend, data_ref_id
FROM file
WHERE deleted_at IS NOT NULL
AND deleted_at < now() - ?::interval
AND deleted_at < now() + ?::interval
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-files!
[{:keys [::db/conn ::sto/storage ::min-age ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-files min-age chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::sto/storage ::deletion-threshold ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-files deletion-threshold chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id deleted-at project-id] :as file}]
(l/trc :hint "permanently delete"
:rel "file"
@ -156,15 +155,15 @@
"SELECT file_id, revn, media_id, deleted_at
FROM file_thumbnail
WHERE deleted_at IS NOT NULL
AND deleted_at < now() - ?::interval
AND deleted_at < now() + ?::interval
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn delete-file-thumbnails!
[{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-thumbnails min-age chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-thumbnails deletion-threshold chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [file-id revn media-id deleted-at]}]
(l/trc :hint "permanently delete"
:rel "file-thumbnail"
@ -185,15 +184,15 @@
"SELECT file_id, object_id, media_id, deleted_at
FROM file_tagged_object_thumbnail
WHERE deleted_at IS NOT NULL
AND deleted_at < now() - ?::interval
AND deleted_at < now() + ?::interval
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn delete-file-object-thumbnails!
[{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-object-thumbnails min-age chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-object-thumbnails deletion-threshold chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [file-id object-id media-id deleted-at]}]
(l/trc :hint "permanently delete"
:rel "file-tagged-object-thumbnail"
@ -214,15 +213,15 @@
"SELECT file_id, id, deleted_at, data_ref_id
FROM file_data_fragment
WHERE deleted_at IS NOT NULL
AND deleted_at < now() - ?::interval
AND deleted_at < now() + ?::interval
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-file-data-fragments!
[{:keys [::db/conn ::sto/storage ::min-age ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-file-data-fragments min-age chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::sto/storage ::deletion-threshold ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-file-data-fragments deletion-threshold chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [file-id id deleted-at data-ref-id]}]
(l/trc :hint "permanently delete"
:rel "file-data-fragment"
@ -240,15 +239,15 @@
"SELECT id, file_id, media_id, thumbnail_id, deleted_at
FROM file_media_object
WHERE deleted_at IS NOT NULL
AND deleted_at < now() - ?::interval
AND deleted_at < now() + ?::interval
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-file-media-objects!
[{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-media-objects min-age chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-media-objects deletion-threshold chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id file-id deleted-at] :as fmo}]
(l/trc :hint "permanently delete"
:rel "file-media-object"
@ -269,15 +268,15 @@
"SELECT id, file_id, deleted_at, data_backend, data_ref_id
FROM file_change
WHERE deleted_at IS NOT NULL
AND deleted_at < now() - ?::interval
AND deleted_at < now() + ?::interval
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-file-changes!
[{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-change min-age chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-change deletion-threshold chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id file-id deleted-at] :as xlog}]
(l/trc :hint "permanently delete"
:rel "file-change"
@ -324,16 +323,13 @@
(defmethod ig/expand-key ::handler
[k v]
{k (assoc v
::min-age (cf/get-deletion-delay)
::chunk-size 100)})
{k (assoc v ::chunk-size 100)})
(defmethod ig/init-key ::handler
[_ cfg]
(fn [{:keys [props] :as task}]
(let [min-age (dt/duration (or (:min-age props) (::min-age cfg)))
cfg (assoc cfg ::min-age (db/interval min-age))]
(let [threshold (dt/duration (get props :deletion-threshold 0))
cfg (assoc cfg ::deletion-threshold (db/interval threshold))]
(loop [procs (map deref deletion-proc-vars)
total 0]
(if-let [proc-fn (first procs)]