mirror of
https://github.com/penpot/penpot.git
synced 2025-06-18 20:31:37 +02:00
♻️ Normalize logical deletion to future dates
Instead of managing the ...
This commit is contained in:
parent
a3b4fc9545
commit
a706907b26
14 changed files with 243 additions and 166 deletions
31
backend/src/app/features/logical_deletion.clj
Normal file
31
backend/src/app/features/logical_deletion.clj
Normal file
|
@ -0,0 +1,31 @@
|
|||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC
|
||||
|
||||
(ns app.features.logical-deletion
|
||||
"A code related to handle logical deletion mechanism"
|
||||
(:require
|
||||
[app.config :as cf]
|
||||
[app.util.time :as dt]))
|
||||
|
||||
(defn get-deletion-delay
|
||||
"Calculate the next deleted-at for a resource (file, team, etc) in function
|
||||
of team settings"
|
||||
[team]
|
||||
(if-let [subscription (get team :subscription)]
|
||||
(cond
|
||||
(and (= (:type subscription) "unlimited")
|
||||
(= (:status subscription) "active"))
|
||||
(dt/duration {:days 30})
|
||||
|
||||
(and (= (:type subscription) "enterprise")
|
||||
(= (:status subscription) "active"))
|
||||
(dt/duration {:days 90})
|
||||
|
||||
:else
|
||||
(cf/get-deletion-delay))
|
||||
|
||||
(cf/get-deletion-delay)))
|
||||
|
|
@ -23,6 +23,7 @@
|
|||
[app.db.sql :as-alias sql]
|
||||
[app.features.fdata :as feat.fdata]
|
||||
[app.features.file-migrations :as feat.fmigr]
|
||||
[app.features.logical-deletion :as ldel]
|
||||
[app.loggers.audit :as-alias audit]
|
||||
[app.loggers.webhooks :as-alias webhooks]
|
||||
[app.rpc :as-alias rpc]
|
||||
|
@ -970,12 +971,13 @@
|
|||
;; --- MUTATION COMMAND: delete-file
|
||||
|
||||
(defn- mark-file-deleted
|
||||
[conn file-id]
|
||||
(let [file (db/update! conn :file
|
||||
{:deleted-at (dt/now)}
|
||||
{:id file-id}
|
||||
{::db/return-keys [:id :name :is-shared :deleted-at
|
||||
:project-id :created-at :modified-at]})]
|
||||
[conn team file-id]
|
||||
(let [delay (ldel/get-deletion-delay team)
|
||||
file (db/update! conn :file
|
||||
{:deleted-at (dt/in-future delay)}
|
||||
{:id file-id}
|
||||
{::db/return-keys [:id :name :is-shared :deleted-at
|
||||
:project-id :created-at :modified-at]})]
|
||||
(wrk/submit! {::db/conn conn
|
||||
::wrk/task :delete-object
|
||||
::wrk/params {:object :file
|
||||
|
@ -991,7 +993,11 @@
|
|||
(defn- delete-file
|
||||
[{:keys [::db/conn] :as cfg} {:keys [profile-id id] :as params}]
|
||||
(check-edition-permissions! conn profile-id id)
|
||||
(let [file (mark-file-deleted conn id)]
|
||||
(let [team (teams/get-team conn
|
||||
:profile-id profile-id
|
||||
:file-id id)
|
||||
file (mark-file-deleted conn team id)]
|
||||
|
||||
(rph/with-meta (rph/wrap)
|
||||
{::audit/props {:project-id (:project-id file)
|
||||
:name (:name file)
|
||||
|
|
|
@ -20,6 +20,7 @@
|
|||
[app.db :as db]
|
||||
[app.features.fdata :as feat.fdata]
|
||||
[app.features.file-migrations :as feat.fmigr]
|
||||
[app.features.logical-deletion :as ldel]
|
||||
[app.http.errors :as errors]
|
||||
[app.loggers.audit :as audit]
|
||||
[app.loggers.webhooks :as webhooks]
|
||||
|
@ -209,7 +210,7 @@
|
|||
|
||||
Only intended for internal use on this module."
|
||||
[{:keys [::db/conn ::wrk/executor ::timestamp] :as cfg}
|
||||
{:keys [profile-id file features changes session-id skip-validate] :as params}]
|
||||
{:keys [profile-id file team features changes session-id skip-validate] :as params}]
|
||||
|
||||
(let [;; Retrieve the file data
|
||||
file (feat.fmigr/resolve-applied-migrations cfg file)
|
||||
|
@ -243,7 +244,7 @@
|
|||
:created-at timestamp
|
||||
:updated-at timestamp
|
||||
:deleted-at (if (::snapshot-data file)
|
||||
(dt/plus timestamp (cf/get-deletion-delay))
|
||||
(dt/plus timestamp (ldel/get-deletion-delay team))
|
||||
(dt/plus timestamp (dt/duration {:hours 1})))
|
||||
:file-id (:id file)
|
||||
:revn (:revn file)
|
||||
|
|
|
@ -12,6 +12,7 @@
|
|||
[app.common.uuid :as uuid]
|
||||
[app.db :as db]
|
||||
[app.db.sql :as-alias sql]
|
||||
[app.features.logical-deletion :as ldel]
|
||||
[app.loggers.audit :as-alias audit]
|
||||
[app.loggers.webhooks :as-alias webhooks]
|
||||
[app.media :as media]
|
||||
|
@ -202,32 +203,40 @@
|
|||
(sv/defmethod ::delete-font
|
||||
{::doc/added "1.18"
|
||||
::webhooks/event? true
|
||||
::sm/params schema:delete-font}
|
||||
[cfg {:keys [::rpc/profile-id id team-id]}]
|
||||
(db/tx-run! cfg
|
||||
(fn [{:keys [::db/conn] :as cfg}]
|
||||
(teams/check-edition-permissions! conn profile-id team-id)
|
||||
(let [fonts (db/query conn :team-font-variant
|
||||
{:team-id team-id
|
||||
:font-id id
|
||||
:deleted-at nil}
|
||||
{::sql/for-update true})
|
||||
tnow (dt/now)]
|
||||
::sm/params schema:delete-font
|
||||
::db/transaction true}
|
||||
[{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id team-id]}]
|
||||
(let [team (teams/get-team conn
|
||||
:profile-id profile-id
|
||||
:team-id team-id)
|
||||
|
||||
(when-not (seq fonts)
|
||||
(ex/raise :type :not-found
|
||||
:code :object-not-found))
|
||||
fonts (db/query conn :team-font-variant
|
||||
{:team-id team-id
|
||||
:font-id id
|
||||
:deleted-at nil}
|
||||
{::sql/for-update true})
|
||||
|
||||
(doseq [font fonts]
|
||||
(db/update! conn :team-font-variant
|
||||
{:deleted-at tnow}
|
||||
{:id (:id font)}))
|
||||
delay (ldel/get-deletion-delay team)
|
||||
tnow (dt/in-future delay)]
|
||||
|
||||
(rph/with-meta (rph/wrap)
|
||||
{::audit/props {:id id
|
||||
:team-id team-id
|
||||
:name (:font-family (peek fonts))
|
||||
:profile-id profile-id}})))))
|
||||
(teams/check-edition-permissions! (:permissions team))
|
||||
|
||||
(when-not (seq fonts)
|
||||
(ex/raise :type :not-found
|
||||
:code :object-not-found))
|
||||
|
||||
|
||||
(doseq [font fonts]
|
||||
(db/update! conn :team-font-variant
|
||||
{:deleted-at tnow}
|
||||
{:id (:id font)}
|
||||
{::db/return-keys false}))
|
||||
|
||||
(rph/with-meta (rph/wrap)
|
||||
{::audit/props {:id id
|
||||
:team-id team-id
|
||||
:name (:font-family (peek fonts))
|
||||
:profile-id profile-id}})))
|
||||
|
||||
;; --- DELETE FONT VARIANT
|
||||
|
||||
|
@ -239,19 +248,23 @@
|
|||
(sv/defmethod ::delete-font-variant
|
||||
{::doc/added "1.18"
|
||||
::webhooks/event? true
|
||||
::sm/params schema:delete-font-variant}
|
||||
[cfg {:keys [::rpc/profile-id id team-id]}]
|
||||
(db/tx-run! cfg
|
||||
(fn [{:keys [::db/conn] :as cfg}]
|
||||
(teams/check-edition-permissions! conn profile-id team-id)
|
||||
(let [variant (db/get conn :team-font-variant
|
||||
{:id id :team-id team-id}
|
||||
{::sql/for-update true})]
|
||||
::sm/params schema:delete-font-variant
|
||||
::db/transaction true}
|
||||
[{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id team-id]}]
|
||||
(let [team (teams/get-team conn
|
||||
:profile-id profile-id
|
||||
:team-id team-id)
|
||||
variant (db/get conn :team-font-variant
|
||||
{:id id :team-id team-id}
|
||||
{::sql/for-update true})
|
||||
delay (ldel/get-deletion-delay team)]
|
||||
|
||||
(db/update! conn :team-font-variant
|
||||
{:deleted-at (dt/now)}
|
||||
{:id (:id variant)})
|
||||
(teams/check-edition-permissions! (:permissions team))
|
||||
(db/update! conn :team-font-variant
|
||||
{:deleted-at (dt/in-future delay)}
|
||||
{:id (:id variant)}
|
||||
{::db/return-keys false})
|
||||
|
||||
(rph/with-meta (rph/wrap)
|
||||
{::audit/props {:font-family (:font-family variant)
|
||||
:font-id (:font-id variant)}})))))
|
||||
(rph/with-meta (rph/wrap)
|
||||
{::audit/props {:font-family (:font-family variant)
|
||||
:font-id (:font-id variant)}})))
|
||||
|
|
|
@ -480,8 +480,7 @@
|
|||
JOIN team AS t ON (t.id = tpr.team_id)
|
||||
WHERE tpr.is_owner IS TRUE
|
||||
AND tpr.profile_id = ?
|
||||
AND (t.deleted_at IS NULL OR
|
||||
t.deleted_at > now())
|
||||
AND t.deleted_at IS NULL
|
||||
)
|
||||
SELECT tpr.team_id AS id,
|
||||
count(tpr.profile_id) - 1 AS participants
|
||||
|
|
|
@ -11,6 +11,7 @@
|
|||
[app.common.schema :as sm]
|
||||
[app.db :as db]
|
||||
[app.db.sql :as-alias sql]
|
||||
[app.features.logical-deletion :as ldel]
|
||||
[app.loggers.audit :as-alias audit]
|
||||
[app.loggers.webhooks :as webhooks]
|
||||
[app.rpc :as-alias rpc]
|
||||
|
@ -253,9 +254,10 @@
|
|||
;; --- MUTATION: Delete Project
|
||||
|
||||
(defn- delete-project
|
||||
[conn project-id]
|
||||
(let [project (db/update! conn :project
|
||||
{:deleted-at (dt/now)}
|
||||
[conn team project-id]
|
||||
(let [delay (ldel/get-deletion-delay team)
|
||||
project (db/update! conn :project
|
||||
{:deleted-at (dt/in-future delay)}
|
||||
{:id project-id}
|
||||
{::db/return-keys true})]
|
||||
|
||||
|
@ -272,7 +274,6 @@
|
|||
|
||||
project))
|
||||
|
||||
|
||||
(def ^:private schema:delete-project
|
||||
[:map {:title "delete-project"}
|
||||
[:id ::sm/uuid]])
|
||||
|
@ -284,7 +285,10 @@
|
|||
::db/transaction true}
|
||||
[{:keys [::db/conn]} {:keys [::rpc/profile-id id] :as params}]
|
||||
(check-edition-permissions! conn profile-id id)
|
||||
(let [project (delete-project conn id)]
|
||||
(let [team (teams/get-team conn
|
||||
:profile-id profile-id
|
||||
:project-id id)
|
||||
project (delete-project conn team id)]
|
||||
(rph/with-meta (rph/wrap)
|
||||
{::audit/props {:team-id (:team-id project)
|
||||
:name (:name project)
|
||||
|
|
|
@ -17,6 +17,7 @@
|
|||
[app.db :as db]
|
||||
[app.db.sql :as sql]
|
||||
[app.email :as eml]
|
||||
[app.features.logical-deletion :as ldel]
|
||||
[app.loggers.audit :as audit]
|
||||
[app.main :as-alias main]
|
||||
[app.media :as media]
|
||||
|
@ -214,39 +215,43 @@
|
|||
(defn get-team
|
||||
[conn & {:keys [profile-id team-id project-id file-id] :as params}]
|
||||
|
||||
(dm/assert!
|
||||
"connection or pool is mandatory"
|
||||
(or (db/connection? conn)
|
||||
(db/pool? conn)))
|
||||
(assert (uuid? profile-id) "profile-id is mandatory")
|
||||
(assert (or (db/connection? conn)
|
||||
(db/pool? conn))
|
||||
"connection or pool is mandatory")
|
||||
|
||||
(dm/assert!
|
||||
"profile-id is mandatory"
|
||||
(uuid? profile-id))
|
||||
(let [{:keys [default-team-id] :as profile}
|
||||
(profile/get-profile conn profile-id)
|
||||
|
||||
(let [{:keys [default-team-id] :as profile} (profile/get-profile conn profile-id)
|
||||
result (cond
|
||||
(some? team-id)
|
||||
(let [sql (str "WITH teams AS (" sql:get-teams-with-permissions
|
||||
") SELECT * FROM teams WHERE id=?")]
|
||||
(db/exec-one! conn [sql default-team-id profile-id team-id]))
|
||||
sql
|
||||
(if (contains? cf/flags :subscriptions)
|
||||
sql:get-teams-with-permissions-and-subscription
|
||||
sql:get-teams-with-permissions)
|
||||
|
||||
(some? project-id)
|
||||
(let [sql (str "WITH teams AS (" sql:get-teams-with-permissions ") "
|
||||
"SELECT t.* FROM teams AS t "
|
||||
" JOIN project AS p ON (p.team_id = t.id) "
|
||||
" WHERE p.id=?")]
|
||||
(db/exec-one! conn [sql default-team-id profile-id project-id]))
|
||||
result
|
||||
(cond
|
||||
(some? team-id)
|
||||
(let [sql (str "WITH teams AS (" sql ") "
|
||||
"SELECT * FROM teams WHERE id=?")]
|
||||
(db/exec-one! conn [sql default-team-id profile-id team-id]))
|
||||
|
||||
(some? file-id)
|
||||
(let [sql (str "WITH teams AS (" sql:get-teams-with-permissions ") "
|
||||
"SELECT t.* FROM teams AS t "
|
||||
" JOIN project AS p ON (p.team_id = t.id) "
|
||||
" JOIN file AS f ON (f.project_id = p.id) "
|
||||
" WHERE f.id=?")]
|
||||
(db/exec-one! conn [sql default-team-id profile-id file-id]))
|
||||
(some? project-id)
|
||||
(let [sql (str "WITH teams AS (" sql ") "
|
||||
"SELECT t.* FROM teams AS t "
|
||||
" JOIN project AS p ON (p.team_id = t.id) "
|
||||
" WHERE p.id=?")]
|
||||
(db/exec-one! conn [sql default-team-id profile-id project-id]))
|
||||
|
||||
:else
|
||||
(throw (IllegalArgumentException. "invalid arguments")))]
|
||||
(some? file-id)
|
||||
(let [sql (str "WITH teams AS (" sql ") "
|
||||
"SELECT t.* FROM teams AS t "
|
||||
" JOIN project AS p ON (p.team_id = t.id) "
|
||||
" JOIN file AS f ON (f.project_id = p.id) "
|
||||
" WHERE f.id=?")]
|
||||
(db/exec-one! conn [sql default-team-id profile-id file-id]))
|
||||
|
||||
:else
|
||||
(throw (IllegalArgumentException. "invalid arguments")))]
|
||||
|
||||
(when-not result
|
||||
(ex/raise :type :not-found
|
||||
|
@ -634,13 +639,13 @@
|
|||
|
||||
(defn- delete-team
|
||||
"Mark a team for deletion"
|
||||
[conn team-id]
|
||||
[conn {:keys [id] :as team}]
|
||||
|
||||
(let [deleted-at (dt/now)
|
||||
team (db/update! conn :team
|
||||
{:deleted-at deleted-at}
|
||||
{:id team-id}
|
||||
{::db/return-keys true})]
|
||||
(let [delay (ldel/get-deletion-delay team)
|
||||
team (db/update! conn :team
|
||||
{:deleted-at (dt/in-future delay)}
|
||||
{:id id}
|
||||
{::db/return-keys true})]
|
||||
|
||||
(when (:is-default team)
|
||||
(ex/raise :type :validation
|
||||
|
@ -650,8 +655,8 @@
|
|||
(wrk/submit! {::db/conn conn
|
||||
::wrk/task :delete-object
|
||||
::wrk/params {:object :team
|
||||
:deleted-at deleted-at
|
||||
:id team-id}})
|
||||
:deleted-at (:deleted-at team)
|
||||
:id id}})
|
||||
team))
|
||||
|
||||
(def ^:private schema:delete-team
|
||||
|
@ -663,12 +668,14 @@
|
|||
::sm/params schema:delete-team
|
||||
::db/transaction true}
|
||||
[{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id] :as params}]
|
||||
(let [perms (get-permissions conn profile-id id)]
|
||||
(let [team (get-team conn :profile-id profile-id :team-id id)
|
||||
perms (get team :permissions)]
|
||||
|
||||
(when-not (:is-owner perms)
|
||||
(ex/raise :type :validation
|
||||
:code :only-owner-can-delete-team))
|
||||
|
||||
(delete-team conn id)
|
||||
(delete-team conn team)
|
||||
nil))
|
||||
|
||||
;; --- Mutation: Team Update Role
|
||||
|
|
|
@ -9,7 +9,6 @@
|
|||
of deleted or unreachable objects."
|
||||
(:require
|
||||
[app.common.logging :as l]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.storage :as sto]
|
||||
[app.util.time :as dt]
|
||||
|
@ -18,15 +17,15 @@
|
|||
(def ^:private sql:get-profiles
|
||||
"SELECT id, photo_id FROM profile
|
||||
WHERE deleted_at IS NOT NULL
|
||||
AND deleted_at < now() - ?::interval
|
||||
AND deleted_at < now() + ?::interval
|
||||
ORDER BY deleted_at ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE
|
||||
SKIP LOCKED")
|
||||
|
||||
(defn- delete-profiles!
|
||||
[{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-profiles min-age chunk-size] {:fetch-size 5})
|
||||
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-profiles deletion-threshold chunk-size] {:fetch-size 5})
|
||||
(reduce (fn [total {:keys [id photo-id]}]
|
||||
(l/trc :hint "permanently delete" :rel "profile" :id (str id))
|
||||
|
||||
|
@ -41,15 +40,15 @@
|
|||
(def ^:private sql:get-teams
|
||||
"SELECT deleted_at, id, photo_id FROM team
|
||||
WHERE deleted_at IS NOT NULL
|
||||
AND deleted_at < now() - ?::interval
|
||||
AND deleted_at < now() + ?::interval
|
||||
ORDER BY deleted_at ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE
|
||||
SKIP LOCKED")
|
||||
|
||||
(defn- delete-teams!
|
||||
[{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-teams min-age chunk-size] {:fetch-size 5})
|
||||
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-teams deletion-threshold chunk-size] {:fetch-size 5})
|
||||
(reduce (fn [total {:keys [id photo-id deleted-at]}]
|
||||
(l/trc :hint "permanently delete"
|
||||
:rel "team"
|
||||
|
@ -69,15 +68,15 @@
|
|||
"SELECT id, team_id, deleted_at, woff1_file_id, woff2_file_id, otf_file_id, ttf_file_id
|
||||
FROM team_font_variant
|
||||
WHERE deleted_at IS NOT NULL
|
||||
AND deleted_at < now() - ?::interval
|
||||
AND deleted_at < now() + ?::interval
|
||||
ORDER BY deleted_at ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE
|
||||
SKIP LOCKED")
|
||||
|
||||
(defn- delete-fonts!
|
||||
[{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-fonts min-age chunk-size] {:fetch-size 5})
|
||||
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-fonts deletion-threshold chunk-size] {:fetch-size 5})
|
||||
(reduce (fn [total {:keys [id team-id deleted-at] :as font}]
|
||||
(l/trc :hint "permanently delete"
|
||||
:rel "team-font-variant"
|
||||
|
@ -101,15 +100,15 @@
|
|||
"SELECT id, deleted_at, team_id
|
||||
FROM project
|
||||
WHERE deleted_at IS NOT NULL
|
||||
AND deleted_at < now() - ?::interval
|
||||
AND deleted_at < now() + ?::interval
|
||||
ORDER BY deleted_at ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE
|
||||
SKIP LOCKED")
|
||||
|
||||
(defn- delete-projects!
|
||||
[{:keys [::db/conn ::min-age ::chunk-size] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-projects min-age chunk-size] {:fetch-size 5})
|
||||
[{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-projects deletion-threshold chunk-size] {:fetch-size 5})
|
||||
(reduce (fn [total {:keys [id team-id deleted-at]}]
|
||||
(l/trc :hint "permanently delete"
|
||||
:rel "project"
|
||||
|
@ -127,15 +126,15 @@
|
|||
"SELECT id, deleted_at, project_id, data_backend, data_ref_id
|
||||
FROM file
|
||||
WHERE deleted_at IS NOT NULL
|
||||
AND deleted_at < now() - ?::interval
|
||||
AND deleted_at < now() + ?::interval
|
||||
ORDER BY deleted_at ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE
|
||||
SKIP LOCKED")
|
||||
|
||||
(defn- delete-files!
|
||||
[{:keys [::db/conn ::sto/storage ::min-age ::chunk-size] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-files min-age chunk-size] {:fetch-size 5})
|
||||
[{:keys [::db/conn ::sto/storage ::deletion-threshold ::chunk-size] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-files deletion-threshold chunk-size] {:fetch-size 5})
|
||||
(reduce (fn [total {:keys [id deleted-at project-id] :as file}]
|
||||
(l/trc :hint "permanently delete"
|
||||
:rel "file"
|
||||
|
@ -156,15 +155,15 @@
|
|||
"SELECT file_id, revn, media_id, deleted_at
|
||||
FROM file_thumbnail
|
||||
WHERE deleted_at IS NOT NULL
|
||||
AND deleted_at < now() - ?::interval
|
||||
AND deleted_at < now() + ?::interval
|
||||
ORDER BY deleted_at ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE
|
||||
SKIP LOCKED")
|
||||
|
||||
(defn delete-file-thumbnails!
|
||||
[{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-file-thumbnails min-age chunk-size] {:fetch-size 5})
|
||||
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-file-thumbnails deletion-threshold chunk-size] {:fetch-size 5})
|
||||
(reduce (fn [total {:keys [file-id revn media-id deleted-at]}]
|
||||
(l/trc :hint "permanently delete"
|
||||
:rel "file-thumbnail"
|
||||
|
@ -185,15 +184,15 @@
|
|||
"SELECT file_id, object_id, media_id, deleted_at
|
||||
FROM file_tagged_object_thumbnail
|
||||
WHERE deleted_at IS NOT NULL
|
||||
AND deleted_at < now() - ?::interval
|
||||
AND deleted_at < now() + ?::interval
|
||||
ORDER BY deleted_at ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE
|
||||
SKIP LOCKED")
|
||||
|
||||
(defn delete-file-object-thumbnails!
|
||||
[{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-file-object-thumbnails min-age chunk-size] {:fetch-size 5})
|
||||
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-file-object-thumbnails deletion-threshold chunk-size] {:fetch-size 5})
|
||||
(reduce (fn [total {:keys [file-id object-id media-id deleted-at]}]
|
||||
(l/trc :hint "permanently delete"
|
||||
:rel "file-tagged-object-thumbnail"
|
||||
|
@ -214,15 +213,15 @@
|
|||
"SELECT file_id, id, deleted_at, data_ref_id
|
||||
FROM file_data_fragment
|
||||
WHERE deleted_at IS NOT NULL
|
||||
AND deleted_at < now() - ?::interval
|
||||
AND deleted_at < now() + ?::interval
|
||||
ORDER BY deleted_at ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE
|
||||
SKIP LOCKED")
|
||||
|
||||
(defn- delete-file-data-fragments!
|
||||
[{:keys [::db/conn ::sto/storage ::min-age ::chunk-size] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-file-data-fragments min-age chunk-size] {:fetch-size 5})
|
||||
[{:keys [::db/conn ::sto/storage ::deletion-threshold ::chunk-size] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-file-data-fragments deletion-threshold chunk-size] {:fetch-size 5})
|
||||
(reduce (fn [total {:keys [file-id id deleted-at data-ref-id]}]
|
||||
(l/trc :hint "permanently delete"
|
||||
:rel "file-data-fragment"
|
||||
|
@ -240,15 +239,15 @@
|
|||
"SELECT id, file_id, media_id, thumbnail_id, deleted_at
|
||||
FROM file_media_object
|
||||
WHERE deleted_at IS NOT NULL
|
||||
AND deleted_at < now() - ?::interval
|
||||
AND deleted_at < now() + ?::interval
|
||||
ORDER BY deleted_at ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE
|
||||
SKIP LOCKED")
|
||||
|
||||
(defn- delete-file-media-objects!
|
||||
[{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-file-media-objects min-age chunk-size] {:fetch-size 5})
|
||||
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-file-media-objects deletion-threshold chunk-size] {:fetch-size 5})
|
||||
(reduce (fn [total {:keys [id file-id deleted-at] :as fmo}]
|
||||
(l/trc :hint "permanently delete"
|
||||
:rel "file-media-object"
|
||||
|
@ -269,15 +268,15 @@
|
|||
"SELECT id, file_id, deleted_at, data_backend, data_ref_id
|
||||
FROM file_change
|
||||
WHERE deleted_at IS NOT NULL
|
||||
AND deleted_at < now() - ?::interval
|
||||
AND deleted_at < now() + ?::interval
|
||||
ORDER BY deleted_at ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE
|
||||
SKIP LOCKED")
|
||||
|
||||
(defn- delete-file-changes!
|
||||
[{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-file-change min-age chunk-size] {:fetch-size 5})
|
||||
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-file-change deletion-threshold chunk-size] {:fetch-size 5})
|
||||
(reduce (fn [total {:keys [id file-id deleted-at] :as xlog}]
|
||||
(l/trc :hint "permanently delete"
|
||||
:rel "file-change"
|
||||
|
@ -324,16 +323,13 @@
|
|||
|
||||
(defmethod ig/expand-key ::handler
|
||||
[k v]
|
||||
{k (assoc v
|
||||
::min-age (cf/get-deletion-delay)
|
||||
::chunk-size 100)})
|
||||
{k (assoc v ::chunk-size 100)})
|
||||
|
||||
(defmethod ig/init-key ::handler
|
||||
[_ cfg]
|
||||
(fn [{:keys [props] :as task}]
|
||||
(let [min-age (dt/duration (or (:min-age props) (::min-age cfg)))
|
||||
cfg (assoc cfg ::min-age (db/interval min-age))]
|
||||
|
||||
(let [threshold (dt/duration (get props :deletion-threshold 0))
|
||||
cfg (assoc cfg ::deletion-threshold (db/interval threshold))]
|
||||
(loop [procs (map deref deletion-proc-vars)
|
||||
total 0]
|
||||
(if-let [proc-fn (first procs)]
|
||||
|
|
|
@ -222,7 +222,7 @@
|
|||
([params]
|
||||
(mark-file-deleted* *system* params))
|
||||
([conn {:keys [id] :as params}]
|
||||
(#'files/mark-file-deleted conn id)))
|
||||
(#'files/mark-file-deleted conn {} id)))
|
||||
|
||||
(defn create-team*
|
||||
([i params] (create-team* *system* i params))
|
||||
|
|
|
@ -8,10 +8,10 @@
|
|||
(:require
|
||||
[app.common.features :as cfeat]
|
||||
[app.common.pprint :as pp]
|
||||
[app.common.pprint :as pp]
|
||||
[app.common.thumbnails :as thc]
|
||||
[app.common.types.shape :as cts]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.db.sql :as sql]
|
||||
[app.http :as http]
|
||||
|
@ -123,8 +123,27 @@
|
|||
:components-v2 true}
|
||||
out (th/command! data)]
|
||||
|
||||
;; (th/print-result! out)
|
||||
|
||||
;; (th/print-result! out)
|
||||
(t/is (nil? (:error out)))
|
||||
|
||||
(let [result (:result out)]
|
||||
(t/is (some? (:deleted-at result)))
|
||||
(t/is (= file-id (:id result)))
|
||||
(t/is (= "new name" (:name result)))
|
||||
(t/is (= 1 (count (get-in result [:data :pages]))))
|
||||
(t/is (nil? (:users result))))))
|
||||
|
||||
(th/db-update! :file
|
||||
{:deleted-at (dt/now)}
|
||||
{:id file-id})
|
||||
|
||||
(t/testing "query single file after delete and wait"
|
||||
(let [data {::th/type :get-file
|
||||
::rpc/profile-id (:id prof)
|
||||
:id file-id
|
||||
:components-v2 true}
|
||||
out (th/command! data)]
|
||||
(let [error (:error out)
|
||||
error-data (ex-data error)]
|
||||
(t/is (th/ex-info? error))
|
||||
|
@ -195,7 +214,7 @@
|
|||
(t/is (= 5 (count rows))))
|
||||
|
||||
;; The objects-gc should remove unused fragments
|
||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
(t/is (= 3 (:processed res))))
|
||||
|
||||
;; Check the number of fragments
|
||||
|
@ -230,7 +249,7 @@
|
|||
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
|
||||
|
||||
;; The objects-gc should remove unused fragments
|
||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
(t/is (= 3 (:processed res))))
|
||||
|
||||
;; Check the number of fragments;
|
||||
|
@ -254,7 +273,7 @@
|
|||
(t/is (= 4 (count rows)))
|
||||
(t/is (= 2 (count (remove (comp some? :deleted-at) rows)))))
|
||||
|
||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
(t/is (= 2 (:processed res))))
|
||||
|
||||
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)})]
|
||||
|
@ -355,7 +374,7 @@
|
|||
(t/is (= 2 (count rows)))
|
||||
(t/is (= 1 (count (remove (comp some? :deleted-at) rows)))))
|
||||
|
||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
(t/is (= 3 (:processed res))))
|
||||
|
||||
;; check file media objects
|
||||
|
@ -386,7 +405,7 @@
|
|||
|
||||
;; This only clears fragments, the file media objects still referenced because
|
||||
;; snapshots are preserved
|
||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
(t/is (= 2 (:processed res))))
|
||||
|
||||
;; Mark all snapshots to be a non-snapshot file change
|
||||
|
@ -395,7 +414,7 @@
|
|||
|
||||
;; Rerun the file-gc and objects-gc
|
||||
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
|
||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
(t/is (= 2 (:processed res))))
|
||||
|
||||
;; Now that file-gc have deleted the file-media-object usage,
|
||||
|
@ -508,7 +527,7 @@
|
|||
;; run the task again
|
||||
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
|
||||
|
||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
(t/is (= 2 (:processed res))))
|
||||
|
||||
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)
|
||||
|
@ -550,7 +569,7 @@
|
|||
|
||||
;; This only removes unused fragments, file media are still
|
||||
;; referenced on snapshots.
|
||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
(t/is (= 2 (:processed res))))
|
||||
|
||||
;; Mark all snapshots to be a non-snapshot file change
|
||||
|
@ -560,7 +579,7 @@
|
|||
;; Rerun file-gc and objects-gc task for the same file once all snapshots are
|
||||
;; "expired/deleted"
|
||||
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
|
||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
(t/is (= 6 (:processed res))))
|
||||
|
||||
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)
|
||||
|
@ -712,7 +731,7 @@
|
|||
;; Now that file-gc have marked for deletion the object
|
||||
;; thumbnail lets execute the objects-gc task which remove
|
||||
;; the rows and mark as touched the storage object rows
|
||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
(t/is (= 5 (:processed res))))
|
||||
|
||||
;; Now that objects-gc have deleted the object thumbnail lets
|
||||
|
@ -741,7 +760,7 @@
|
|||
(t/is (= 1 (count rows)))
|
||||
(t/is (= 0 (count (remove (comp some? :deleted-at) rows)))))
|
||||
|
||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
;; (pp/pprint res)
|
||||
(t/is (= 3 (:processed res))))
|
||||
|
||||
|
@ -876,7 +895,7 @@
|
|||
:profile-id (:id profile1)})]
|
||||
;; file is not deleted because it does not meet all
|
||||
;; conditions to be deleted.
|
||||
(let [result (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [result (th/run-task! :objects-gc {})]
|
||||
(t/is (= 0 (:processed result))))
|
||||
|
||||
;; query the list of files
|
||||
|
@ -907,7 +926,7 @@
|
|||
(t/is (= 0 (count result)))))
|
||||
|
||||
;; run permanent deletion (should be noop)
|
||||
(let [result (th/run-task! :objects-gc {:min-age (dt/duration {:minutes 1})})]
|
||||
(let [result (th/run-task! :objects-gc {})]
|
||||
(t/is (= 0 (:processed result))))
|
||||
|
||||
;; query the list of file libraries of a after hard deletion
|
||||
|
@ -921,7 +940,7 @@
|
|||
(t/is (= 0 (count result)))))
|
||||
|
||||
;; run permanent deletion
|
||||
(let [result (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
|
||||
(t/is (= 1 (:processed result))))
|
||||
|
||||
;; query the list of file libraries of a after hard deletion
|
||||
|
@ -1176,7 +1195,7 @@
|
|||
(t/is (= 2 (count rows)))
|
||||
(t/is (= 1 (count (remove :deleted-at rows)))))
|
||||
|
||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
(t/is (= 4 (:processed res))))
|
||||
|
||||
(let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})]
|
||||
|
@ -1232,7 +1251,7 @@
|
|||
(t/is (= 2 (count rows)))
|
||||
(t/is (= 1 (count (remove (comp some? :deleted-at) rows)))))
|
||||
|
||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
(t/is (= 2 (:processed res))))
|
||||
|
||||
(let [rows (th/db-query :file-thumbnail {:file-id (:id file)})]
|
||||
|
@ -1251,7 +1270,7 @@
|
|||
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
|
||||
|
||||
;; Preventive objects-gc
|
||||
(let [result (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [result (th/run-task! :objects-gc {})]
|
||||
(t/is (= 1 (:processed result))))
|
||||
|
||||
;; Check the number of fragments before adding the page
|
||||
|
@ -1272,7 +1291,7 @@
|
|||
(th/run-pending-tasks!))
|
||||
|
||||
;; Clean objects after file-gc
|
||||
(let [result (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [result (th/run-task! :objects-gc {})]
|
||||
(t/is (= 1 (:processed result))))
|
||||
|
||||
;; Check the number of fragments before adding the page
|
||||
|
@ -1324,7 +1343,7 @@
|
|||
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
|
||||
|
||||
;; The objects-gc should remove unused fragments
|
||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
(t/is (= 2 (:processed res))))
|
||||
|
||||
;; Check the number of fragments before adding the page
|
||||
|
@ -1821,8 +1840,7 @@
|
|||
(t/is (= (:id file-2) (:file-id (get rows 1))))
|
||||
(t/is (nil? (:deleted-at (get rows 1)))))
|
||||
|
||||
(th/run-task! :objects-gc
|
||||
{:min-age 0})
|
||||
(th/run-task! :objects-gc {})
|
||||
|
||||
(let [rows (th/db-exec! ["SELECT * FROM file_media_object ORDER BY created_at ASC"])]
|
||||
(t/is (= 1 (count rows)))
|
||||
|
|
|
@ -7,6 +7,7 @@
|
|||
(ns backend-tests.rpc-font-test
|
||||
(:require
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.http :as http]
|
||||
[app.rpc :as-alias rpc]
|
||||
|
@ -144,7 +145,7 @@
|
|||
(t/is (= 0 (:freeze res)))
|
||||
(t/is (= 0 (:delete res))))
|
||||
|
||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
|
||||
(t/is (= 2 (:processed res))))
|
||||
|
||||
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
|
||||
|
@ -204,7 +205,7 @@
|
|||
(t/is (= 0 (:freeze res)))
|
||||
(t/is (= 0 (:delete res))))
|
||||
|
||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
|
||||
(t/is (= 1 (:processed res))))
|
||||
|
||||
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
|
||||
|
@ -263,7 +264,7 @@
|
|||
(t/is (= 0 (:freeze res)))
|
||||
(t/is (= 0 (:delete res))))
|
||||
|
||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
|
||||
(t/is (= 1 (:processed res))))
|
||||
|
||||
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
|
||||
|
|
|
@ -209,16 +209,16 @@
|
|||
::rpc/profile-id (:id prof1)
|
||||
:id (:id team1)}
|
||||
out (th/command! params)]
|
||||
;; (th/print-result! out)
|
||||
;; (th/print-result! out)
|
||||
|
||||
(let [team (th/db-get :team {:id (:id team1)} {::db/remove-deleted false})]
|
||||
(t/is (dt/instant? (:deleted-at team)))))
|
||||
|
||||
;; Request profile to be deleted
|
||||
;; Request profile to be deleted
|
||||
(let [params {::th/type :delete-profile
|
||||
::rpc/profile-id (:id prof1)}
|
||||
out (th/command! params)]
|
||||
;; (th/print-result! out)
|
||||
;; (th/print-result! out)
|
||||
(t/is (nil? (:result out)))
|
||||
(t/is (nil? (:error out)))))))
|
||||
|
||||
|
|
|
@ -7,6 +7,7 @@
|
|||
(ns backend-tests.rpc-project-test
|
||||
(:require
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.http :as http]
|
||||
[app.rpc :as-alias rpc]
|
||||
|
@ -178,7 +179,7 @@
|
|||
|
||||
;; project is not deleted because it does not meet all
|
||||
;; conditions to be deleted.
|
||||
(let [result (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [result (th/run-task! :objects-gc {})]
|
||||
(t/is (= 0 (:processed result))))
|
||||
|
||||
;; query the list of projects
|
||||
|
@ -210,7 +211,7 @@
|
|||
(t/is (= 1 (count result)))))
|
||||
|
||||
;; run permanent deletion (should be noop)
|
||||
(let [result (th/run-task! :objects-gc {:min-age (dt/duration {:minutes 1})})]
|
||||
(let [result (th/run-task! :objects-gc {})]
|
||||
(t/is (= 0 (:processed result))))
|
||||
|
||||
;; query the list of files of a after soft deletion
|
||||
|
@ -224,7 +225,7 @@
|
|||
(t/is (= 0 (count result)))))
|
||||
|
||||
;; run permanent deletion
|
||||
(let [result (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
|
||||
(t/is (= 1 (:processed result))))
|
||||
|
||||
;; query the list of files of a after hard deletion
|
||||
|
|
|
@ -8,6 +8,7 @@
|
|||
(:require
|
||||
[app.common.logging :as l]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.http :as http]
|
||||
[app.rpc :as-alias rpc]
|
||||
|
@ -459,7 +460,7 @@
|
|||
|
||||
;; team is not deleted because it does not meet all
|
||||
;; conditions to be deleted.
|
||||
(let [result (th/run-task! :objects-gc {:min-age (dt/duration 0)})]
|
||||
(let [result (th/run-task! :objects-gc {})]
|
||||
(t/is (= 0 (:processed result))))
|
||||
|
||||
;; query the list of teams
|
||||
|
@ -493,7 +494,7 @@
|
|||
(th/run-pending-tasks!)
|
||||
|
||||
;; run permanent deletion (should be noop)
|
||||
(let [result (th/run-task! :objects-gc {:min-age (dt/duration {:minutes 1})})]
|
||||
(let [result (th/run-task! :objects-gc {})]
|
||||
(t/is (= 0 (:processed result))))
|
||||
|
||||
;; query the list of projects after hard deletion
|
||||
|
@ -507,7 +508,7 @@
|
|||
(t/is (= :not-found (:type edata)))))
|
||||
|
||||
;; run permanent deletion
|
||||
(let [result (th/run-task! :objects-gc {:min-age (dt/duration 0)})]
|
||||
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
|
||||
(t/is (= 2 (:processed result))))
|
||||
|
||||
;; query the list of projects of a after hard deletion
|
||||
|
@ -521,7 +522,6 @@
|
|||
(let [edata (-> out :error ex-data)]
|
||||
(t/is (= :not-found (:type edata)))))))
|
||||
|
||||
|
||||
(t/deftest team-deletion-2
|
||||
(let [storage (-> (:app.storage/storage th/*system*)
|
||||
(assoc ::sto/backend :assets-fs))
|
||||
|
@ -564,7 +564,7 @@
|
|||
(t/is (= 1 (count rows)))
|
||||
(t/is (dt/instant? (:deleted-at (first rows)))))
|
||||
|
||||
(let [result (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
|
||||
(t/is (= 5 (:processed result))))))
|
||||
|
||||
(t/deftest create-team-access-request
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue