3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-06-19 01:21:39 +02:00
Code Issues Projects Releases Packages Wiki Activity

♻️ Normalize logical deletion to future dates

Browse source 
Instead of managing the ...
This commit is contained in:
Andrey Antukh 2025-05-20 13:03:14 +02:00
parent a3b4fc9545
commit a706907b26
14 changed files with 243 additions and 166 deletions
Download patch file Download diff file Expand all files Collapse all files

31
backend/src/app/features/logical_deletion.clj Normal file
View file

@ -0,0 +1,31 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC
(ns app.features.logical-deletion
"A code related to handle logical deletion mechanism"
(:require
[app.config :as cf]
[app.util.time :as dt]))
(defn get-deletion-delay
"Calculate the next deleted-at for a resource (file, team, etc) in function
of team settings"
[team]
(if-let [subscription (get team :subscription)]
(cond
(and (= (:type subscription) "unlimited")
(= (:status subscription) "active"))
(dt/duration {:days 30})
(and (= (:type subscription) "enterprise")
(= (:status subscription) "active"))
(dt/duration {:days 90})
:else
(cf/get-deletion-delay))
(cf/get-deletion-delay)))

14
backend/src/app/rpc/commands/files.clj
View file

@ -23,6 +23,7 @@
[app.db.sql :as-alias sql]
[app.features.fdata :as feat.fdata]
[app.features.file-migrations :as feat.fmigr]
[app.features.logical-deletion :as ldel]
[app.loggers.audit :as-alias audit]
[app.loggers.webhooks :as-alias webhooks]
[app.rpc :as-alias rpc]
@ -970,9 +971,10 @@
;; --- MUTATION COMMAND: delete-file
(defn- mark-file-deleted
[conn file-id]
(let [file (db/update! conn :file
{:deleted-at (dt/now)}
[conn team file-id]
(let [delay (ldel/get-deletion-delay team)
file (db/update! conn :file
{:deleted-at (dt/in-future delay)}
{:id file-id}
{::db/return-keys [:id :name :is-shared :deleted-at
:project-id :created-at :modified-at]})]
@ -991,7 +993,11 @@
(defn- delete-file
[{:keys [::db/conn] :as cfg} {:keys [profile-id id] :as params}]
(check-edition-permissions! conn profile-id id)
(let [file (mark-file-deleted conn id)]
(let [team (teams/get-team conn
:profile-id profile-id
:file-id id)
file (mark-file-deleted conn team id)]
(rph/with-meta (rph/wrap)
{::audit/props {:project-id (:project-id file)
:name (:name file)

5
backend/src/app/rpc/commands/files_update.clj
View file

@ -20,6 +20,7 @@
[app.db :as db]
[app.features.fdata :as feat.fdata]
[app.features.file-migrations :as feat.fmigr]
[app.features.logical-deletion :as ldel]
[app.http.errors :as errors]
[app.loggers.audit :as audit]
[app.loggers.webhooks :as webhooks]
@ -209,7 +210,7 @@
Only intended for internal use on this module."
[{:keys [::db/conn ::wrk/executor ::timestamp] :as cfg}
{:keys [profile-id file features changes session-id skip-validate] :as params}]
{:keys [profile-id file team features changes session-id skip-validate] :as params}]
(let [;; Retrieve the file data
file (feat.fmigr/resolve-applied-migrations cfg file)
@ -243,7 +244,7 @@
:created-at timestamp
:updated-at timestamp
:deleted-at (if (::snapshot-data file)
(dt/plus timestamp (cf/get-deletion-delay))
(dt/plus timestamp (ldel/get-deletion-delay team))
(dt/plus timestamp (dt/duration {:hours 1})))
:file-id (:id file)
:revn (:revn file)

51
backend/src/app/rpc/commands/fonts.clj
View file

@ -12,6 +12,7 @@
[app.common.uuid :as uuid]
[app.db :as db]
[app.db.sql :as-alias sql]
[app.features.logical-deletion :as ldel]
[app.loggers.audit :as-alias audit]
[app.loggers.webhooks :as-alias webhooks]
[app.media :as media]
@ -202,32 +203,40 @@
(sv/defmethod ::delete-font
{::doc/added "1.18"
::webhooks/event? true
::sm/params schema:delete-font}
[cfg {:keys [::rpc/profile-id id team-id]}]
(db/tx-run! cfg
(fn [{:keys [::db/conn] :as cfg}]
(teams/check-edition-permissions! conn profile-id team-id)
(let [fonts (db/query conn :team-font-variant
::sm/params schema:delete-font
::db/transaction true}
[{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id team-id]}]
(let [team (teams/get-team conn
:profile-id profile-id
:team-id team-id)
fonts (db/query conn :team-font-variant
{:team-id team-id
:font-id id
:deleted-at nil}
{::sql/for-update true})
tnow (dt/now)]
delay (ldel/get-deletion-delay team)
tnow (dt/in-future delay)]
(teams/check-edition-permissions! (:permissions team))
(when-not (seq fonts)
(ex/raise :type :not-found
:code :object-not-found))
(doseq [font fonts]
(db/update! conn :team-font-variant
{:deleted-at tnow}
{:id (:id font)}))
{:id (:id font)}
{::db/return-keys false}))
(rph/with-meta (rph/wrap)
{::audit/props {:id id
:team-id team-id
:name (:font-family (peek fonts))
:profile-id profile-id}})))))
:profile-id profile-id}})))
;; --- DELETE FONT VARIANT
@ -239,19 +248,23 @@
(sv/defmethod ::delete-font-variant
{::doc/added "1.18"
::webhooks/event? true
::sm/params schema:delete-font-variant}
[cfg {:keys [::rpc/profile-id id team-id]}]
(db/tx-run! cfg
(fn [{:keys [::db/conn] :as cfg}]
(teams/check-edition-permissions! conn profile-id team-id)
(let [variant (db/get conn :team-font-variant
::sm/params schema:delete-font-variant
::db/transaction true}
[{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id team-id]}]
(let [team (teams/get-team conn
:profile-id profile-id
:team-id team-id)
variant (db/get conn :team-font-variant
{:id id :team-id team-id}
{::sql/for-update true})]
{::sql/for-update true})
delay (ldel/get-deletion-delay team)]
(teams/check-edition-permissions! (:permissions team))
(db/update! conn :team-font-variant
{:deleted-at (dt/now)}
{:id (:id variant)})
{:deleted-at (dt/in-future delay)}
{:id (:id variant)}
{::db/return-keys false})
(rph/with-meta (rph/wrap)
{::audit/props {:font-family (:font-family variant)
:font-id (:font-id variant)}})))))
:font-id (:font-id variant)}})))

3
backend/src/app/rpc/commands/profile.clj
View file

@ -480,8 +480,7 @@
JOIN team AS t ON (t.id = tpr.team_id)
WHERE tpr.is_owner IS TRUE
AND tpr.profile_id = ?
AND (t.deleted_at IS NULL OR
t.deleted_at > now())
AND t.deleted_at IS NULL
)
SELECT tpr.team_id AS id,
count(tpr.profile_id) - 1 AS participants

14
backend/src/app/rpc/commands/projects.clj
View file

@ -11,6 +11,7 @@
[app.common.schema :as sm]
[app.db :as db]
[app.db.sql :as-alias sql]
[app.features.logical-deletion :as ldel]
[app.loggers.audit :as-alias audit]
[app.loggers.webhooks :as webhooks]
[app.rpc :as-alias rpc]
@ -253,9 +254,10 @@
;; --- MUTATION: Delete Project
(defn- delete-project
[conn project-id]
(let [project (db/update! conn :project
{:deleted-at (dt/now)}
[conn team project-id]
(let [delay (ldel/get-deletion-delay team)
project (db/update! conn :project
{:deleted-at (dt/in-future delay)}
{:id project-id}
{::db/return-keys true})]
@ -272,7 +274,6 @@
project))
(def ^:private schema:delete-project
[:map {:title "delete-project"}
[:id ::sm/uuid]])
@ -284,7 +285,10 @@
::db/transaction true}
[{:keys [::db/conn]} {:keys [::rpc/profile-id id] :as params}]
(check-edition-permissions! conn profile-id id)
(let [project (delete-project conn id)]
(let [team (teams/get-team conn
:profile-id profile-id
:project-id id)
project (delete-project conn team id)]
(rph/with-meta (rph/wrap)
{::audit/props {:team-id (:team-id project)
:name (:name project)

49
backend/src/app/rpc/commands/teams.clj
View file

@ -17,6 +17,7 @@
[app.db :as db]
[app.db.sql :as sql]
[app.email :as eml]
[app.features.logical-deletion :as ldel]
[app.loggers.audit :as audit]
[app.main :as-alias main]
[app.media :as media]
@ -214,31 +215,35 @@
(defn get-team
[conn & {:keys [profile-id team-id project-id file-id] :as params}]
(dm/assert!
"connection or pool is mandatory"
(or (db/connection? conn)
(db/pool? conn)))
(assert (uuid? profile-id) "profile-id is mandatory")
(assert (or (db/connection? conn)
(db/pool? conn))
"connection or pool is mandatory")
(dm/assert!
"profile-id is mandatory"
(uuid? profile-id))
(let [{:keys [default-team-id] :as profile}
(profile/get-profile conn profile-id)
(let [{:keys [default-team-id] :as profile} (profile/get-profile conn profile-id)
result (cond
sql
(if (contains? cf/flags :subscriptions)
sql:get-teams-with-permissions-and-subscription
sql:get-teams-with-permissions)
result
(cond
(some? team-id)
(let [sql (str "WITH teams AS (" sql:get-teams-with-permissions
") SELECT * FROM teams WHERE id=?")]
(let [sql (str "WITH teams AS (" sql ") "
"SELECT * FROM teams WHERE id=?")]
(db/exec-one! conn [sql default-team-id profile-id team-id]))
(some? project-id)
(let [sql (str "WITH teams AS (" sql:get-teams-with-permissions ") "
(let [sql (str "WITH teams AS (" sql ") "
"SELECT t.* FROM teams AS t "
" JOIN project AS p ON (p.team_id = t.id) "
" WHERE p.id=?")]
(db/exec-one! conn [sql default-team-id profile-id project-id]))
(some? file-id)
(let [sql (str "WITH teams AS (" sql:get-teams-with-permissions ") "
(let [sql (str "WITH teams AS (" sql ") "
"SELECT t.* FROM teams AS t "
" JOIN project AS p ON (p.team_id = t.id) "
" JOIN file AS f ON (f.project_id = p.id) "
@ -634,12 +639,12 @@
(defn- delete-team
"Mark a team for deletion"
[conn team-id]
[conn {:keys [id] :as team}]
(let [deleted-at (dt/now)
(let [delay (ldel/get-deletion-delay team)
team (db/update! conn :team
{:deleted-at deleted-at}
{:id team-id}
{:deleted-at (dt/in-future delay)}
{:id id}
{::db/return-keys true})]
(when (:is-default team)
@ -650,8 +655,8 @@
(wrk/submit! {::db/conn conn
::wrk/task :delete-object
::wrk/params {:object :team
:deleted-at deleted-at
:id team-id}})
:deleted-at (:deleted-at team)
:id id}})
team))
(def ^:private schema:delete-team
@ -663,12 +668,14 @@
::sm/params schema:delete-team
::db/transaction true}
[{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id] :as params}]
(let [perms (get-permissions conn profile-id id)]
(let [team (get-team conn :profile-id profile-id :team-id id)
perms (get team :permissions)]
(when-not (:is-owner perms)
(ex/raise :type :validation
:code :only-owner-can-delete-team))
(delete-team conn id)
(delete-team conn team)
nil))
;; --- Mutation: Team Update Role

70
backend/src/app/tasks/objects_gc.clj
View file

@ -9,7 +9,6 @@
of deleted or unreachable objects."
(:require
[app.common.logging :as l]
[app.config :as cf]
[app.db :as db]
[app.storage :as sto]
[app.util.time :as dt]
@ -18,15 +17,15 @@
(def ^:private sql:get-profiles
"SELECT id, photo_id FROM profile
WHERE deleted_at IS NOT NULL
AND deleted_at < now() - ?::interval
AND deleted_at < now() + ?::interval
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-profiles!
[{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-profiles min-age chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-profiles deletion-threshold chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id photo-id]}]
(l/trc :hint "permanently delete" :rel "profile" :id (str id))
@ -41,15 +40,15 @@
(def ^:private sql:get-teams
"SELECT deleted_at, id, photo_id FROM team
WHERE deleted_at IS NOT NULL
AND deleted_at < now() - ?::interval
AND deleted_at < now() + ?::interval
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-teams!
[{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-teams min-age chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-teams deletion-threshold chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id photo-id deleted-at]}]
(l/trc :hint "permanently delete"
:rel "team"
@ -69,15 +68,15 @@
"SELECT id, team_id, deleted_at, woff1_file_id, woff2_file_id, otf_file_id, ttf_file_id
FROM team_font_variant
WHERE deleted_at IS NOT NULL
AND deleted_at < now() - ?::interval
AND deleted_at < now() + ?::interval
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-fonts!
[{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-fonts min-age chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-fonts deletion-threshold chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id team-id deleted-at] :as font}]
(l/trc :hint "permanently delete"
:rel "team-font-variant"
@ -101,15 +100,15 @@
"SELECT id, deleted_at, team_id
FROM project
WHERE deleted_at IS NOT NULL
AND deleted_at < now() - ?::interval
AND deleted_at < now() + ?::interval
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-projects!
[{:keys [::db/conn ::min-age ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-projects min-age chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-projects deletion-threshold chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id team-id deleted-at]}]
(l/trc :hint "permanently delete"
:rel "project"
@ -127,15 +126,15 @@
"SELECT id, deleted_at, project_id, data_backend, data_ref_id
FROM file
WHERE deleted_at IS NOT NULL
AND deleted_at < now() - ?::interval
AND deleted_at < now() + ?::interval
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-files!
[{:keys [::db/conn ::sto/storage ::min-age ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-files min-age chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::sto/storage ::deletion-threshold ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-files deletion-threshold chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id deleted-at project-id] :as file}]
(l/trc :hint "permanently delete"
:rel "file"
@ -156,15 +155,15 @@
"SELECT file_id, revn, media_id, deleted_at
FROM file_thumbnail
WHERE deleted_at IS NOT NULL
AND deleted_at < now() - ?::interval
AND deleted_at < now() + ?::interval
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn delete-file-thumbnails!
[{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-thumbnails min-age chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-thumbnails deletion-threshold chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [file-id revn media-id deleted-at]}]
(l/trc :hint "permanently delete"
:rel "file-thumbnail"
@ -185,15 +184,15 @@
"SELECT file_id, object_id, media_id, deleted_at
FROM file_tagged_object_thumbnail
WHERE deleted_at IS NOT NULL
AND deleted_at < now() - ?::interval
AND deleted_at < now() + ?::interval
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn delete-file-object-thumbnails!
[{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-object-thumbnails min-age chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-object-thumbnails deletion-threshold chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [file-id object-id media-id deleted-at]}]
(l/trc :hint "permanently delete"
:rel "file-tagged-object-thumbnail"
@ -214,15 +213,15 @@
"SELECT file_id, id, deleted_at, data_ref_id
FROM file_data_fragment
WHERE deleted_at IS NOT NULL
AND deleted_at < now() - ?::interval
AND deleted_at < now() + ?::interval
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-file-data-fragments!
[{:keys [::db/conn ::sto/storage ::min-age ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-file-data-fragments min-age chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::sto/storage ::deletion-threshold ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-file-data-fragments deletion-threshold chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [file-id id deleted-at data-ref-id]}]
(l/trc :hint "permanently delete"
:rel "file-data-fragment"
@ -240,15 +239,15 @@
"SELECT id, file_id, media_id, thumbnail_id, deleted_at
FROM file_media_object
WHERE deleted_at IS NOT NULL
AND deleted_at < now() - ?::interval
AND deleted_at < now() + ?::interval
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-file-media-objects!
[{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-media-objects min-age chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-media-objects deletion-threshold chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id file-id deleted-at] :as fmo}]
(l/trc :hint "permanently delete"
:rel "file-media-object"
@ -269,15 +268,15 @@
"SELECT id, file_id, deleted_at, data_backend, data_ref_id
FROM file_change
WHERE deleted_at IS NOT NULL
AND deleted_at < now() - ?::interval
AND deleted_at < now() + ?::interval
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-file-changes!
[{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-change min-age chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-change deletion-threshold chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id file-id deleted-at] :as xlog}]
(l/trc :hint "permanently delete"
:rel "file-change"
@ -324,16 +323,13 @@
(defmethod ig/expand-key ::handler
[k v]
{k (assoc v
::min-age (cf/get-deletion-delay)
::chunk-size 100)})
{k (assoc v ::chunk-size 100)})
(defmethod ig/init-key ::handler
[_ cfg]
(fn [{:keys [props] :as task}]
(let [min-age (dt/duration (or (:min-age props) (::min-age cfg)))
cfg (assoc cfg ::min-age (db/interval min-age))]
(let [threshold (dt/duration (get props :deletion-threshold 0))
cfg (assoc cfg ::deletion-threshold (db/interval threshold))]
(loop [procs (map deref deletion-proc-vars)
total 0]
(if-let [proc-fn (first procs)]

2
backend/test/backend_tests/helpers.clj
View file

@ -222,7 +222,7 @@
([params]
(mark-file-deleted* *system* params))
([conn {:keys [id] :as params}]
(#'files/mark-file-deleted conn id)))
(#'files/mark-file-deleted conn {} id)))
(defn create-team*
([i params] (create-team* *system* i params))

64
backend/test/backend_tests/rpc_file_test.clj
View file

@ -8,10 +8,10 @@
(:require
[app.common.features :as cfeat]
[app.common.pprint :as pp]
[app.common.pprint :as pp]
[app.common.thumbnails :as thc]
[app.common.types.shape :as cts]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.db.sql :as sql]
[app.http :as http]
@ -123,8 +123,27 @@
:components-v2 true}
out (th/command! data)]
;; (th/print-result! out)
;; (th/print-result! out)
(t/is (nil? (:error out)))
(let [result (:result out)]
(t/is (some? (:deleted-at result)))
(t/is (= file-id (:id result)))
(t/is (= "new name" (:name result)))
(t/is (= 1 (count (get-in result [:data :pages]))))
(t/is (nil? (:users result))))))
(th/db-update! :file
{:deleted-at (dt/now)}
{:id file-id})
(t/testing "query single file after delete and wait"
(let [data {::th/type :get-file
::rpc/profile-id (:id prof)
:id file-id
:components-v2 true}
out (th/command! data)]
(let [error (:error out)
error-data (ex-data error)]
(t/is (th/ex-info? error))
@ -195,7 +214,7 @@
(t/is (= 5 (count rows))))
;; The objects-gc should remove unused fragments
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 3 (:processed res))))
;; Check the number of fragments
@ -230,7 +249,7 @@
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
;; The objects-gc should remove unused fragments
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 3 (:processed res))))
;; Check the number of fragments;
@ -254,7 +273,7 @@
(t/is (= 4 (count rows)))
(t/is (= 2 (count (remove (comp some? :deleted-at) rows)))))
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 2 (:processed res))))
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)})]
@ -355,7 +374,7 @@
(t/is (= 2 (count rows)))
(t/is (= 1 (count (remove (comp some? :deleted-at) rows)))))
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 3 (:processed res))))
;; check file media objects
@ -386,7 +405,7 @@
;; This only clears fragments, the file media objects still referenced because
;; snapshots are preserved
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 2 (:processed res))))
;; Mark all snapshots to be a non-snapshot file change
@ -395,7 +414,7 @@
;; Rerun the file-gc and objects-gc
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 2 (:processed res))))
;; Now that file-gc have deleted the file-media-object usage,
@ -508,7 +527,7 @@
;; run the task again
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 2 (:processed res))))
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)
@ -550,7 +569,7 @@
;; This only removes unused fragments, file media are still
;; referenced on snapshots.
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 2 (:processed res))))
;; Mark all snapshots to be a non-snapshot file change
@ -560,7 +579,7 @@
;; Rerun file-gc and objects-gc task for the same file once all snapshots are
;; "expired/deleted"
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 6 (:processed res))))
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)
@ -712,7 +731,7 @@
;; Now that file-gc have marked for deletion the object
;; thumbnail lets execute the objects-gc task which remove
;; the rows and mark as touched the storage object rows
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 5 (:processed res))))
;; Now that objects-gc have deleted the object thumbnail lets
@ -741,7 +760,7 @@
(t/is (= 1 (count rows)))
(t/is (= 0 (count (remove (comp some? :deleted-at) rows)))))
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
;; (pp/pprint res)
(t/is (= 3 (:processed res))))
@ -876,7 +895,7 @@
:profile-id (:id profile1)})]
;; file is not deleted because it does not meet all
;; conditions to be deleted.
(let [result (th/run-task! :objects-gc {:min-age 0})]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 0 (:processed result))))
;; query the list of files
@ -907,7 +926,7 @@
(t/is (= 0 (count result)))))
;; run permanent deletion (should be noop)
(let [result (th/run-task! :objects-gc {:min-age (dt/duration {:minutes 1})})]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 0 (:processed result))))
;; query the list of file libraries of a after hard deletion
@ -921,7 +940,7 @@
(t/is (= 0 (count result)))))
;; run permanent deletion
(let [result (th/run-task! :objects-gc {:min-age 0})]
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 1 (:processed result))))
;; query the list of file libraries of a after hard deletion
@ -1176,7 +1195,7 @@
(t/is (= 2 (count rows)))
(t/is (= 1 (count (remove :deleted-at rows)))))
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 4 (:processed res))))
(let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})]
@ -1232,7 +1251,7 @@
(t/is (= 2 (count rows)))
(t/is (= 1 (count (remove (comp some? :deleted-at) rows)))))
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 2 (:processed res))))
(let [rows (th/db-query :file-thumbnail {:file-id (:id file)})]
@ -1251,7 +1270,7 @@
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
;; Preventive objects-gc
(let [result (th/run-task! :objects-gc {:min-age 0})]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 1 (:processed result))))
;; Check the number of fragments before adding the page
@ -1272,7 +1291,7 @@
(th/run-pending-tasks!))
;; Clean objects after file-gc
(let [result (th/run-task! :objects-gc {:min-age 0})]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 1 (:processed result))))
;; Check the number of fragments before adding the page
@ -1324,7 +1343,7 @@
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
;; The objects-gc should remove unused fragments
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 2 (:processed res))))
;; Check the number of fragments before adding the page
@ -1821,8 +1840,7 @@
(t/is (= (:id file-2) (:file-id (get rows 1))))
(t/is (nil? (:deleted-at (get rows 1)))))
(th/run-task! :objects-gc
{:min-age 0})
(th/run-task! :objects-gc {})
(let [rows (th/db-exec! ["SELECT * FROM file_media_object ORDER BY created_at ASC"])]
(t/is (= 1 (count rows)))

7
backend/test/backend_tests/rpc_font_test.clj
View file

@ -7,6 +7,7 @@
(ns backend-tests.rpc-font-test
(:require
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.http :as http]
[app.rpc :as-alias rpc]
@ -144,7 +145,7 @@
(t/is (= 0 (:freeze res)))
(t/is (= 0 (:delete res))))
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 2 (:processed res))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
@ -204,7 +205,7 @@
(t/is (= 0 (:freeze res)))
(t/is (= 0 (:delete res))))
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 1 (:processed res))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
@ -263,7 +264,7 @@
(t/is (= 0 (:freeze res)))
(t/is (= 0 (:delete res))))
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 1 (:processed res))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]

7
backend/test/backend_tests/rpc_project_test.clj
View file

@ -7,6 +7,7 @@
(ns backend-tests.rpc-project-test
(:require
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.http :as http]
[app.rpc :as-alias rpc]
@ -178,7 +179,7 @@
;; project is not deleted because it does not meet all
;; conditions to be deleted.
(let [result (th/run-task! :objects-gc {:min-age 0})]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 0 (:processed result))))
;; query the list of projects
@ -210,7 +211,7 @@
(t/is (= 1 (count result)))))
;; run permanent deletion (should be noop)
(let [result (th/run-task! :objects-gc {:min-age (dt/duration {:minutes 1})})]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 0 (:processed result))))
;; query the list of files of a after soft deletion
@ -224,7 +225,7 @@
(t/is (= 0 (count result)))))
;; run permanent deletion
(let [result (th/run-task! :objects-gc {:min-age 0})]
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 1 (:processed result))))
;; query the list of files of a after hard deletion

10
backend/test/backend_tests/rpc_team_test.clj
View file

@ -8,6 +8,7 @@
(:require
[app.common.logging :as l]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.http :as http]
[app.rpc :as-alias rpc]
@ -459,7 +460,7 @@
;; team is not deleted because it does not meet all
;; conditions to be deleted.
(let [result (th/run-task! :objects-gc {:min-age (dt/duration 0)})]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 0 (:processed result))))
;; query the list of teams
@ -493,7 +494,7 @@
(th/run-pending-tasks!)
;; run permanent deletion (should be noop)
(let [result (th/run-task! :objects-gc {:min-age (dt/duration {:minutes 1})})]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 0 (:processed result))))
;; query the list of projects after hard deletion
@ -507,7 +508,7 @@
(t/is (= :not-found (:type edata)))))
;; run permanent deletion
(let [result (th/run-task! :objects-gc {:min-age (dt/duration 0)})]
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 2 (:processed result))))
;; query the list of projects of a after hard deletion
@ -521,7 +522,6 @@
(let [edata (-> out :error ex-data)]
(t/is (= :not-found (:type edata)))))))
(t/deftest team-deletion-2
(let [storage (-> (:app.storage/storage th/*system*)
(assoc ::sto/backend :assets-fs))
@ -564,7 +564,7 @@
(t/is (= 1 (count rows)))
(t/is (dt/instant? (:deleted-at (first rows)))))
(let [result (th/run-task! :objects-gc {:min-age 0})]
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 5 (:processed result))))))
(t/deftest create-team-access-request