From 71ba0242c708cedb0e62a1afe84eb81649bbdd16 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 2 Sep 2024 16:34:03 +0200 Subject: [PATCH 1/3] :bug: Add missing type decoding on changes schema --- common/src/app/common/files/changes.cljc | 17 +++++++++++------ common/src/app/common/types/page.cljc | 18 +++++++++++------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/common/src/app/common/files/changes.cljc b/common/src/app/common/files/changes.cljc index a46c02352..40a253759 100644 --- a/common/src/app/common/files/changes.cljc +++ b/common/src/app/common/files/changes.cljc @@ -31,8 +31,7 @@ ;; SCHEMAS ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(def ^:private - schema:operation +(def schema:operation [:multi {:dispatch :type :title "Operation" :decode/json #(update % :type keyword) @@ -61,9 +60,12 @@ [:type [:= :set-remote-synced]] [:remote-synced {:optional true} [:maybe :boolean]]]]]) -(sm/register! ::change +(def schema:change [:schema - [:multi {:dispatch :type :title "Change" ::smd/simplified true} + [:multi {:dispatch :type + :title "Change" + :decode/json #(update % :type keyword) + ::smd/simplified true} [:set-option [:map {:title "SetOptionChange"} [:type [:= :set-option]] @@ -256,8 +258,11 @@ [:type [:= :del-typography]] [:id ::sm/uuid]]]]]) -(sm/register! ::changes - [:sequential {:gen/max 2} ::change]) +(def schema:changes + [:sequential {:gen/max 5 :gen/min 1} schema:change]) + +(sm/register! ::change schema:change) +(sm/register! ::changes schema:changes) (def check-change! (sm/check-fn ::change)) diff --git a/common/src/app/common/types/page.cljc b/common/src/app/common/types/page.cljc index 1c3bef33b..b49755115 100644 --- a/common/src/app/common/types/page.cljc +++ b/common/src/app/common/types/page.cljc @@ -18,20 +18,20 @@ ;; SCHEMAS ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(sm/register! ::flow - [:map {:title "PageFlow"} +(def schema:flow + [:map {:title "Flow"} [:id ::sm/uuid] [:name :string] [:starting-frame ::sm/uuid]]) -(sm/register! ::guide - [:map {:title "PageGuide"} +(def schema:guide + [:map {:title "Guide"} [:id ::sm/uuid] [:axis [::sm/one-of #{:x :y}]] [:position ::sm/safe-number] [:frame-id {:optional true} [:maybe ::sm/uuid]]]) -(sm/register! ::page +(def schema:page [:map {:title "FilePage"} [:id ::sm/uuid] [:name :string] @@ -42,11 +42,15 @@ [:background {:optional true} ::ctc/rgb-color] [:saved-grids {:optional true} ::ctg/saved-grids] [:flows {:optional true} - [:vector {:gen/max 2} ::flow]] + [:vector {:gen/max 2} schema:flow]] [:guides {:optional true} - [:map-of {:gen/max 2} ::sm/uuid ::guide]] + [:map-of {:gen/max 2} ::sm/uuid schema:guide]] [:plugin-data {:optional true} ::ctpg/plugin-data]]]]) +(sm/register! ::page schema:page) +(sm/register! ::guide schema:guide) +(sm/register! ::flow schema:flow) + (def check-page-guide! (sm/check-fn ::guide)) From 50df2279a7b3258c25b52c85a39d55cb5781e469 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 2 Sep 2024 16:40:18 +0200 Subject: [PATCH 2/3] :bug: Make the media cleaning on file-gc task aware of snapshots It now takes in account the snapshots, and prevents deletion of media files used in snapshots. --- backend/src/app/migrations.clj | 5 +- .../sql/0130-mod-file-change-table.sql | 2 + .../src/app/rpc/commands/files_snapshot.clj | 5 +- backend/src/app/rpc/commands/files_update.clj | 1 + backend/src/app/tasks/file_gc.clj | 60 +++++++++++-------- backend/src/app/tasks/file_xlog_gc.clj | 5 +- frontend/src/debug.cljs | 6 +- 7 files changed, 53 insertions(+), 31 deletions(-) create mode 100644 backend/src/app/migrations/sql/0130-mod-file-change-table.sql diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index bc61a89d1..5226e5152 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -409,7 +409,10 @@ :fn (mg/resource "app/migrations/sql/0128-mod-task-table.sql")} {:name "0129-mod-file-change-table" - :fn (mg/resource "app/migrations/sql/0129-mod-file-change-table.sql")}]) + :fn (mg/resource "app/migrations/sql/0129-mod-file-change-table.sql")} + + {:name "0130-mod-file-change-table" + :fn (mg/resource "app/migrations/sql/0130-mod-file-change-table.sql")}]) (defn apply-migrations! [pool name migrations] diff --git a/backend/src/app/migrations/sql/0130-mod-file-change-table.sql b/backend/src/app/migrations/sql/0130-mod-file-change-table.sql new file mode 100644 index 000000000..272828fc2 --- /dev/null +++ b/backend/src/app/migrations/sql/0130-mod-file-change-table.sql @@ -0,0 +1,2 @@ +ALTER TABLE file_change + ADD COLUMN version integer NULL; diff --git a/backend/src/app/rpc/commands/files_snapshot.clj b/backend/src/app/rpc/commands/files_snapshot.clj index ced6a64de..2fdb262a0 100644 --- a/backend/src/app/rpc/commands/files_snapshot.clj +++ b/backend/src/app/rpc/commands/files_snapshot.clj @@ -103,6 +103,7 @@ (db/update! conn :file {:data (:data snapshot) :revn (inc (:revn file)) + :version (:version snapshot) :data-backend nil :data-ref-id nil :has-media-trimmed false @@ -170,7 +171,7 @@ (update :data blob/encode))))) (defn take-file-snapshot! - [cfg {:keys [file-id label]}] + [cfg {:keys [file-id label ::rpc/profile-id]}] (let [file (get-file cfg file-id) id (uuid/next)] @@ -182,7 +183,9 @@ {:id id :revn (:revn file) :data (:data file) + :version (:version file) :features (:features file) + :profile-id profile-id :file-id (:id file) :label label} {::db/return-keys false}) diff --git a/backend/src/app/rpc/commands/files_update.clj b/backend/src/app/rpc/commands/files_update.clj index 007b3b68a..f010ac7b7 100644 --- a/backend/src/app/rpc/commands/files_update.clj +++ b/backend/src/app/rpc/commands/files_update.clj @@ -251,6 +251,7 @@ :created-at created-at :file-id (:id file) :revn (:revn file) + :version (:version file) :label (::snapshot-label file) :data (::snapshot-data file) :features (db/create-array conn "text" (:features file)) diff --git a/backend/src/app/tasks/file_gc.clj b/backend/src/app/tasks/file_gc.clj index e88cfcef0..07bc7ffba 100644 --- a/backend/src/app/tasks/file_gc.clj +++ b/backend/src/app/tasks/file_gc.clj @@ -11,7 +11,6 @@ inactivity (the default threshold is 72h)." (:require [app.binfile.common :as bfc] - [app.common.exceptions :as ex] [app.common.files.migrations :as fmg] [app.common.files.validate :as cfv] [app.common.logging :as l] @@ -35,16 +34,36 @@ (declare ^:private decode-file) (declare ^:private persist-file!) +(def ^:private sql:get-snapshots + "SELECT f.file_id AS id, + f.data, + f.revn, + f.version, + f.features, + f.data_backend, + f.data_ref_id + FROM file_change AS f + WHERE f.file_id = ? + AND f.label IS NOT NULL + ORDER BY f.created_at ASC") + (def ^:private sql:mark-file-media-object-deleted "UPDATE file_media_object SET deleted_at = now() WHERE file_id = ? AND id != ALL(?::uuid[]) RETURNING id") +(def ^:private xf:collect-used-media + (comp (map :data) (mapcat bfc/collect-used-media))) + (defn- clean-file-media! "Performs the garbage collection of file media objects." - [{:keys [::db/conn]} {:keys [id data] :as file}] - (let [used (bfc/collect-used-media data) + [{:keys [::db/conn] :as cfg} {:keys [id] :as file}] + (let [used (into #{} + xf:collect-used-media + (cons file + (->> (db/cursor conn [sql:get-snapshots id]) + (map (partial decode-file cfg))))) ids (db/create-array conn "uuid" used) unused (->> (db/exec! conn [sql:mark-file-media-object-deleted id ids]) (into #{} (map :id)))] @@ -170,11 +189,6 @@ (l/dbg :hint "clean" :rel "components" :file-id (str file-id) :total (count unused)) file)) -(def ^:private sql:get-changes - "SELECT id, data FROM file_change - WHERE file_id = ? AND data IS NOT NULL - ORDER BY created_at ASC") - (def ^:private sql:mark-deleted-data-fragments "UPDATE file_data_fragment SET deleted_at = now() @@ -190,8 +204,7 @@ (defn- clean-data-fragments! [{:keys [::db/conn]} {:keys [id] :as file}] - (let [used (into #{} xf:collect-pointers - (cons file (db/cursor conn [sql:get-changes id]))) + (let [used (into #{} xf:collect-pointers [file]) unused (let [ids (db/create-array conn "uuid" used)] (->> (db/exec! conn [sql:mark-deleted-data-fragments id ids]) @@ -220,7 +233,9 @@ f.revn, f.version, f.features, - f.modified_at + f.modified_at, + f.data_backend, + f.data_ref_id FROM file AS f WHERE f.has_media_trimmed IS false AND f.modified_at < now() - ?::interval @@ -236,18 +251,8 @@ (defn- decode-file [cfg {:keys [id] :as file}] - ;; NOTE: a preventive check that does not allow proceed the gc for - ;; already offloaded file; if this exception happens, means that - ;; something external modified the file flag without preloading the - ;; file back again to the table - (when (feat.fdata/offloaded? file) - (ex/raise :hint "unable to run file-gc on an already offloaded file" - :type :internal - :code :file-already-offloaded - :file-id id)) - (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)] - (-> file + (-> (feat.fdata/resolve-file-data cfg file) (update :features db/decode-pgarray #{}) (update :data blob/decode) (update :data feat.fdata/process-pointers deref) @@ -256,7 +261,7 @@ (fmg/migrate-file)))) (defn- persist-file! - [{:keys [::db/conn] :as cfg} {:keys [id] :as file}] + [{:keys [::db/conn ::sto/storage] :as cfg} {:keys [id] :as file}] (let [file (if (contains? (:features file) "fdata/objects-map") (feat.fdata/enable-objects-map file) file) @@ -272,11 +277,18 @@ (update :features db/encode-pgarray conn "text") (update :data blob/encode))] + ;; If file was already offloaded, we touch the underlying storage + ;; object for properly trigger storage-gc-touched task + (when (feat.fdata/offloaded? file) + (some->> (:data-ref-id file) (sto/touch-object! storage))) + (db/update! conn :file {:has-media-trimmed true :features (:features file) :version (:version file) - :data (:data file)} + :data (:data file) + :data-backend nil + :data-ref-id nil} {:id id} {::db/return-keys true}))) diff --git a/backend/src/app/tasks/file_xlog_gc.clj b/backend/src/app/tasks/file_xlog_gc.clj index 07253f0d8..6bbacd250 100644 --- a/backend/src/app/tasks/file_xlog_gc.clj +++ b/backend/src/app/tasks/file_xlog_gc.clj @@ -10,6 +10,7 @@ (:require [app.common.logging :as l] [app.db :as db] + [app.features.fdata :as feat.fdata] [app.storage :as sto] [app.util.time :as dt] [clojure.spec.alpha :as s] @@ -26,8 +27,8 @@ (def xf:filter-offloded (comp - (filter #(= "objects-storage" (:data-backend %))) - (map :data-ref-id))) + (filter feat.fdata/offloaded?) + (keep :data-ref-id))) (defn- delete-in-chunks [{:keys [::chunk-size ::threshold] :as cfg}] diff --git a/frontend/src/debug.cljs b/frontend/src/debug.cljs index 818028f52..9ccbd442a 100644 --- a/frontend/src/debug.cljs +++ b/frontend/src/debug.cljs @@ -506,6 +506,6 @@ (rx/mapcat rp/handle-response) (rx/subs! (fn [_] (println "Snapshot restored " (or snapshot-id label))) - #_(.reload js/location)) - (fn [cause] - (js/console.log "EE:" cause)))))) + #_(.reload js/location) + (fn [cause] + (js/console.log "EE:" cause))))))) From d5f5c440dd594166b1ba1d3655f8d6ba5781b8c1 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 4 Sep 2024 11:09:05 +0200 Subject: [PATCH 3/3] :bug: Fix issues with srepl helper for profile deletion in bulk --- backend/src/app/srepl/main.clj | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index 98cb5b9cb..922466675 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -727,13 +727,15 @@ deleted 0 total 0] (if-let [email (first emails)] - (if-let [profile (db/get* system :profile - {:email (str/lower email)} - {::db/remove-deleted false})] + (if-let [profile (some-> (db/get* system :profile + {:email (str/lower email)} + {::db/remove-deleted false}) + (profile/decode-row))] (do (audit/insert! system {::audit/name "delete-profile" ::audit/type "action" + ::audit/profile-id (:id profile) ::audit/tracked-at deleted-at ::audit/props (audit/profile->props profile) ::audit/context {:triggered-by "srepl"