File history versions management

This commit is contained in:
alonso.torres 2024-10-15 12:11:32 +02:00
parent fa4f2aa5cc
commit ecb7f0a2f6
33 changed files with 1100 additions and 102 deletions

View file

@ -89,6 +89,10 @@
(dissoc ::s/problems ::s/value ::s/spec ::sm/explain)
(cond-> explain (assoc :explain explain)))})
(= code :vern-conflict)
{::yres/status 409 ;; 409 - Conflict
::yres/body data}
(= code :request-body-too-large)
{::yres/status 413 ::yres/body data}

View file

@ -415,7 +415,13 @@
:fn (mg/resource "app/migrations/sql/0130-mod-file-change-table.sql")}
{:name "0131-mod-webhook-table"
:fn (mg/resource "app/migrations/sql/0131-mod-webhook-table.sql")}])
:fn (mg/resource "app/migrations/sql/0131-mod-webhook-table.sql")}
{:name "0132-mod-file-change-table"
:fn (mg/resource "app/migrations/sql/0132-mod-file-change-table.sql")}
{:name "0133-mod-file-table"
:fn (mg/resource "app/migrations/sql/0133-mod-file-table.sql")}])
(defn apply-migrations!
[pool name migrations]

View file

@ -0,0 +1,2 @@
ALTER TABLE file_change
ADD COLUMN created_by text NOT NULL DEFAULT 'system';

View file

@ -0,0 +1,2 @@
ALTER TABLE file
ADD COLUMN vern int NOT NULL DEFAULT 0;

View file

@ -182,6 +182,7 @@
[:comment-thread-seqn [::sm/int {:min 0}]]
[:name [:string {:max 250}]]
[:revn [::sm/int {:min 0}]]
[:vern [::sm/int {:min 0}]]
[:modified-at ::dt/instant]
[:is-shared ::sm/boolean]
[:project-id ::sm/uuid]
@ -270,7 +271,7 @@
(defn get-minimal-file
[cfg id & {:as opts}]
(let [opts (assoc opts ::sql/columns [:id :modified-at :deleted-at :revn :data-ref-id :data-backend])]
(let [opts (assoc opts ::sql/columns [:id :modified-at :deleted-at :revn :vern :data-ref-id :data-backend])]
(db/get cfg :file {:id id} opts)))
(defn- get-minimal-file-with-perms
@ -280,8 +281,8 @@
(assoc mfile :permissions perms)))
(defn get-file-etag
[{:keys [::rpc/profile-id]} {:keys [modified-at revn permissions]}]
(str profile-id "/" revn "/"
[{:keys [::rpc/profile-id]} {:keys [modified-at revn vern permissions]}]
(str profile-id "/" revn "/" vern "/"
(dt/format-instant modified-at :iso)
"/"
(uri/map->query-string permissions)))
@ -371,6 +372,7 @@
f.modified_at,
f.name,
f.revn,
f.vern,
f.is_shared,
ft.media_id
from file as f
@ -526,6 +528,7 @@
(def ^:private sql:team-shared-files
"select f.id,
f.revn,
f.vern,
f.data,
f.project_id,
f.created_at,
@ -609,6 +612,7 @@
l.deleted_at,
l.name,
l.revn,
l.vern,
l.synced_at
FROM libs AS l
WHERE l.deleted_at IS NULL OR l.deleted_at > now();")
@ -670,6 +674,7 @@
"with recent_files as (
select f.id,
f.revn,
f.vern,
f.project_id,
f.created_at,
f.modified_at,

View file

@ -10,14 +10,13 @@
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.db.sql :as-alias sql]
[app.features.fdata :as feat.fdata]
[app.main :as-alias main]
[app.msgbus :as mbus]
[app.rpc :as-alias rpc]
[app.rpc.commands.files :as files]
[app.rpc.commands.profile :as profile]
[app.rpc.doc :as-alias doc]
[app.storage :as sto]
[app.util.blob :as blob]
@ -26,22 +25,12 @@
[app.util.time :as dt]
[cuerdas.core :as str]))
(defn check-authorized!
[{:keys [::db/pool]} profile-id]
(when-not (or (= "devenv" (cf/get :host))
(let [profile (ex/ignoring (profile/get-profile pool profile-id))
admins (or (cf/get :admins) #{})]
(contains? admins (:email profile))))
(ex/raise :type :authentication
:code :authentication-required
:hint "only admins allowed")))
(def sql:get-file-snapshots
"SELECT id, label, revn, created_at
"SELECT id, label, revn, created_at, created_by, profile_id
FROM file_change
WHERE file_id = ?
AND created_at < ?
AND label IS NOT NULL
AND data IS NOT NULL
ORDER BY created_at DESC
LIMIT ?")
@ -50,25 +39,23 @@
:or {limit Long/MAX_VALUE}}]
(let [start-at (or start-at (dt/now))
limit (min limit 20)]
(->> (db/exec! conn [sql:get-file-snapshots file-id start-at limit])
(mapv (fn [row]
(update row :created-at dt/format-instant :rfc1123))))))
(db/exec! conn [sql:get-file-snapshots file-id start-at limit])))
(def ^:private schema:get-file-snapshots
[:map [:file-id ::sm/uuid]])
[:map {:title "get-file-snapshots"}
[:file-id ::sm/uuid]])
(sv/defmethod ::get-file-snapshots
{::doc/added "1.20"
::doc/skip true
::sm/params schema:get-file-snapshots}
[cfg {:keys [::rpc/profile-id] :as params}]
(check-authorized! cfg profile-id)
[cfg params]
(db/run! cfg get-file-snapshots params))
(defn restore-file-snapshot!
[{:keys [::db/conn] :as cfg} {:keys [file-id id]}]
[{:keys [::db/conn ::mbus/msgbus] :as cfg} {:keys [file-id id]}]
(let [storage (sto/resolve cfg {::db/reuse-conn true})
file (files/get-minimal-file conn file-id {::db/for-update true})
vern (rand-int Integer/MAX_VALUE)
snapshot (db/get* conn :file-change
{:file-id file-id
:id id}
@ -103,6 +90,7 @@
(db/update! conn :file
{:data (:data snapshot)
:revn (inc (:revn file))
:vern vern
:version (:version snapshot)
:data-backend nil
:data-ref-id nil
@ -126,38 +114,28 @@
(doseq [media-id (into #{} (keep :media-id) res)]
(sto/touch-object! storage media-id)))
;; Send to the clients a notification to reload the file
(mbus/pub! msgbus
:topic (:id file)
:message {:type :file-restore
:file-id (:id file)
:vern vern})
{:id (:id snapshot)
:label (:label snapshot)})))
(defn- resolve-snapshot-by-label
[conn file-id label]
(->> (db/query conn :file-change
{:file-id file-id
:label label}
{::sql/order-by [[:created-at :desc]]
::sql/columns [:file-id :id :label]})
(first)))
(def ^:private
schema:restore-file-snapshot
[:and
[:map
[:file-id ::sm/uuid]
[:id {:optional true} ::sm/uuid]
[:label {:optional true} :string]]
[:id {:optional true} ::sm/uuid]]
[::sm/contains-any #{:id :label}]])
(sv/defmethod ::restore-file-snapshot
{::doc/added "1.20"
::doc/skip true
::sm/params schema:restore-file-snapshot}
[cfg {:keys [::rpc/profile-id file-id id label] :as params}]
(check-authorized! cfg profile-id)
(db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
(let [params (cond-> params
(and (not id) (string? label))
(merge (resolve-snapshot-by-label conn file-id label)))]
(restore-file-snapshot! cfg params)))))
[cfg params]
(db/tx-run! cfg restore-file-snapshot! params))
(defn- get-file
[cfg file-id]
@ -170,29 +148,7 @@
(update :data feat.fdata/process-objects (partial into {}))
(update :data blob/encode)))))
(defn take-file-snapshot!
[cfg {:keys [file-id label ::rpc/profile-id]}]
(let [file (get-file cfg file-id)
id (uuid/next)]
(l/debug :hint "creating file snapshot"
:file-id (str file-id)
:label label)
(db/insert! cfg :file-change
{:id id
:revn (:revn file)
:data (:data file)
:version (:version file)
:features (:features file)
:profile-id profile-id
:file-id (:id file)
:label label}
{::db/return-keys false})
{:id id :label label}))
(defn generate-snapshot-label
(defn- generate-snapshot-label
[]
(let [ts (-> (dt/now)
(dt/format-instant)
@ -200,17 +156,92 @@
(str/rtrim "Z"))]
(str "snapshot-" ts)))
(defn take-file-snapshot!
[cfg {:keys [file-id label ::rpc/profile-id]}]
(let [label (or label (generate-snapshot-label))
file (-> (get-file cfg file-id)
(update :data
(fn [data]
(-> data
(blob/decode)
(assoc :id file-id)))))
snapshot-id
(uuid/next)
snapshot-data
(-> (:data file)
(feat.fdata/process-pointers deref)
(feat.fdata/process-objects (partial into {}))
(blob/encode))]
(l/debug :hint "creating file snapshot"
:file-id (str file-id)
:id (str snapshot-id)
:label label)
(db/insert! cfg :file-change
{:id snapshot-id
:revn (:revn file)
:data snapshot-data
:version (:version file)
:features (:features file)
:profile-id profile-id
:file-id (:id file)
:label label
:created-by "user"}
{::db/return-keys false})
{:id snapshot-id :label label}))
(def ^:private schema:take-file-snapshot
[:map [:file-id ::sm/uuid]])
[:map
[:file-id ::sm/uuid]
[:label {:optional true} :string]])
(sv/defmethod ::take-file-snapshot
{::doc/added "1.20"
::doc/skip true
::sm/params schema:take-file-snapshot}
[cfg {:keys [::rpc/profile-id] :as params}]
(check-authorized! cfg profile-id)
(db/tx-run! cfg (fn [cfg]
(let [params (update params :label (fn [label]
(or label (generate-snapshot-label))))]
(take-file-snapshot! cfg params)))))
[cfg params]
(db/tx-run! cfg take-file-snapshot! params))
(def ^:private schema:update-file-snapshot
[:map {:title "update-file-snapshot"}
[:id ::sm/uuid]
[:label ::sm/text]])
(defn update-file-snapshot!
[{:keys [::db/conn] :as cfg} {:keys [id label]}]
(let [result
(db/update! conn :file-change
{:label label
:created-by "user"}
{:id id}
{::db/return-keys true})]
(select-keys result [:id :label :revn :created-at :profile-id :created-by])))
(sv/defmethod ::update-file-snapshot
{::doc/added "1.20"
::sm/params schema:update-file-snapshot}
[cfg params]
(db/tx-run! cfg update-file-snapshot! params))
(def ^:private schema:remove-file-snapshot
[:map {:title "remove-file-snapshot"}
[:id ::sm/uuid]])
(defn remove-file-snapshot!
[{:keys [::db/conn] :as cfg} {:keys [id]}]
(db/delete! conn :file-change
{:id id :created-by "user"}
{::db/return-keys false})
nil)
(sv/defmethod ::remove-file-snapshot
{::doc/added "1.20"
::sm/params schema:remove-file-snapshot}
[cfg params]
(db/tx-run! cfg remove-file-snapshot! params))

View file

@ -60,6 +60,7 @@
[:id ::sm/uuid]
[:session-id ::sm/uuid]
[:revn {:min 0} ::sm/int]
[:vern {:min 0} ::sm/int]
[:features {:optional true} ::cfeat/features]
[:changes {:optional true} [:vector ::cpc/change]]
[:changes-with-metadata {:optional true}
@ -157,6 +158,14 @@
tpoint (dt/tpoint)]
(when (not= (:vern params)
(:vern file))
(ex/raise :type :validation
:code :vern-conflict
:hint "A different version has been restored for the file."
:context {:incoming-revn (:revn params)
:stored-revn (:revn file)}))
(when (> (:revn params)
(:revn file))
(ex/raise :type :validation
@ -455,7 +464,7 @@
"SELECT fch.id, fch.created_at
FROM file_change AS fch
WHERE fch.file_id = ?
AND fch.label LIKE 'internal/%'
AND fch.created_by = 'system'
ORDER BY fch.created_at DESC
LIMIT ?")
@ -465,7 +474,7 @@
"UPDATE file_change
SET label = NULL
WHERE file_id = ?
AND label LIKE 'internal/%'
AND created_by LIKE 'system'
AND created_at < ?")
(defn- delete-old-snapshots!
@ -502,6 +511,7 @@
:file-id (:id file)
:session-id (:session-id params)
:revn (:revn file)
:vern (:vern file)
:changes changes})
(when (and (:is-shared file) (seq lchanges))

View file

@ -336,8 +336,7 @@
(defn take-team-snapshot!
[team-id & {:keys [label rollback?] :or {rollback? true}}]
(let [team-id (h/parse-uuid team-id)
label (or label (fsnap/generate-snapshot-label))]
(let [team-id (h/parse-uuid team-id)]
(-> (assoc main/system ::db/rollback rollback?)
(db/tx-run! h/take-team-snapshot! team-id label))))

View file

@ -36,6 +36,7 @@
:id file-id
:session-id (uuid/random)
:revn revn
:vern 0
:features cfeat/supported-features
:changes changes}
out (th/command! params)]
@ -55,6 +56,7 @@
:file-id (:id file)
:profile-id (:id profile)
:revn 0
:vern 0
:changes
[{:type :add-page
:name "test 1"
@ -67,6 +69,7 @@
:file-id (:id file)
:profile-id (:id profile)
:revn 0
:vern 0
:changes
[{:type :add-obj
:page-id page-id-1

View file

@ -312,6 +312,7 @@
(#'files.update/update-file* system
{:id file-id
:revn revn
:vern 0
:file file
:features (:features file)
:changes changes
@ -327,6 +328,7 @@
:id file-id
:session-id (uuid/random)
:revn revn
:vern 0
:features features
:changes changes}
out (command! params)]

View file

@ -32,6 +32,7 @@
:id file-id
:session-id (uuid/random)
:revn revn
:vern 0
:features cfeat/supported-features
:changes changes}
out (th/command! params)]
@ -147,6 +148,7 @@
:id file-id
:session-id (uuid/random)
:revn revn
:vern 0
:features cfeat/supported-features
:changes changes}
out (th/command! params)]
@ -174,6 +176,7 @@
:file-id (:id file)
:profile-id (:id profile)
:revn 0
:vern 0
:changes
[{:type :add-page
:name "test"
@ -203,6 +206,7 @@
:file-id (:id file)
:profile-id (:id profile)
:revn 0
:vern 0
:changes
[{:type :add-obj
:page-id page-id
@ -279,6 +283,7 @@
:id file-id
:session-id (uuid/random)
:revn revn
:vern 0
:features cfeat/supported-features
:changes changes}
out (th/command! params)]
@ -305,6 +310,7 @@
:file-id (:id file)
:profile-id (:id profile)
:revn 0
:vern 0
:changes
[{:type :add-obj
:page-id page-id
@ -367,6 +373,7 @@
:file-id (:id file)
:profile-id (:id profile)
:revn 0
:vern 0
:changes [{:type :del-obj
:page-id (first (get-in file [:data :pages]))
:id shid}])
@ -418,6 +425,7 @@
:id file-id
:session-id (uuid/random)
:revn revn
:vern 0
:components-v2 true
:changes changes}
out (th/command! params)]
@ -452,6 +460,7 @@
:file-id (:id file)
:profile-id (:id profile)
:revn 0
:vern 0
:changes
[{:type :add-obj
:page-id page-id
@ -528,6 +537,7 @@
:file-id (:id file)
:profile-id (:id profile)
:revn 0
:vern 0
:changes [{:type :del-obj
:page-id (first (get-in file [:data :pages]))
:id s-shid}
@ -622,6 +632,7 @@
:file-id (:id file)
:profile-id (:id profile)
:revn 0
:vern 0
:changes
[{:type :add-obj
:page-id page-id
@ -688,6 +699,7 @@
:file-id file-id
:profile-id (:id profile)
:revn 0
:vern 0
:changes [{:type :del-obj
:page-id page-id
:id frame-id-2}])
@ -721,6 +733,7 @@
:file-id file-id
:profile-id (:id profile)
:revn 0
:vern 0
:changes [{:type :del-obj
:page-id page-id
:id frame-id-1}])
@ -978,6 +991,7 @@
(th/update-file* {:file-id (:id file)
:profile-id (:id prof)
:revn 0
:vern 0
:components-v2 true
:changes changes})
@ -1178,6 +1192,7 @@
file (th/create-file* 1 {:profile-id (:id prof)
:project-id (:default-project-id prof)
:revn 2
:vern 0
:is-shared false})]
(t/testing "create a file thumbnail"
@ -1286,6 +1301,7 @@
:file-id (:id file)
:profile-id (:id profile)
:revn 0
:vern 0
:changes
[{:type :add-page
:name "test"

View file

@ -42,6 +42,7 @@
:file-id (:id file)
:profile-id (:id profile)
:revn 0
:vern 0
:changes
[{:type :add-obj
:page-id page-id
@ -253,7 +254,8 @@
file (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)
:is-shared false
:revn 3})
:revn 3
:vern 0})
data1 {::th/type :create-file-thumbnail
::rpc/profile-id (:id profile)

View file

@ -30,6 +30,7 @@
:id file-id
:session-id (uuid/random)
:revn revn
:vern 0
:features cfeat/supported-features
:changes changes}
out (th/command! params)]
@ -65,6 +66,7 @@
:file-id (:id file1)
:profile-id (:id profile)
:revn 0
:vern 0
:changes
[{:type :add-media
:object mobj}])
@ -195,6 +197,7 @@
:file-id (:id file1)
:profile-id (:id profile)
:revn 0
:vern 0
:changes
[{:type :add-media
:object mobj}])