diff --git a/backend/resources/app/templates/api-doc.css b/backend/resources/app/templates/api-doc.css index 33e2022ca..35476a55f 100644 --- a/backend/resources/app/templates/api-doc.css +++ b/backend/resources/app/templates/api-doc.css @@ -156,7 +156,7 @@ h4 { } .rpc-row-info > .module { - width: 120px; + width: 150px; font-weight: bold; border-right: 1px dotted #777; text-align: right; diff --git a/backend/src/app/db.clj b/backend/src/app/db.clj index 03f56b2bb..13046c15c 100644 --- a/backend/src/app/db.clj +++ b/backend/src/app/db.clj @@ -5,7 +5,7 @@ ;; Copyright (c) KALEIDOS INC (ns app.db - (:refer-clojure :exclude [get]) + (:refer-clojure :exclude [get run!]) (:require [app.common.data :as d] [app.common.exceptions :as ex] @@ -391,6 +391,52 @@ ([^Connection conn ^Savepoint sp] (.rollback conn sp))) +(defn tx-run! + [cfg f] + (cond + (connection? cfg) + (tx-run! {::conn cfg} f) + + (pool? cfg) + (tx-run! {::pool cfg} f) + + (::conn cfg) + (let [conn (::conn cfg) + sp (savepoint conn)] + (try + (let [result (f cfg)] + (release! conn sp) + result) + (catch Throwable cause + (rollback! sp) + (throw cause)))) + + (::pool cfg) + (with-atomic [conn (::pool cfg)] + (f (assoc cfg ::conn conn))) + + :else + (throw (IllegalArgumentException. "invalid arguments")))) + +(defn run! + [cfg f] + (cond + (connection? cfg) + (run! {::conn cfg} f) + + (pool? cfg) + (run! {::pool cfg} f) + + (::conn cfg) + (f cfg) + + (::pool cfg) + (with-open [^Connection conn (open (::pool cfg))] + (f (assoc cfg ::conn conn))) + + :else + (throw (IllegalArgumentException. "invalid arguments")))) + (defn interval [o] (cond diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index 787d34450..ecaeae7c3 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -324,6 +324,9 @@ {:name "0104-mod-file-thumbnail-table" :fn (mg/resource "app/migrations/sql/0104-mod-file-thumbnail-table.sql")} + {:name "0105-mod-file-change-table" + :fn (mg/resource "app/migrations/sql/0105-mod-file-change-table.sql")} + ]) (defn apply-migrations! diff --git a/backend/src/app/migrations/sql/0105-mod-file-change-table.sql b/backend/src/app/migrations/sql/0105-mod-file-change-table.sql new file mode 100644 index 000000000..d8385df9e --- /dev/null +++ b/backend/src/app/migrations/sql/0105-mod-file-change-table.sql @@ -0,0 +1,9 @@ +ALTER TABLE file_change + ADD COLUMN label text NULL; + +ALTER TABLE file_change + ALTER COLUMN label SET STORAGE external; + +CREATE INDEX file_change__label__idx + ON file_change (file_id, label) + WHERE label is not null; diff --git a/backend/src/app/rpc.clj b/backend/src/app/rpc.clj index 63077eec7..201e83062 100644 --- a/backend/src/app/rpc.clj +++ b/backend/src/app/rpc.clj @@ -214,6 +214,7 @@ 'app.rpc.commands.files-share 'app.rpc.commands.files-temp 'app.rpc.commands.files-update + 'app.rpc.commands.files-snapshot 'app.rpc.commands.files-thumbnails 'app.rpc.commands.ldap 'app.rpc.commands.management diff --git a/backend/src/app/rpc/commands/audit.clj b/backend/src/app/rpc/commands/audit.clj index 9475a5a04..8049595c9 100644 --- a/backend/src/app/rpc/commands/audit.clj +++ b/backend/src/app/rpc/commands/audit.clj @@ -68,6 +68,7 @@ ::climit/key-fn ::rpc/profile-id ::sm/params schema:push-audit-events ::audit/skip true + ::doc/skip true ::doc/added "1.17"} [{:keys [::db/pool] :as cfg} params] (if (or (db/read-only? pool) diff --git a/backend/src/app/rpc/commands/files_snapshot.clj b/backend/src/app/rpc/commands/files_snapshot.clj new file mode 100644 index 000000000..b81d825de --- /dev/null +++ b/backend/src/app/rpc/commands/files_snapshot.clj @@ -0,0 +1,136 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns app.rpc.commands.files-snapshot + (:require + [app.common.exceptions :as ex] + [app.common.logging :as l] + [app.common.schema :as sm] + [app.common.uuid :as uuid] + [app.config :as cf] + [app.db :as db] + [app.main :as-alias main] + [app.media :as media] + [app.rpc :as-alias rpc] + [app.rpc.commands.profile :as profile] + [app.rpc.doc :as-alias doc] + [app.storage :as sto] + [app.util.services :as sv] + [app.util.time :as dt])) + +(defn check-authorized! + [{:keys [::db/pool]} profile-id] + (when-not (or (= "devenv" (cf/get :host)) + (let [profile (ex/ignoring (profile/get-profile pool profile-id)) + admins (or (cf/get :admins) #{})] + (contains? admins (:email profile)))) + (ex/raise :type :authentication + :code :authentication-required + :hint "only admins allowed"))) + +(defn get-file-snapshots + [{:keys [::db/conn]} {:keys [file-id limit start-at] + :or {limit Long/MAX_VALUE}}] + (let [query (str "select id, label, revn, created_at " + " from file_change " + " where file_id = ? " + " and created_at < ? " + " and label is not null " + " and data is not null " + " order by created_at desc " + " limit ?") + start-at (or start-at (dt/now)) + limit (min limit 20)] + + (->> (db/exec! conn [query file-id start-at limit]) + (mapv (fn [row] + (update row :created-at dt/format-instant :rfc1123)))))) + +(def ^:private schema:get-file-snapshots + [:map [:file-id ::sm/uuid]]) + +(sv/defmethod ::get-file-snapshots + {::doc/added "1.20" + ::doc/skip true + ::sm/params schema:get-file-snapshots} + [cfg {:keys [::rpc/profile-id] :as params}] + (check-authorized! cfg profile-id) + (db/run! cfg #(get-file-snapshots % params))) + +(defn restore-file-snapshot! + [{:keys [::db/conn ::sto/storage] :as cfg} {:keys [file-id id]}] + (let [storage (media/configure-assets-storage storage conn) + params {:id id :file-id file-id} + options {:columns [:id :data :revn]} + snapshot (db/get* conn :file-change params options)] + + (when (and (some? snapshot) + (some? (:data snapshot))) + + (l/debug :hint "snapshot found" + :snapshot-id (:id snapshot) + :file-id file-id) + + (db/update! conn :file + {:data (:data snapshot)} + {:id file-id}) + + ;; clean object thumbnails + (let [sql (str "delete from file_object_thumbnail " + " where file_id=? returning media_id") + res (db/exec! conn [sql file-id])] + + (doseq [media-id (into #{} (keep :media-id) res)] + (sto/del-object! storage media-id))) + + ;; clean object thumbnails + (let [sql (str "delete from file_thumbnail " + " where file_id=? returning media_id") + res (db/exec! conn [sql file-id])] + (doseq [media-id (into #{} (keep :media-id) res)] + (sto/del-object! storage media-id))) + + {:id (:id snapshot)}))) + +(def ^:private schema:restore-file-snapshot + [:map + [:file-id ::sm/uuid] + [:id ::sm/uuid]]) + +(sv/defmethod ::restore-file-snapshot + {::doc/added "1.20" + ::doc/skip true + ::sm/params schema:restore-file-snapshot} + [cfg {:keys [::rpc/profile-id] :as params}] + (check-authorized! cfg profile-id) + (db/tx-run! cfg #(restore-file-snapshot! % params))) + +(defn take-file-snapshot! + [{:keys [::db/conn]} {:keys [file-id label]}] + (when-let [file (db/get* conn :file {:id file-id})] + (let [id (uuid/next) + label (or label (str "Snapshot at " (dt/format-instant (dt/now) :rfc1123)))] + (l/debug :hint "persisting file snapshot" :file-id file-id :label label) + (db/insert! conn :file-change + {:id id + :revn (:revn file) + :data (:data file) + :features (:features file) + :file-id (:id file) + :label label}) + {:id id}))) + +(def ^:private schema:take-file-snapshot + [:map [:file-id ::sm/uuid]]) + +(sv/defmethod ::take-file-snapshot + {::doc/added "1.20" + ::doc/skip true + ::sm/params schema:take-file-snapshot} + [cfg {:keys [::rpc/profile-id] :as params}] + (check-authorized! cfg profile-id) + (db/tx-run! cfg #(take-file-snapshot! % params))) + diff --git a/backend/src/app/rpc/doc.clj b/backend/src/app/rpc/doc.clj index c797ffc1e..89d8c2280 100644 --- a/backend/src/app/rpc/doc.clj +++ b/backend/src/app/rpc/doc.clj @@ -75,6 +75,7 @@ (->> methods (map val) (map first) + (remove ::skip) (map get-context) (sort-by (juxt :module :name)))})) diff --git a/backend/src/app/srepl/helpers.clj b/backend/src/app/srepl/helpers.clj index 96fbe6c4c..5c53270b3 100644 --- a/backend/src/app/srepl/helpers.clj +++ b/backend/src/app/srepl/helpers.clj @@ -6,6 +6,7 @@ (ns app.srepl.helpers "A main namespace for server repl." + (:refer-clojure :exclude [parse-uuid]) #_:clj-kondo/ignore (:require [app.auth :refer [derive-password]] @@ -39,6 +40,26 @@ (def ^:dynamic *conn*) (def ^:dynamic *pool*) +(defn println! + [& params] + (locking println + (apply println params))) + +(defn parse-uuid + [v] + (if (uuid? v) + v + (d/parse-uuid v))) + +(defn resolve-connectable + [o] + (if (db/connection? o) + o + (if (db/pool? o) + o + (or (::db/conn o) + (::db/pool o))))) + (defn reset-password! "Reset a password to a specific one for a concrete user or all users if email is `:all` keyword." @@ -104,7 +125,7 @@ (dissoc file :data)))))) (def ^:private sql:retrieve-files-chunk - "SELECT id, name, created_at, data FROM file + "SELECT id, name, created_at, revn, data FROM file WHERE created_at < ? AND deleted_at is NULL ORDER BY created_at desc LIMIT ?") @@ -150,11 +171,6 @@ (when (fn? on-end) (on-end)))) -(defn- println! - [& params] - (locking println - (apply println params))) - (defn process-files! "Apply a function to all files in the database, reading them in batches." diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index 77413995b..bd4dbfbc8 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -8,20 +8,25 @@ "A collection of adhoc fixes scripts." #_:clj-kondo/ignore (:require + [app.common.data :as d] [app.common.logging :as l] [app.common.pprint :as p] [app.common.spec :as us] + [app.common.uuid :as uuid] [app.db :as db] + [app.media :as media] [app.rpc.commands.auth :as auth] [app.rpc.commands.profile :as profile] + [app.rpc.commands.files-snapshot :as fsnap] [app.srepl.fixes :as f] [app.srepl.helpers :as h] + [app.storage :as sto] [app.util.blob :as blob] [app.util.objects-map :as omap] [app.util.pointer-map :as pmap] [app.util.time :as dt] [app.worker :as wrk] - [clojure.pprint :refer [pprint]] + [clojure.pprint :refer [pprint print-table]] [cuerdas.core :as str])) (defn print-available-tasks @@ -101,7 +106,6 @@ (db/delete! conn :http-session {:profile-id (:id profile)}) :blocked)))) - (defn enable-objects-map-feature-on-file! [system & {:keys [save? id]}] (letfn [(update-file [{:keys [features] :as file}] @@ -164,3 +168,32 @@ (alter-var-root var (fn [f] (or (::original (meta f)) f)))) +(defn take-file-snapshot! + "An internal helper that persist the file snapshot using non-gc + collectable file-changes entry." + [system & {:keys [file-id label]}] + (let [file-id (h/parse-uuid file-id)] + (db/tx-run! system + (fn [cfg] + (fsnap/take-file-snapshot! cfg {:file-id file-id :label label}))))) + +(defn restore-file-snapshot! + [system & {:keys [file-id id]}] + (db/tx-run! system + (fn [cfg] + (let [file-id (h/parse-uuid file-id) + id (h/parse-uuid id)] + + (if (and (uuid? id) (uuid? file-id)) + (fsnap/restore-file-snapshot! cfg {:id id :file-id file-id}) + (println "=> invalid parameters")))))) + + +(defn list-file-snapshots! + [system & {:keys [file-id limit]}] + (db/tx-run! system (fn [system] + (let [params {:file-id (h/parse-uuid file-id) + :limit limit}] + (->> (fsnap/get-file-snapshots system (d/without-nils params)) + (print-table [:id :revn :created-at :label])))))) + diff --git a/backend/src/app/tasks/file_xlog_gc.clj b/backend/src/app/tasks/file_xlog_gc.clj index 5ee2e1bbc..c88f42a84 100644 --- a/backend/src/app/tasks/file_xlog_gc.clj +++ b/backend/src/app/tasks/file_xlog_gc.clj @@ -17,7 +17,8 @@ (def ^:private sql:delete-files-xlog "delete from file_change - where created_at < now() - ?::interval") + where created_at < now() - ?::interval + and label is NULL") (defmethod ig/pre-init-spec ::handler [_] (s/keys :req [::db/pool])) diff --git a/frontend/src/debug.cljs b/frontend/src/debug.cljs index de9391bdc..bed89944c 100644 --- a/frontend/src/debug.cljs +++ b/frontend/src/debug.cljs @@ -11,7 +11,9 @@ [app.common.math :as mth] [app.common.transit :as t] [app.common.types.file :as ctf] + [app.common.uri :as u] [app.common.uuid :as uuid] + [app.config :as cf] [app.main.data.dashboard.shortcuts] [app.main.data.viewer.shortcuts] [app.main.data.workspace :as dw] @@ -20,6 +22,7 @@ [app.main.data.workspace.shortcuts] [app.main.store :as st] [app.util.dom :as dom] + [app.util.http :as http] [app.util.object :as obj] [app.util.timers :as timers] [beicon.core :as rx] @@ -410,3 +413,47 @@ [id shape-ref] (st/emit! (dw/set-shape-ref id shape-ref))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; SNAPSHOTS +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + +(defn ^:export list-available-snapshots + [file-id] + (let [file-id (d/parse-uuid file-id)] + (->> (http/send! {:method :post + :uri (u/join cf/public-uri "api/rpc/command/get-file-snapshots") + :body (http/transit-data {:file-id file-id})}) + (rx/map http/conditional-decode-transit) + (rx/map :body) + (rx/subs (fn [result] + (let [result (->> result + (map (fn [row] + (update row :id str))))] + (js/console.table (clj->js result)))))))) + + +(defn ^:export take-snapshot + [file-id label] + (let [file-id (d/parse-uuid file-id)] + (->> (http/send! {:method :post + :uri (u/join cf/public-uri "api/rpc/command/take-file-snapshot") + :body (http/transit-data {:file-id file-id :label label})}) + (rx/map http/conditional-decode-transit) + (rx/map :body) + (rx/subs (fn [{:keys [id]}] + (println "Snapshot saved:" (str id))))))) + +(defn ^:export restore-snapshot + [file-id id] + (let [file-id (d/parse-uuid file-id) + id (d/parse-uuid id)] + (->> (http/send! {:method :post + :uri (u/join cf/public-uri "api/rpc/command/restore-file-snapshot") + :body (http/transit-data {:file-id file-id :id id})}) + (rx/map http/conditional-decode-transit) + (rx/map :body) + (rx/subs (fn [_] + (println "Snapshot restored " id) + #_(.reload js/location))))))