mirror of
https://github.com/penpot/penpot.git
synced 2025-05-10 13:56:39 +02:00
✨ Improve file snapshoting mechanism
This commit is contained in:
parent
3eaa997145
commit
5cf54c6384
8 changed files with 100 additions and 40 deletions
|
@ -24,6 +24,7 @@ export PENPOT_FLAGS="\
|
||||||
enable-rpc-climit \
|
enable-rpc-climit \
|
||||||
enable-rpc-rlimit \
|
enable-rpc-rlimit \
|
||||||
enable-soft-rpc-rlimit \
|
enable-soft-rpc-rlimit \
|
||||||
|
enable-file-snapshot \
|
||||||
enable-webhooks \
|
enable-webhooks \
|
||||||
enable-access-tokens \
|
enable-access-tokens \
|
||||||
enable-file-validation \
|
enable-file-validation \
|
||||||
|
|
|
@ -17,6 +17,7 @@ export PENPOT_FLAGS="\
|
||||||
disable-secure-session-cookies \
|
disable-secure-session-cookies \
|
||||||
enable-rpc-climit \
|
enable-rpc-climit \
|
||||||
enable-smtp \
|
enable-smtp \
|
||||||
|
enable-file-snapshot \
|
||||||
enable-access-tokens \
|
enable-access-tokens \
|
||||||
enable-file-validation \
|
enable-file-validation \
|
||||||
enable-file-schema-validation";
|
enable-file-schema-validation";
|
||||||
|
|
|
@ -42,8 +42,9 @@
|
||||||
:rpc-rlimit-config "resources/rlimit.edn"
|
:rpc-rlimit-config "resources/rlimit.edn"
|
||||||
:rpc-climit-config "resources/climit.edn"
|
:rpc-climit-config "resources/climit.edn"
|
||||||
|
|
||||||
:file-change-snapshot-every 5
|
:file-snapshot-total 10
|
||||||
:file-change-snapshot-timeout "3h"
|
:file-snapshot-every 5
|
||||||
|
:file-snapshot-timeout "3h"
|
||||||
|
|
||||||
:public-uri "http://localhost:3449"
|
:public-uri "http://localhost:3449"
|
||||||
:host "localhost"
|
:host "localhost"
|
||||||
|
@ -100,8 +101,9 @@
|
||||||
[:telemetry-uri {:optional true} :string]
|
[:telemetry-uri {:optional true} :string]
|
||||||
[:telemetry-with-taiga {:optional true} :boolean] ;; DELETE
|
[:telemetry-with-taiga {:optional true} :boolean] ;; DELETE
|
||||||
|
|
||||||
[:file-change-snapshot-every {:optional true} :int]
|
[:file-snapshot-total {:optional true} :int]
|
||||||
[:file-change-snapshot-timeout {:optional true} ::dt/duration]
|
[:file-snapshot-every {:optional true} :int]
|
||||||
|
[:file-snapshot-timeout {:optional true} ::dt/duration]
|
||||||
|
|
||||||
[:media-max-file-size {:optional true} :int]
|
[:media-max-file-size {:optional true} :int]
|
||||||
[:deletion-delay {:optional true} ::dt/duration] ;; REVIEW
|
[:deletion-delay {:optional true} ::dt/duration] ;; REVIEW
|
||||||
|
|
|
@ -123,12 +123,13 @@
|
||||||
(feat.fdata/persist-pointers! cfg id)
|
(feat.fdata/persist-pointers! cfg id)
|
||||||
result))))
|
result))))
|
||||||
|
|
||||||
(declare get-lagged-changes)
|
(declare ^:private delete-old-snapshots!)
|
||||||
(declare send-notifications!)
|
(declare ^:private get-lagged-changes)
|
||||||
(declare update-file)
|
(declare ^:private send-notifications!)
|
||||||
(declare update-file*)
|
(declare ^:private take-snapshot?)
|
||||||
(declare update-file-data)
|
(declare ^:private update-file)
|
||||||
(declare take-snapshot?)
|
(declare ^:private update-file*)
|
||||||
|
(declare ^:private update-file-data)
|
||||||
|
|
||||||
;; If features are specified from params and the final feature
|
;; If features are specified from params and the final feature
|
||||||
;; set is different than the persisted one, update it on the
|
;; set is different than the persisted one, update it on the
|
||||||
|
@ -238,12 +239,15 @@
|
||||||
:created-at created-at
|
:created-at created-at
|
||||||
:file-id (:id file)
|
:file-id (:id file)
|
||||||
:revn (:revn file)
|
:revn (:revn file)
|
||||||
|
:label (::snapshot-label file)
|
||||||
|
:data (::snapshot-data file)
|
||||||
:features (db/create-array conn "text" (:features file))
|
:features (db/create-array conn "text" (:features file))
|
||||||
:data (when (take-snapshot? file)
|
|
||||||
(:data file))
|
|
||||||
:changes (blob/encode changes)}
|
:changes (blob/encode changes)}
|
||||||
{::db/return-keys false})
|
{::db/return-keys false})
|
||||||
|
|
||||||
|
(when (::snapshot-data file)
|
||||||
|
(delete-old-snapshots! cfg file))
|
||||||
|
|
||||||
(db/update! conn :file
|
(db/update! conn :file
|
||||||
{:revn (:revn file)
|
{:revn (:revn file)
|
||||||
:data (:data file)
|
:data (:data file)
|
||||||
|
@ -286,7 +290,6 @@
|
||||||
(-> data
|
(-> data
|
||||||
(blob/decode)
|
(blob/decode)
|
||||||
(assoc :id (:id file)))))
|
(assoc :id (:id file)))))
|
||||||
|
|
||||||
;; For avoid unnecesary overhead of creating multiple pointers
|
;; For avoid unnecesary overhead of creating multiple pointers
|
||||||
;; and handly internally with objects map in their worst
|
;; and handly internally with objects map in their worst
|
||||||
;; case (when probably all shapes and all pointers will be
|
;; case (when probably all shapes and all pointers will be
|
||||||
|
@ -322,8 +325,27 @@
|
||||||
file (-> (files/check-version! file)
|
file (-> (files/check-version! file)
|
||||||
(update :revn inc)
|
(update :revn inc)
|
||||||
(update :data cpc/process-changes changes)
|
(update :data cpc/process-changes changes)
|
||||||
(update :data d/without-nils))]
|
(update :data d/without-nils))
|
||||||
|
|
||||||
|
file (if (take-snapshot? file)
|
||||||
|
(let [tpoint (dt/tpoint)
|
||||||
|
snapshot (-> (:data file)
|
||||||
|
(feat.fdata/process-pointers deref)
|
||||||
|
(feat.fdata/process-objects (partial into {}))
|
||||||
|
(blob/encode))
|
||||||
|
elapsed (tpoint)
|
||||||
|
label (str "internal/snapshot/" (:revn file))]
|
||||||
|
|
||||||
|
(l/trc :hint "take snapshot"
|
||||||
|
:file-id (str (:id file))
|
||||||
|
:revn (:revn file)
|
||||||
|
:label label
|
||||||
|
:elapsed (dt/format-duration elapsed))
|
||||||
|
|
||||||
|
(-> file
|
||||||
|
(assoc ::snapshot-data snapshot)
|
||||||
|
(assoc ::snapshot-label label)))
|
||||||
|
file)]
|
||||||
|
|
||||||
(binding [pmap/*tracked* nil]
|
(binding [pmap/*tracked* nil]
|
||||||
(when (contains? cf/flags :soft-file-validation)
|
(when (contains? cf/flags :soft-file-validation)
|
||||||
|
@ -353,13 +375,42 @@
|
||||||
(defn- take-snapshot?
|
(defn- take-snapshot?
|
||||||
"Defines the rule when file `data` snapshot should be saved."
|
"Defines the rule when file `data` snapshot should be saved."
|
||||||
[{:keys [revn modified-at] :as file}]
|
[{:keys [revn modified-at] :as file}]
|
||||||
(let [freq (or (cf/get :file-change-snapshot-every) 20)
|
(when (contains? cf/flags :file-snapshot)
|
||||||
timeout (or (cf/get :file-change-snapshot-timeout)
|
(let [freq (or (cf/get :file-snapshot-every) 20)
|
||||||
|
timeout (or (cf/get :file-snapshot-timeout)
|
||||||
(dt/duration {:hours 1}))]
|
(dt/duration {:hours 1}))]
|
||||||
|
|
||||||
(or (= 1 freq)
|
(or (= 1 freq)
|
||||||
(zero? (mod revn freq))
|
(zero? (mod revn freq))
|
||||||
(> (inst-ms (dt/diff modified-at (dt/now)))
|
(> (inst-ms (dt/diff modified-at (dt/now)))
|
||||||
(inst-ms timeout)))))
|
(inst-ms timeout))))))
|
||||||
|
|
||||||
|
;; Get the latest available snapshots without exceeding the total
|
||||||
|
;; snapshot limit.
|
||||||
|
(def ^:private sql:get-latest-snapshots
|
||||||
|
"SELECT fch.id, fch.created_at
|
||||||
|
FROM file_change AS fch
|
||||||
|
WHERE fch.file_id = ?
|
||||||
|
AND fch.label LIKE 'internal/%'
|
||||||
|
ORDER BY fch.created_at DESC
|
||||||
|
LIMIT ?")
|
||||||
|
|
||||||
|
;; Mark all snapshots that are outside the allowed total threshold
|
||||||
|
;; available for the GC.
|
||||||
|
(def ^:private sql:delete-snapshots
|
||||||
|
"UPDATE file_change
|
||||||
|
SET label = NULL
|
||||||
|
WHERE file_id = ?
|
||||||
|
AND label IS NOT NULL
|
||||||
|
AND created_at < ?")
|
||||||
|
|
||||||
|
(defn- delete-old-snapshots!
|
||||||
|
[{:keys [::db/conn] :as cfg} {:keys [id] :as file}]
|
||||||
|
(when-let [snapshots (not-empty (db/exec! conn [sql:get-latest-snapshots id
|
||||||
|
(cf/get :file-snapshot-total 10)]))]
|
||||||
|
(let [last-date (-> snapshots peek :created-at)
|
||||||
|
result (db/exec-one! conn [sql:delete-snapshots id last-date])]
|
||||||
|
(l/trc :hint "delete old snapshots" :file-id (str id) :total (db/get-update-count result)))))
|
||||||
|
|
||||||
(def ^:private
|
(def ^:private
|
||||||
sql:lagged-changes
|
sql:lagged-changes
|
||||||
|
|
|
@ -314,7 +314,7 @@
|
||||||
0
|
0
|
||||||
(get-candidates cfg))]
|
(get-candidates cfg))]
|
||||||
|
|
||||||
(l/inf :hint "task finished"
|
(l/inf :hint "finished"
|
||||||
:min-age (dt/format-duration min-age)
|
:min-age (dt/format-duration min-age)
|
||||||
:processed total)
|
:processed total)
|
||||||
|
|
||||||
|
|
|
@ -61,7 +61,7 @@
|
||||||
(def default
|
(def default
|
||||||
{:database-uri "postgresql://postgres/penpot_test"
|
{:database-uri "postgresql://postgres/penpot_test"
|
||||||
:redis-uri "redis://redis/1"
|
:redis-uri "redis://redis/1"
|
||||||
:file-change-snapshot-every 1})
|
:file-snapshot-every 1})
|
||||||
|
|
||||||
(def config
|
(def config
|
||||||
(cf/read-config :prefix "penpot-test"
|
(cf/read-config :prefix "penpot-test"
|
||||||
|
@ -76,6 +76,7 @@
|
||||||
:enable-feature-fdata-pointer-map
|
:enable-feature-fdata-pointer-map
|
||||||
:enable-feature-fdata-objets-map
|
:enable-feature-fdata-objets-map
|
||||||
:enable-feature-components-v2
|
:enable-feature-components-v2
|
||||||
|
:enable-file-snapshot
|
||||||
:disable-file-validation])
|
:disable-file-validation])
|
||||||
|
|
||||||
(defn state-init
|
(defn state-init
|
||||||
|
|
|
@ -166,6 +166,10 @@
|
||||||
:name "test"
|
:name "test"
|
||||||
:id page-id}])
|
:id page-id}])
|
||||||
|
|
||||||
|
;; Check the number of fragments before adding the page
|
||||||
|
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)})]
|
||||||
|
(t/is (= 3 (count rows))))
|
||||||
|
|
||||||
;; The file-gc should mark for remove unused fragments
|
;; The file-gc should mark for remove unused fragments
|
||||||
(let [res (th/run-task! :file-gc {:min-age 0})]
|
(let [res (th/run-task! :file-gc {:min-age 0})]
|
||||||
(t/is (= 1 (:processed res))))
|
(t/is (= 1 (:processed res))))
|
||||||
|
@ -176,11 +180,11 @@
|
||||||
|
|
||||||
;; The objects-gc should remove unused fragments
|
;; The objects-gc should remove unused fragments
|
||||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||||
(t/is (= 1 (:processed res))))
|
(t/is (= 3 (:processed res))))
|
||||||
|
|
||||||
;; Check the number of fragments
|
;; Check the number of fragments
|
||||||
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)})]
|
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)})]
|
||||||
(t/is (= 4 (count rows))))
|
(t/is (= 2 (count rows))))
|
||||||
|
|
||||||
;; Add shape to page that should add a new fragment
|
;; Add shape to page that should add a new fragment
|
||||||
(update-file!
|
(update-file!
|
||||||
|
@ -203,7 +207,7 @@
|
||||||
|
|
||||||
;; Check the number of fragments
|
;; Check the number of fragments
|
||||||
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)})]
|
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)})]
|
||||||
(t/is (= 5 (count rows))))
|
(t/is (= 3 (count rows))))
|
||||||
|
|
||||||
;; The file-gc should mark for remove unused fragments
|
;; The file-gc should mark for remove unused fragments
|
||||||
(let [res (th/run-task! :file-gc {:min-age 0})]
|
(let [res (th/run-task! :file-gc {:min-age 0})]
|
||||||
|
@ -211,13 +215,13 @@
|
||||||
|
|
||||||
;; The objects-gc should remove unused fragments
|
;; The objects-gc should remove unused fragments
|
||||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||||
(t/is (= 1 (:processed res))))
|
(t/is (= 3 (:processed res))))
|
||||||
|
|
||||||
;; Check the number of fragments; should be 3 because changes
|
;; Check the number of fragments; should be 3 because changes
|
||||||
;; are also holding pointers to fragments;
|
;; are also holding pointers to fragments;
|
||||||
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)
|
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)
|
||||||
:deleted-at nil})]
|
:deleted-at nil})]
|
||||||
(t/is (= 6 (count rows))))
|
(t/is (= 2 (count rows))))
|
||||||
|
|
||||||
;; Lets proceed to delete all changes
|
;; Lets proceed to delete all changes
|
||||||
(th/db-delete! :file-change {:file-id (:id file)})
|
(th/db-delete! :file-change {:file-id (:id file)})
|
||||||
|
@ -233,11 +237,11 @@
|
||||||
;; Check the number of fragments;
|
;; Check the number of fragments;
|
||||||
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)})]
|
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)})]
|
||||||
;; (pp/pprint rows)
|
;; (pp/pprint rows)
|
||||||
(t/is (= 8 (count rows)))
|
(t/is (= 4 (count rows)))
|
||||||
(t/is (= 2 (count (remove (comp some? :deleted-at) rows)))))
|
(t/is (= 2 (count (remove (comp some? :deleted-at) rows)))))
|
||||||
|
|
||||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||||
(t/is (= 6 (:processed res))))
|
(t/is (= 2 (:processed res))))
|
||||||
|
|
||||||
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)})]
|
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)})]
|
||||||
(t/is (= 2 (count rows)))))))
|
(t/is (= 2 (count rows)))))))
|
||||||
|
@ -338,7 +342,7 @@
|
||||||
(t/is (= 1 (count (remove (comp some? :deleted-at) rows)))))
|
(t/is (= 1 (count (remove (comp some? :deleted-at) rows)))))
|
||||||
|
|
||||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||||
(t/is (= 2 (:processed res))))
|
(t/is (= 3 (:processed res))))
|
||||||
|
|
||||||
;; check file media objects
|
;; check file media objects
|
||||||
(let [rows (th/db-query :file-media-object {:file-id (:id file)})]
|
(let [rows (th/db-query :file-media-object {:file-id (:id file)})]
|
||||||
|
@ -367,7 +371,7 @@
|
||||||
(t/is (= 1 (:processed res))))
|
(t/is (= 1 (:processed res))))
|
||||||
|
|
||||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||||
(t/is (= 2 (:processed res))))
|
(t/is (= 3 (:processed res))))
|
||||||
|
|
||||||
;; Now that file-gc have deleted the file-media-object usage,
|
;; Now that file-gc have deleted the file-media-object usage,
|
||||||
;; lets execute the touched-gc task, we should see that two of
|
;; lets execute the touched-gc task, we should see that two of
|
||||||
|
@ -494,11 +498,11 @@
|
||||||
(t/is (= 1 (:processed res))))
|
(t/is (= 1 (:processed res))))
|
||||||
|
|
||||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||||
(t/is (= 1 (:processed res))))
|
(t/is (= 2 (:processed res))))
|
||||||
|
|
||||||
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)
|
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)
|
||||||
:deleted-at nil})]
|
:deleted-at nil})]
|
||||||
(t/is (= (count rows) 2)))
|
(t/is (= (count rows) 1)))
|
||||||
|
|
||||||
;; retrieve file and check trimmed attribute
|
;; retrieve file and check trimmed attribute
|
||||||
(let [row (th/db-get :file {:id (:id file)})]
|
(let [row (th/db-get :file {:id (:id file)})]
|
||||||
|
@ -535,11 +539,11 @@
|
||||||
(t/is (= 1 (:processed res))))
|
(t/is (= 1 (:processed res))))
|
||||||
|
|
||||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||||
(t/is (= 6 (:processed res))))
|
(t/is (= 7 (:processed res))))
|
||||||
|
|
||||||
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)
|
(let [rows (th/db-query :file-data-fragment {:file-id (:id file)
|
||||||
:deleted-at nil})]
|
:deleted-at nil})]
|
||||||
(t/is (= (count rows) 3)))
|
(t/is (= (count rows) 1)))
|
||||||
|
|
||||||
;; Now that file-gc have deleted the file-media-object usage,
|
;; Now that file-gc have deleted the file-media-object usage,
|
||||||
;; lets execute the touched-gc task, we should see that two of
|
;; lets execute the touched-gc task, we should see that two of
|
||||||
|
@ -702,7 +706,7 @@
|
||||||
;; thumbnail lets execute the objects-gc task which remove
|
;; thumbnail lets execute the objects-gc task which remove
|
||||||
;; the rows and mark as touched the storage object rows
|
;; the rows and mark as touched the storage object rows
|
||||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||||
(t/is (= 3 (:processed res))))
|
(t/is (= 5 (:processed res))))
|
||||||
|
|
||||||
;; Now that objects-gc have deleted the object thumbnail lets
|
;; Now that objects-gc have deleted the object thumbnail lets
|
||||||
;; execute the touched-gc task
|
;; execute the touched-gc task
|
||||||
|
@ -732,7 +736,7 @@
|
||||||
|
|
||||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||||
;; (pp/pprint res)
|
;; (pp/pprint res)
|
||||||
(t/is (= 2 (:processed res))))
|
(t/is (= 3 (:processed res))))
|
||||||
|
|
||||||
;; We still have th storage objects in the table
|
;; We still have th storage objects in the table
|
||||||
(let [rows (th/db-query :storage-object {:deleted-at nil})]
|
(let [rows (th/db-query :storage-object {:deleted-at nil})]
|
||||||
|
@ -1168,7 +1172,7 @@
|
||||||
(t/is (= 1 (count (remove :deleted-at rows)))))
|
(t/is (= 1 (count (remove :deleted-at rows)))))
|
||||||
|
|
||||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||||
(t/is (= 3 (:processed res))))
|
(t/is (= 4 (:processed res))))
|
||||||
|
|
||||||
(let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})]
|
(let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})]
|
||||||
;; (app.common.pprint/pprint rows)
|
;; (app.common.pprint/pprint rows)
|
||||||
|
|
|
@ -118,7 +118,7 @@
|
||||||
(t/is (= 1 (:processed result))))
|
(t/is (= 1 (:processed result))))
|
||||||
|
|
||||||
(let [result (th/run-task! :objects-gc {:min-age 0})]
|
(let [result (th/run-task! :objects-gc {:min-age 0})]
|
||||||
(t/is (= 2 (:processed result))))
|
(t/is (= 3 (:processed result))))
|
||||||
|
|
||||||
;; check if row2 related thumbnail row still exists
|
;; check if row2 related thumbnail row still exists
|
||||||
(let [[row :as rows] (th/db-query :file-tagged-object-thumbnail
|
(let [[row :as rows] (th/db-query :file-tagged-object-thumbnail
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue