mirror of
https://github.com/penpot/penpot.git
synced 2025-08-06 04:38:19 +02:00
🎉 Add inplace binfile import support
This commit is contained in:
Andrey Antukh
parent
fd62141c04
commit
37cec8891f
8 changed files with 255 additions and 144 deletions
|
|
@ -15,13 +15,14 @@
|
|||
[app.common.files.migrations :as fmg]
|
||||
[app.common.files.validate :as fval]
|
||||
[app.common.logging :as l]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.types.file :as ctf]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.db.sql :as sql]
|
||||
[app.features.fdata :as feat.fdata]
|
||||
[app.features.file-migrations :as feat.fmigr]
|
||||
[app.features.fdata :as fdata]
|
||||
[app.features.file-migrations :as fmigr]
|
||||
[app.loggers.audit :as-alias audit]
|
||||
[app.loggers.webhooks :as-alias webhooks]
|
||||
[app.storage :as sto]
|
||||
|
|
@ -32,12 +33,14 @@
|
|||
[clojure.set :as set]
|
||||
[cuerdas.core :as str]
|
||||
[datoteka.fs :as fs]
|
||||
[datoteka.io :as io]))
|
||||
[datoteka.io :as io]
|
||||
[promesa.exec :as px]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(def ^:dynamic *state* nil)
|
||||
(def ^:dynamic *options* nil)
|
||||
(def ^:dynamic *reference-file* nil)
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; DEFAULTS
|
||||
|
|
@ -53,17 +56,12 @@
|
|||
(* 1024 1024 100))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(declare get-resolved-file-libraries)
|
||||
(declare update-file!)
|
||||
|
||||
(def file-attrs
|
||||
#{:id
|
||||
:name
|
||||
:migrations
|
||||
:features
|
||||
:project-id
|
||||
:is-shared
|
||||
:version
|
||||
:data})
|
||||
(sm/keys ctf/schema:file))
|
||||
|
||||
(defn parse-file-format
|
||||
[template]
|
||||
|
|
@ -151,22 +149,33 @@
|
|||
changes (assoc :changes (blob/decode changes))
|
||||
data (assoc :data (blob/decode data)))))
|
||||
|
||||
(def sql:get-minimal-file
|
||||
"SELECT f.id,
|
||||
f.revn,
|
||||
f.modified_at,
|
||||
f.deleted_at
|
||||
FROM file AS f
|
||||
WHERE f.id = ?")
|
||||
|
||||
(defn get-minimal-file
|
||||
[cfg id & {:as opts}]
|
||||
(db/get-with-sql cfg [sql:get-minimal-file id] opts))
|
||||
|
||||
(defn decode-file
|
||||
"A general purpose file decoding function that resolves all external
|
||||
pointers, run migrations and return plain vanilla file map"
|
||||
[cfg {:keys [id] :as file} & {:keys [migrate?] :or {migrate? true}}]
|
||||
(binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
|
||||
(binding [pmap/*load-fn* (partial fdata/load-pointer cfg id)]
|
||||
(let [file (->> file
|
||||
(feat.fmigr/resolve-applied-migrations cfg)
|
||||
(feat.fdata/resolve-file-data cfg))
|
||||
(fmigr/resolve-applied-migrations cfg)
|
||||
(fdata/resolve-file-data cfg))
|
||||
libs (delay (get-resolved-file-libraries cfg file))]
|
||||
|
||||
(-> file
|
||||
(update :features db/decode-pgarray #{})
|
||||
(update :data blob/decode)
|
||||
(update :data feat.fdata/process-pointers deref)
|
||||
(update :data feat.fdata/process-objects (partial into {}))
|
||||
(update :data fdata/process-pointers deref)
|
||||
(update :data fdata/process-objects (partial into {}))
|
||||
(update :data assoc :id id)
|
||||
(cond-> migrate? (fmg/migrate-file libs))))))
|
||||
|
||||
|
|
@ -421,6 +430,27 @@
|
|||
(db/exec-one! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"])
|
||||
(db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])))
|
||||
|
||||
(defn invalidate-thumbnails
|
||||
[cfg file-id]
|
||||
(let [storage (sto/resolve cfg)
|
||||
|
||||
sql-1
|
||||
(str "update file_tagged_object_thumbnail "
|
||||
" set deleted_at = now() "
|
||||
" where file_id=? returning media_id")
|
||||
|
||||
sql-2
|
||||
(str "update file_thumbnail "
|
||||
" set deleted_at = now() "
|
||||
" where file_id=? returning media_id")]
|
||||
|
||||
(run! #(sto/touch-object! storage %)
|
||||
(sequence
|
||||
(keep :media-id)
|
||||
(concat
|
||||
(db/exec! cfg [sql-1 file-id])
|
||||
(db/exec! cfg [sql-2 file-id]))))))
|
||||
|
||||
(defn process-file
|
||||
[cfg {:keys [id] :as file}]
|
||||
(let [libs (delay (get-resolved-file-libraries cfg file))]
|
||||
|
|
@ -445,77 +475,79 @@
|
|||
(vary-meta dissoc ::fmg/migrated))))
|
||||
|
||||
(defn encode-file
|
||||
[{:keys [::db/conn] :as cfg} {:keys [id features] :as file}]
|
||||
(let [file (if (contains? features "fdata/objects-map")
|
||||
(feat.fdata/enable-objects-map file)
|
||||
[{:keys [::wrk/executor] :as cfg} {:keys [id features] :as file}]
|
||||
(let [file (if (and (contains? features "fdata/objects-map")
|
||||
(:data file))
|
||||
(fdata/enable-objects-map file)
|
||||
file)
|
||||
|
||||
file (if (contains? features "fdata/pointer-map")
|
||||
(binding [pmap/*tracked* (pmap/create-tracked)]
|
||||
(let [file (feat.fdata/enable-pointer-map file)]
|
||||
(feat.fdata/persist-pointers! cfg id)
|
||||
file (if (and (contains? features "fdata/pointer-map")
|
||||
(:data file))
|
||||
|
||||
(binding [pmap/*tracked* (pmap/create-tracked :inherit true)]
|
||||
(let [file (fdata/enable-pointer-map file)]
|
||||
(fdata/persist-pointers! cfg id)
|
||||
file))
|
||||
file)]
|
||||
|
||||
(-> file
|
||||
(update :features db/encode-pgarray conn "text")
|
||||
(update :data blob/encode))))
|
||||
(d/update-when :features into-array)
|
||||
(d/update-when :data (fn [data] (px/invoke! executor #(blob/encode data)))))))
|
||||
|
||||
(defn get-params-from-file
|
||||
(defn- file->params
|
||||
[file]
|
||||
(let [params {:has-media-trimmed (:has-media-trimmed file)
|
||||
:ignore-sync-until (:ignore-sync-until file)
|
||||
:project-id (:project-id file)
|
||||
:features (:features file)
|
||||
:name (:name file)
|
||||
:is-shared (:is-shared file)
|
||||
:version (:version file)
|
||||
:data (:data file)
|
||||
:id (:id file)
|
||||
:deleted-at (:deleted-at file)
|
||||
:created-at (:created-at file)
|
||||
:modified-at (:modified-at file)
|
||||
:revn (:revn file)
|
||||
:vern (:vern file)}]
|
||||
|
||||
(-> (d/without-nils params)
|
||||
(assoc :data-backend nil)
|
||||
(assoc :data-ref-id nil))))
|
||||
(-> (select-keys file file-attrs)
|
||||
(dissoc :team-id)
|
||||
(dissoc :migrations)))
|
||||
|
||||
(defn insert-file!
|
||||
"Insert a new file into the database table"
|
||||
"Insert a new file into the database table. Expectes a not-encoded file.
|
||||
Returns nil."
|
||||
[{:keys [::db/conn] :as cfg} file & {:as opts}]
|
||||
(feat.fmigr/upsert-migrations! conn file)
|
||||
(let [params (-> (encode-file cfg file)
|
||||
(get-params-from-file))]
|
||||
(db/insert! conn :file params opts)))
|
||||
|
||||
(when (:migrations file)
|
||||
(fmigr/upsert-migrations! conn file))
|
||||
|
||||
(let [file (encode-file cfg file)]
|
||||
(db/insert! conn :file
|
||||
(file->params file)
|
||||
{::db/return-keys false})
|
||||
nil))
|
||||
|
||||
(defn update-file!
|
||||
"Update an existing file on the database."
|
||||
[{:keys [::db/conn ::sto/storage] :as cfg} {:keys [id] :as file} & {:as opts}]
|
||||
(let [file (encode-file cfg file)
|
||||
params (-> (get-params-from-file file)
|
||||
(dissoc :id))]
|
||||
"Update an existing file on the database. Expects not encoded file."
|
||||
[{:keys [::db/conn] :as cfg} {:keys [id] :as file} & {:as opts}]
|
||||
|
||||
;; If file was already offloaded, we touch the underlying storage
|
||||
;; object for properly trigger storage-gc-touched task
|
||||
(when (feat.fdata/offloaded? file)
|
||||
(some->> (:data-ref-id file) (sto/touch-object! storage)))
|
||||
(if (::reset-migrations opts false)
|
||||
(fmigr/reset-migrations! conn file)
|
||||
(fmigr/upsert-migrations! conn file))
|
||||
|
||||
(feat.fmigr/upsert-migrations! conn file)
|
||||
(db/update! conn :file params {:id id} opts)))
|
||||
(let [file
|
||||
(encode-file cfg file)
|
||||
|
||||
params
|
||||
(file->params (dissoc file :id))]
|
||||
|
||||
(db/update! conn :file params
|
||||
{:id id}
|
||||
{::db/return-keys false})
|
||||
|
||||
nil))
|
||||
|
||||
(defn save-file!
|
||||
"Applies all the final validations and perist the file, binfile
|
||||
specific, should not be used outside of binfile domain"
|
||||
specific, should not be used outside of binfile domain.
|
||||
|
||||
Returns nil"
|
||||
[{:keys [::timestamp] :as cfg} file & {:as opts}]
|
||||
|
||||
(assert (dt/instant? timestamp) "expected valid timestamp")
|
||||
|
||||
(let [file (-> file
|
||||
(assoc :created-at timestamp)
|
||||
(assoc :modified-at timestamp)
|
||||
(assoc :ignore-sync-until (dt/plus timestamp (dt/duration {:seconds 5})))
|
||||
(cond-> (not (::overwrite cfg))
|
||||
(assoc :ignore-sync-until (dt/plus timestamp (dt/duration {:seconds 5}))))
|
||||
(update :revn inc)
|
||||
(update :features
|
||||
(fn [features]
|
||||
(-> (::features cfg #{})
|
||||
|
|
@ -532,8 +564,9 @@
|
|||
(when (ex/exception? result)
|
||||
(l/error :hint "file schema validation error" :cause result))))
|
||||
|
||||
(insert-file! cfg file opts)))
|
||||
|
||||
(if (::overwrite cfg)
|
||||
(update-file! cfg file (assoc opts ::reset-migrations true))
|
||||
(insert-file! cfg file opts))))
|
||||
|
||||
(def ^:private sql:get-file-libraries
|
||||
"WITH RECURSIVE libs AS (
|
||||
|
|
@ -558,7 +591,8 @@
|
|||
l.revn,
|
||||
l.vern,
|
||||
l.synced_at,
|
||||
l.is_shared
|
||||
l.is_shared,
|
||||
l.version
|
||||
FROM libs AS l
|
||||
INNER JOIN project AS p ON (p.id = l.project_id)
|
||||
WHERE l.deleted_at IS NULL OR l.deleted_at > now();")
|
||||
|
|
@ -573,6 +607,8 @@
|
|||
(map decode-row))
|
||||
(db/exec! conn [sql:get-file-libraries file-id])))
|
||||
|
||||
;; FIXME: this will use a lot of memory if file uses too many big
|
||||
;; libraries, we should load required libraries on demand
|
||||
(defn get-resolved-file-libraries
|
||||
"A helper for preload file libraries"
|
||||
[{:keys [::db/conn] :as cfg} file]
|
||||
|
|
|
|||
|
|
@ -284,10 +284,12 @@
|
|||
(assoc :options (:options data))
|
||||
|
||||
:always
|
||||
(dissoc :data)
|
||||
(dissoc :data))
|
||||
|
||||
file (cond-> file
|
||||
:always
|
||||
(encode-file))
|
||||
|
||||
path (str "files/" file-id ".json")]
|
||||
(write-entry! output path file))
|
||||
|
||||
|
|
@ -544,15 +546,18 @@
|
|||
(json/read reader)))
|
||||
|
||||
(defn- read-file
|
||||
[{:keys [::bfc/input ::file-id]}]
|
||||
[{:keys [::bfc/input ::bfc/timestamp]} file-id]
|
||||
(let [path (str "files/" file-id ".json")
|
||||
entry (get-zip-entry input path)]
|
||||
(-> (read-entry input entry)
|
||||
(decode-file)
|
||||
(update :revn d/nilv 1)
|
||||
(update :created-at d/nilv timestamp)
|
||||
(update :modified-at d/nilv timestamp)
|
||||
(validate-file))))
|
||||
|
||||
(defn- read-file-plugin-data
|
||||
[{:keys [::bfc/input ::file-id]}]
|
||||
[{:keys [::bfc/input]} file-id]
|
||||
(let [path (str "files/" file-id "/plugin-data.json")
|
||||
entry (get-zip-entry* input path)]
|
||||
(some->> entry
|
||||
|
|
@ -561,7 +566,7 @@
|
|||
(validate-plugin-data))))
|
||||
|
||||
(defn- read-file-media
|
||||
[{:keys [::bfc/input ::file-id ::entries]}]
|
||||
[{:keys [::bfc/input ::entries]} file-id]
|
||||
(->> (keep (match-media-entry-fn file-id) entries)
|
||||
(reduce (fn [result {:keys [id entry]}]
|
||||
(let [object (->> (read-entry input entry)
|
||||
|
|
@ -581,7 +586,7 @@
|
|||
(not-empty)))
|
||||
|
||||
(defn- read-file-colors
|
||||
[{:keys [::bfc/input ::file-id ::entries]}]
|
||||
[{:keys [::bfc/input ::entries]} file-id]
|
||||
(->> (keep (match-color-entry-fn file-id) entries)
|
||||
(reduce (fn [result {:keys [id entry]}]
|
||||
(let [object (->> (read-entry input entry)
|
||||
|
|
@ -594,7 +599,7 @@
|
|||
(not-empty)))
|
||||
|
||||
(defn- read-file-components
|
||||
[{:keys [::bfc/input ::file-id ::entries]}]
|
||||
[{:keys [::bfc/input ::entries]} file-id]
|
||||
(let [clean-component-post-decode
|
||||
(fn [component]
|
||||
(d/update-when component :objects
|
||||
|
|
@ -625,7 +630,7 @@
|
|||
(not-empty))))
|
||||
|
||||
(defn- read-file-typographies
|
||||
[{:keys [::bfc/input ::file-id ::entries]}]
|
||||
[{:keys [::bfc/input ::entries]} file-id]
|
||||
(->> (keep (match-typography-entry-fn file-id) entries)
|
||||
(reduce (fn [result {:keys [id entry]}]
|
||||
(let [object (->> (read-entry input entry)
|
||||
|
|
@ -638,14 +643,14 @@
|
|||
(not-empty)))
|
||||
|
||||
(defn- read-file-tokens-lib
|
||||
[{:keys [::bfc/input ::file-id ::entries]}]
|
||||
[{:keys [::bfc/input ::entries]} file-id]
|
||||
(when-let [entry (d/seek (match-tokens-lib-entry-fn file-id) entries)]
|
||||
(->> (read-plain-entry input entry)
|
||||
(decode-tokens-lib)
|
||||
(validate-tokens-lib))))
|
||||
|
||||
(defn- read-file-shapes
|
||||
[{:keys [::bfc/input ::file-id ::page-id ::entries] :as cfg}]
|
||||
[{:keys [::bfc/input ::entries] :as cfg} file-id page-id]
|
||||
(->> (keep (match-shape-entry-fn file-id page-id) entries)
|
||||
(reduce (fn [result {:keys [id entry]}]
|
||||
(let [object (->> (read-entry input entry)
|
||||
|
|
@ -659,15 +664,14 @@
|
|||
(not-empty)))
|
||||
|
||||
(defn- read-file-pages
|
||||
[{:keys [::bfc/input ::file-id ::entries] :as cfg}]
|
||||
[{:keys [::bfc/input ::entries] :as cfg} file-id]
|
||||
(->> (keep (match-page-entry-fn file-id) entries)
|
||||
(keep (fn [{:keys [id entry]}]
|
||||
(let [page (->> (read-entry input entry)
|
||||
(decode-page))
|
||||
page (dissoc page :options)]
|
||||
(when (= id (:id page))
|
||||
(let [objects (-> (assoc cfg ::page-id id)
|
||||
(read-file-shapes))]
|
||||
(let [objects (read-file-shapes cfg file-id id)]
|
||||
(assoc page :objects objects))))))
|
||||
(sort-by :index)
|
||||
(reduce (fn [result {:keys [id] :as page}]
|
||||
|
|
@ -675,7 +679,7 @@
|
|||
(d/ordered-map))))
|
||||
|
||||
(defn- read-file-thumbnails
|
||||
[{:keys [::bfc/input ::file-id ::entries] :as cfg}]
|
||||
[{:keys [::bfc/input ::entries] :as cfg} file-id]
|
||||
(->> (keep (match-thumbnail-entry-fn file-id) entries)
|
||||
(reduce (fn [result {:keys [page-id frame-id tag entry]}]
|
||||
(let [object (->> (read-entry input entry)
|
||||
|
|
@ -690,13 +694,13 @@
|
|||
(not-empty)))
|
||||
|
||||
(defn- read-file-data
|
||||
[cfg]
|
||||
(let [colors (read-file-colors cfg)
|
||||
typographies (read-file-typographies cfg)
|
||||
tokens-lib (read-file-tokens-lib cfg)
|
||||
components (read-file-components cfg)
|
||||
plugin-data (read-file-plugin-data cfg)
|
||||
pages (read-file-pages cfg)]
|
||||
[cfg file-id]
|
||||
(let [colors (read-file-colors cfg file-id)
|
||||
typographies (read-file-typographies cfg file-id)
|
||||
tokens-lib (read-file-tokens-lib cfg file-id)
|
||||
components (read-file-components cfg file-id)
|
||||
plugin-data (read-file-plugin-data cfg file-id)
|
||||
pages (read-file-pages cfg file-id)]
|
||||
{:pages (-> pages keys vec)
|
||||
:pages-index (into {} pages)
|
||||
:colors colors
|
||||
|
|
@ -706,11 +710,11 @@
|
|||
:plugin-data plugin-data}))
|
||||
|
||||
(defn- import-file
|
||||
[{:keys [::bfc/project-id ::file-id ::file-name] :as cfg}]
|
||||
[{:keys [::bfc/project-id] :as cfg} {file-id :id file-name :name}]
|
||||
(let [file-id' (bfc/lookup-index file-id)
|
||||
file (read-file cfg)
|
||||
media (read-file-media cfg)
|
||||
thumbnails (read-file-thumbnails cfg)]
|
||||
file (read-file cfg file-id)
|
||||
media (read-file-media cfg file-id)
|
||||
thumbnails (read-file-thumbnails cfg file-id)]
|
||||
|
||||
(l/dbg :hint "processing file"
|
||||
:id (str file-id')
|
||||
|
|
@ -740,7 +744,7 @@
|
|||
(vswap! bfc/*state* update :index bfc/update-index (map :media-id thumbnails))
|
||||
(vswap! bfc/*state* update :thumbnails into thumbnails))
|
||||
|
||||
(let [data (-> (read-file-data cfg)
|
||||
(let [data (-> (read-file-data cfg file-id)
|
||||
(d/without-nils)
|
||||
(assoc :id file-id')
|
||||
(cond-> (:options file)
|
||||
|
|
@ -757,7 +761,7 @@
|
|||
file (ctf/check-file file)]
|
||||
|
||||
(bfm/register-pending-migrations! cfg file)
|
||||
(bfc/save-file! cfg file ::db/return-keys false)
|
||||
(bfc/save-file! cfg file)
|
||||
|
||||
file-id')))
|
||||
|
||||
|
|
@ -853,7 +857,8 @@
|
|||
:file-id (str (:file-id params))
|
||||
::l/sync? true)
|
||||
|
||||
(db/insert! conn :file-media-object params))))
|
||||
(db/insert! conn :file-media-object params
|
||||
::db/on-conflict-do-nothing? (::bfc/overwrite cfg)))))
|
||||
|
||||
(defn- import-file-thumbnails
|
||||
[{:keys [::db/conn] :as cfg}]
|
||||
|
|
@ -873,17 +878,77 @@
|
|||
:media-id (str media-id)
|
||||
::l/sync? true)
|
||||
|
||||
(db/insert! conn :file-tagged-object-thumbnail params))))
|
||||
(db/insert! conn :file-tagged-object-thumbnail params
|
||||
{::db/on-conflict-do-nothing? true}))))
|
||||
|
||||
(defn- import-files*
|
||||
[{:keys [::manifest] :as cfg}]
|
||||
(bfc/disable-database-timeouts! cfg)
|
||||
|
||||
(vswap! bfc/*state* update :index bfc/update-index (:files manifest) :id)
|
||||
|
||||
(let [files (get manifest :files)
|
||||
result (reduce (fn [result {:keys [id] :as file}]
|
||||
(let [name' (get file :name)
|
||||
name' (if (map? name)
|
||||
(get name id)
|
||||
name')
|
||||
file (assoc file :name name')]
|
||||
(conj result (import-file cfg file))))
|
||||
[]
|
||||
files)]
|
||||
|
||||
(import-file-relations cfg)
|
||||
(import-storage-objects cfg)
|
||||
(import-file-media cfg)
|
||||
(import-file-thumbnails cfg)
|
||||
|
||||
(bfm/apply-pending-migrations! cfg)
|
||||
|
||||
result))
|
||||
|
||||
(defn- import-file-and-overwrite*
|
||||
[{:keys [::manifest ::bfc/file-id] :as cfg}]
|
||||
|
||||
(when (not= 1 (count (:files manifest)))
|
||||
(ex/raise :type :validation
|
||||
:code :invalid-condition
|
||||
:hint "unable to perform in-place update with binfile containing more than 1 file"
|
||||
:manifest manifest))
|
||||
|
||||
(bfc/disable-database-timeouts! cfg)
|
||||
|
||||
(let [ref-file (bfc/get-minimal-file cfg file-id ::db/for-update true)
|
||||
file (first (get manifest :files))
|
||||
cfg (assoc cfg ::bfc/overwrite true)]
|
||||
|
||||
(vswap! bfc/*state* update :index assoc (:id file) file-id)
|
||||
|
||||
(binding [bfc/*options* cfg
|
||||
bfc/*reference-file* ref-file]
|
||||
|
||||
(import-file cfg file)
|
||||
(import-storage-objects cfg)
|
||||
(import-file-media cfg)
|
||||
|
||||
(bfc/invalidate-thumbnails cfg file-id)
|
||||
(bfm/apply-pending-migrations! cfg)
|
||||
|
||||
[file-id])))
|
||||
|
||||
(defn- import-files
|
||||
[{:keys [::bfc/timestamp ::bfc/input ::bfc/name] :or {timestamp (dt/now)} :as cfg}]
|
||||
[{:keys [::bfc/timestamp ::bfc/input] :or {timestamp (dt/now)} :as cfg}]
|
||||
|
||||
(assert (instance? ZipFile input) "expected zip file")
|
||||
(assert (dt/instant? timestamp) "expected valid instant")
|
||||
|
||||
(let [manifest (-> (read-manifest input)
|
||||
(validate-manifest))
|
||||
entries (read-zip-entries input)]
|
||||
entries (read-zip-entries input)
|
||||
cfg (-> cfg
|
||||
(assoc ::entries entries)
|
||||
(assoc ::manifest manifest)
|
||||
(assoc ::bfc/timestamp timestamp))]
|
||||
|
||||
(when-not (= "penpot/export-files" (:type manifest))
|
||||
(ex/raise :type :validation
|
||||
|
|
@ -891,7 +956,6 @@
|
|||
:hint "unexpected type on manifest"
|
||||
:manifest manifest))
|
||||
|
||||
|
||||
;; Check if all files referenced on manifest are present
|
||||
(doseq [{file-id :id features :features} (:files manifest)]
|
||||
(let [path (str "files/" file-id ".json")]
|
||||
|
|
@ -907,35 +971,10 @@
|
|||
|
||||
(events/tap :progress {:section :manifest})
|
||||
|
||||
(let [index (bfc/update-index (map :id (:files manifest)))
|
||||
state {:media [] :index index}
|
||||
cfg (-> cfg
|
||||
(assoc ::entries entries)
|
||||
(assoc ::manifest manifest)
|
||||
(assoc ::bfc/timestamp timestamp))]
|
||||
|
||||
(binding [bfc/*state* (volatile! state)]
|
||||
(db/tx-run! cfg (fn [cfg]
|
||||
(bfc/disable-database-timeouts! cfg)
|
||||
(let [ids (->> (:files manifest)
|
||||
(reduce (fn [result {:keys [id] :as file}]
|
||||
(let [name' (get file :name)
|
||||
name' (if (map? name)
|
||||
(get name id)
|
||||
name')]
|
||||
(conj result (-> cfg
|
||||
(assoc ::file-id id)
|
||||
(assoc ::file-name name')
|
||||
(import-file)))))
|
||||
[]))]
|
||||
(import-file-relations cfg)
|
||||
(import-storage-objects cfg)
|
||||
(import-file-media cfg)
|
||||
(import-file-thumbnails cfg)
|
||||
|
||||
(bfm/apply-pending-migrations! cfg)
|
||||
|
||||
ids)))))))
|
||||
(binding [bfc/*state* (volatile! {:media [] :index {}})]
|
||||
(if (::bfc/file-id cfg)
|
||||
(db/tx-run! cfg import-file-and-overwrite*)
|
||||
(db/tx-run! cfg import-files*)))))
|
||||
|
||||
;; --- PUBLIC API
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,9 @@
|
|||
[app.storage :as sto]
|
||||
[app.util.blob :as blob]
|
||||
[app.util.objects-map :as omap]
|
||||
[app.util.pointer-map :as pmap]))
|
||||
[app.util.pointer-map :as pmap]
|
||||
[app.worker :as wrk]
|
||||
[promesa.exec :as px]))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; OFFLOAD
|
||||
|
|
@ -81,6 +83,12 @@
|
|||
(let [data (get-file-data system file)]
|
||||
(assoc file :data data)))
|
||||
|
||||
(defn decode-file-data
|
||||
[{:keys [::wrk/executor]} {:keys [data] :as file}]
|
||||
(cond-> file
|
||||
(bytes? data)
|
||||
(assoc :data (px/invoke! executor #(blob/decode data)))))
|
||||
|
||||
(defn load-pointer
|
||||
"A database loader pointer helper"
|
||||
[system file-id id]
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
"Backend specific code for file migrations. Implemented as permanent feature of files."
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.files.migrations :as fmg :refer [xf:map-name]]
|
||||
[app.db :as db]
|
||||
[app.db.sql :as-alias sql]))
|
||||
|
|
@ -26,12 +27,19 @@
|
|||
(defn upsert-migrations!
|
||||
"Persist or update file migrations. Return the updated/inserted number
|
||||
of rows"
|
||||
[conn {:keys [id] :as file}]
|
||||
(let [migrations (or (-> file meta ::fmg/migrated)
|
||||
(-> file :migrations not-empty)
|
||||
fmg/available-migrations)
|
||||
[cfg {:keys [id] :as file}]
|
||||
(let [conn (db/get-connection cfg)
|
||||
migrations (or (-> file meta ::fmg/migrated)
|
||||
(-> file :migrations))
|
||||
columns [:file-id :name]
|
||||
rows (mapv (fn [name] [id name]) migrations)]
|
||||
rows (->> migrations
|
||||
(mapv (fn [name] [id name]))
|
||||
(not-empty))]
|
||||
|
||||
(when-not rows
|
||||
(ex/raise :type :internal
|
||||
:code :missing-migrations
|
||||
:hint "no migrations available on file"))
|
||||
|
||||
(-> (db/insert-many! conn :file-migration columns rows
|
||||
{::db/return-keys false
|
||||
|
|
@ -40,6 +48,6 @@
|
|||
|
||||
(defn reset-migrations!
|
||||
"Replace file migrations"
|
||||
[conn {:keys [id] :as file}]
|
||||
(db/delete! conn :file-migration {:file-id id})
|
||||
(upsert-migrations! conn file))
|
||||
[cfg {:keys [id] :as file}]
|
||||
(db/delete! cfg :file-migration {:file-id id})
|
||||
(upsert-migrations! cfg file))
|
||||
|
|
|
|||
|
|
@ -125,21 +125,35 @@
|
|||
[:name [:or [:string {:max 250}]
|
||||
[:map-of ::sm/uuid [:string {:max 250}]]]]
|
||||
[:project-id ::sm/uuid]
|
||||
[:file-id {:optional true} ::sm/uuid]
|
||||
[:version {:optional true} ::sm/int]
|
||||
[:file ::media/upload]])
|
||||
|
||||
(sv/defmethod ::import-binfile
|
||||
"Import a penpot file in a binary format."
|
||||
"Import a penpot file in a binary format. If `file-id` is provided,
|
||||
an in-place import will be performed instead of creating a new file.
|
||||
|
||||
The in-place imports are only supported for binfile-v3 and when a
|
||||
.penpot file only contains one penpot file.
|
||||
"
|
||||
{::doc/added "1.15"
|
||||
::doc/changes ["1.20" "Add file-id param for in-place import"
|
||||
"1.20" "Set default version to 3"]
|
||||
|
||||
::webhooks/event? true
|
||||
::sse/stream? true
|
||||
::sm/params schema:import-binfile}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id version file] :as params}]
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id version file-id file] :as params}]
|
||||
(projects/check-edition-permissions! pool profile-id project-id)
|
||||
(let [version (or version 1)
|
||||
(let [version (or version 3)
|
||||
params (-> params
|
||||
(assoc :profile-id profile-id)
|
||||
(assoc :version version))
|
||||
|
||||
cfg (cond-> cfg
|
||||
(uuid? file-id)
|
||||
(assoc ::bfc/file-id file-id))
|
||||
|
||||
manifest (case (int version)
|
||||
1 nil
|
||||
3 (bf.v3/get-manifest (:path file)))]
|
||||
|
|
@ -147,5 +161,6 @@
|
|||
(with-meta
|
||||
(sse/response (partial import-binfile cfg params))
|
||||
{::audit/props {:file nil
|
||||
:file-id file-id
|
||||
:generated-by (:generated-by manifest)
|
||||
:referer (:referer manifest)}})))
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@
|
|||
[conn {:keys [file-id profile-id role]}]
|
||||
(let [params {:file-id file-id
|
||||
:profile-id profile-id}]
|
||||
|
||||
(->> (perms/assign-role-flags params role)
|
||||
(db/insert! conn :file-profile-rel))))
|
||||
|
||||
|
|
@ -51,12 +52,12 @@
|
|||
:is-shared is-shared
|
||||
:features features
|
||||
:ignore-sync-until ignore-sync-until
|
||||
:modified-at modified-at
|
||||
:created-at modified-at
|
||||
:deleted-at deleted-at}
|
||||
{:create-page create-page
|
||||
:page-id page-id})
|
||||
file (-> (bfc/insert-file! cfg file)
|
||||
(bfc/decode-row))]
|
||||
:page-id page-id})]
|
||||
|
||||
(bfc/insert-file! cfg file)
|
||||
|
||||
(->> (assoc params :file-id (:id file) :role :owner)
|
||||
(create-file-role! conn))
|
||||
|
|
|
|||
|
|
@ -61,8 +61,10 @@
|
|||
(declare create)
|
||||
|
||||
(defn create-tracked
|
||||
[]
|
||||
(atom {}))
|
||||
[& {:keys [inherit]}]
|
||||
(if inherit
|
||||
(atom (if *tracked* @*tracked* {}))
|
||||
(atom {})))
|
||||
|
||||
(defprotocol IPointerMap
|
||||
(get-id [_])
|
||||
|
|
|
|||
|
|
@ -102,6 +102,7 @@
|
|||
[:project-id {:optional true} ::sm/uuid]
|
||||
[:team-id {:optional true} ::sm/uuid]
|
||||
[:is-shared {:optional true} ::sm/boolean]
|
||||
[:has-media-trimmed {:optional true} ::sm/boolean]
|
||||
[:data {:optional true} schema:data]
|
||||
[:version :int]
|
||||
[:features ::cfeat/features]
|
||||
|
|
@ -188,6 +189,7 @@
|
|||
:features features
|
||||
:migrations migrations
|
||||
:ignore-sync-until ignore-sync-until
|
||||
:has-media-trimmed false
|
||||
:created-at created-at
|
||||
:modified-at modified-at
|
||||
:deleted-at deleted-at})]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue