🎉 Add binfile-v3 export/import file format

backend/src/app/binfile/common.clj

@@ -37,6 +37,21 @@
 (def ^:dynamic *state* nil)
 (def ^:dynamic *options* nil)
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; DEFAULTS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Threshold in MiB when we pass from using
+;; in-memory byte-array's to use temporal files.
+(def temp-file-threshold
+  (* 1024 1024 2))
+
+;; A maximum (storage) object size allowed: 100MiB
+(def ^:const max-object-size
+  (* 1024 1024 100))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
 (def xf-map-id
   (map :id))
 
@@ -56,6 +71,13 @@
 (def conj-vec
   (fnil conj []))
 
+(defn initial-state
+  []
+  {:storage-objects #{}
+   :files #{}
+   :teams #{}
+   :projects #{}})
+
 (defn collect-storage-objects
   [state items]
   (update state :storage-objects into xf-map-media-id items))
@@ -87,6 +109,8 @@
           attrs))
 
 (defn update-index
+  ([coll]
+   (update-index {} coll identity))
   ([index coll]
    (update-index index coll identity))
   ([index coll attr]
@@ -114,6 +138,16 @@
   [cfg project-id]
   (db/get cfg :project {:id project-id}))
 
+(def ^:private sql:get-teams
+  "SELECT t.* FROM team WHERE id = ANY(?)")
+
+(defn get-teams
+  [cfg ids]
+  (let [conn (db/get-connection cfg)
+        ids  (db/create-array conn "uuid" ids)]
+    (->> (db/exec! conn [sql:get-teams ids])
+         (map decode-row))))
+
 (defn get-team
   [cfg team-id]
   (-> (db/get cfg :team {:id team-id})
@@ -167,9 +201,10 @@
 (defn get-file-object-thumbnails
   "Return all file object thumbnails for a given file."
   [cfg file-id]
-  (db/query cfg :file-tagged-object-thumbnail
-            {:file-id file-id
-             :deleted-at nil}))
+  (->> (db/query cfg :file-tagged-object-thumbnail
+                 {:file-id file-id
+                  :deleted-at nil})
+       (not-empty)))
 
 (defn get-file-thumbnail
   "Return the thumbnail for the specified file-id"
@@ -224,26 +259,26 @@
                    (->> (db/exec! conn [sql ids])
                         (mapv #(assoc % :file-id id)))))))
 
-(def ^:private sql:get-team-files
+(def ^:private sql:get-team-files-ids
   "SELECT f.id FROM file AS f
      JOIN project AS p ON (p.id = f.project_id)
     WHERE p.team_id = ?")
 
-(defn get-team-files
+(defn get-team-files-ids
   "Get a set of file ids for the specified team-id"
   [{:keys [::db/conn]} team-id]
-  (->> (db/exec! conn [sql:get-team-files team-id])
+  (->> (db/exec! conn [sql:get-team-files-ids team-id])
        (into #{} xf-map-id)))
 
 (def ^:private sql:get-team-projects
-  "SELECT p.id FROM project AS p
+  "SELECT p.* FROM project AS p
     WHERE p.team_id = ?
       AND p.deleted_at IS NULL")
 
 (defn get-team-projects
   "Get a set of project ids for the team"
-  [{:keys [::db/conn]} team-id]
-  (->> (db/exec! conn [sql:get-team-projects team-id])
+  [cfg team-id]
+  (->> (db/exec! cfg [sql:get-team-projects team-id])
        (into #{} xf-map-id)))
 
 (def ^:private sql:get-project-files
@@ -257,6 +292,10 @@
   (->> (db/exec! conn [sql:get-project-files project-id])
        (into #{} xf-map-id)))
 
+(defn remap-thumbnail-object-id
+  [object-id file-id]
+  (str/replace-first object-id #"^(.*?)/" (str file-id "/")))
+
 (defn- relink-shapes
   "A function responsible to analyze all file data and
   replace the old :component-file reference with the new
@@ -339,6 +378,12 @@
             data
             library-ids)))
 
+(defn disable-database-timeouts!
+  [cfg]
+  (let [conn (db/get-connection cfg)]
+    (db/exec-one! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"])
+    (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])))
+
 (defn- fix-version
   [file]
   (let [file (fmg/fix-version file)]
@@ -432,6 +477,20 @@
 
     file))
 
+
+(defn register-pending-migrations
+  "All features that are enabled and requires explicit migration are
+  added to the state for a posterior migration step."
+  [cfg {:keys [id features] :as file}]
+  (doseq [feature (-> (::features cfg)
+                      (set/difference cfeat/no-migration-features)
+                      (set/difference cfeat/backend-only-features)
+                      (set/difference features))]
+    (vswap! *state* update :pending-to-migrate (fnil conj []) [feature id]))
+
+  file)
+
+
 (defn apply-pending-migrations!
   "Apply alredy registered pending migrations to files"
   [cfg]

backend/src/app/binfile/v1.clj

@@ -49,15 +49,6 @@
 
 (set! *warn-on-reflection* true)
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; DEFAULTS
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; Threshold in MiB when we pass from using
-;; in-memory byte-array's to use temporal files.
-(def temp-file-threshold
-  (* 1024 1024 2))
-
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; LOW LEVEL STREAM IO API
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -65,11 +56,6 @@
 (def ^:const buffer-size (:xnio/buffer-size yt/defaults))
 (def ^:const penpot-magic-number 800099563638710213)
 
-
-;; A maximum (storage) object size allowed: 100MiB
-(def ^:const max-object-size
-  (* 1024 1024 100))
-
 (def ^:dynamic *position* nil)
 
 (defn get-mark
@@ -258,12 +244,12 @@
         p (tmp/tempfile :prefix "penpot.binfile.")]
     (assert-mark m :stream)
 
-    (when (> s max-object-size)
+    (when (> s bfc/max-object-size)
       (ex/raise :type :validation
                 :code :max-file-size-reached
                 :hint (str/ffmt "unable to import storage object with size % bytes" s)))
 
-    (if (> s temp-file-threshold)
+    (if (> s bfc/temp-file-threshold)
       (with-open [^OutputStream output (io/output-stream p)]
         (let [readed (io/copy! input output :offset 0 :size s)]
           (l/trace :fn "read-stream*!" :expected s :readed readed :position @*position* ::l/sync? true)
@@ -381,10 +367,12 @@
              ::l/sync? true)
 
       (doseq [item media]
-        (l/dbg :hint "write penpot file media object" :id (:id item) ::l/sync? true))
+        (l/dbg :hint "write penpot file media object"
+               :id (:id item) ::l/sync? true))
 
       (doseq [item thumbnails]
-        (l/dbg :hint "write penpot file object thumbnail" :media-id (str (:media-id item)) ::l/sync? true))
+        (l/dbg :hint "write penpot file object thumbnail"
+               :media-id (str (:media-id item)) ::l/sync? true))
 
       (doto output
         (write-obj! file)
@@ -466,8 +454,8 @@
 
 (defn- read-import-v1
   [{:keys [::db/conn ::project-id ::profile-id ::input] :as cfg}]
-  (db/exec-one! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"])
-  (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])
+
+  (bfc/disable-database-timeouts! cfg)
 
   (pu/with-open [input (zstd-input-stream input)
                  input (io/data-input-stream input)]
@@ -559,7 +547,9 @@
 
       (when (seq thumbnails)
         (let [thumbnails (remap-thumbnails thumbnails file-id')]
-          (l/dbg :hint "updated index with thumbnails" :total (count thumbnails) ::l/sync? true)
+          (l/dbg :hint "updated index with thumbnails"
+                 :total (count thumbnails)
+                 ::l/sync? true)
           (vswap! bfc/*state* update :thumbnails bfc/into-vec thumbnails)))
 
       (when (seq media)
@@ -738,7 +728,7 @@
                 :cause @cs)))))
 
 (defn import-files!
-  [cfg input]
+  [{:keys [::input] :as cfg}]
 
   (dm/assert!
    "expected valid profile-id and project-id on `cfg`"

backend/src/app/binfile/v2.clj

@@ -141,16 +141,15 @@
       (write! cfg :team-font-variant id font))))
 
 (defn- write-project!
-  [cfg project-id]
-  (let [project (bfc/get-project cfg project-id)]
-    (events/tap :progress
-                {:op :export
-                 :section :write-project
-                 :id project-id
-                 :name (:name project)})
-    (l/trc :hint "write" :obj "project" :id (str project-id))
-    (write! cfg :project (str project-id) project)
-    (vswap! bfc/*state* update :projects conj project-id)))
+  [cfg project]
+  (events/tap :progress
+              {:op :export
+               :section :write-project
+               :id (:id project)
+               :name (:name project)})
+  (l/trc :hint "write" :obj "project" :id (str (:id project)))
+  (write! cfg :project (str (:id project)) project)
+  (vswap! bfc/*state* update :projects conj (:id project)))
 
 (defn- write-file!
   [cfg file-id]
@@ -363,7 +362,7 @@
                                 (bfc/get-team-projects cfg team-id))
 
                           (run! (partial write-file! cfg)
-                                (bfc/get-team-files cfg team-id))
+                                (bfc/get-team-files-ids cfg team-id))
 
                           (run! (partial write-storage-object! cfg)
                                 (-> bfc/*state* deref :storage-objects))

backend/src/app/binfile/v3.clj

@@ -0,0 +1,957 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.binfile.v3
+  "A ZIP based binary file exportation"
+  (:refer-clojure :exclude [read])
+  (:require
+   [app.binfile.common :as bfc]
+   [app.common.data :as d]
+   [app.common.data.macros :as dm]
+   [app.common.exceptions :as ex]
+   [app.common.json :as json]
+   [app.common.logging :as l]
+   [app.common.schema :as sm]
+   [app.common.thumbnails :as cth]
+   [app.common.types.color :as ctcl]
+   [app.common.types.component :as ctc]
+   [app.common.types.file :as ctf]
+   [app.common.types.page :as ctp]
+   [app.common.types.plugins :as ctpg]
+   [app.common.types.shape :as cts]
+   [app.common.types.typography :as cty]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.storage :as sto]
+   [app.storage.impl :as sto.impl]
+   [app.util.events :as events]
+   [app.util.time :as dt]
+   [clojure.java.io :as jio]
+   [cuerdas.core :as str]
+   [datoteka.fs :as fs]
+   [datoteka.io :as io])
+  (:import
+   java.io.InputStream
+   java.io.OutputStreamWriter
+   java.util.zip.ZipEntry
+   java.util.zip.ZipFile
+   java.util.zip.ZipOutputStream))
+
+;; --- SCHEMA
+
+(def ^:private schema:manifest
+  [:map {:title "Manifest"}
+   [:version ::sm/int]
+   [:type :string]
+
+   [:generated-by {:optional true} :string]
+
+   [:files
+    [:vector
+     [:map
+      [:id ::sm/uuid]
+      [:name :string]
+      [:project-id ::sm/uuid]]]]
+
+   [:relations {:optional true}
+    [:vector
+     [:tuple ::sm/uuid ::sm/uuid]]]])
+
+(def ^:private schema:storage-object
+  [:map {:title "StorageObject"}
+   [:id ::sm/uuid]
+   [:size ::sm/int]
+   [:content-type :string]
+   [:bucket [::sm/one-of {:format :string} sto/valid-buckets]]
+   [:hash :string]])
+
+(def ^:private schema:file-thumbnail
+  [:map {:title "FileThumbnail"}
+   [:file-id ::sm/uuid]
+   [:page-id ::sm/uuid]
+   [:frame-id ::sm/uuid]
+   [:tag :string]
+   [:media-id ::sm/uuid]])
+
+;; --- ENCODERS
+
+(def encode-file
+  (sm/encoder ::ctf/file sm/json-transformer))
+
+(def encode-page
+  (sm/encoder ::ctp/page sm/json-transformer))
+
+(def encode-shape
+  (sm/encoder ::cts/shape sm/json-transformer))
+
+(def encode-media
+  (sm/encoder ::ctf/media sm/json-transformer))
+
+(def encode-component
+  (sm/encoder ::ctc/component sm/json-transformer))
+
+(def encode-color
+  (sm/encoder ::ctcl/color sm/json-transformer))
+
+(def encode-typography
+  (sm/encoder ::cty/typography sm/json-transformer))
+
+(def encode-plugin-data
+  (sm/encoder ::ctpg/plugin-data sm/json-transformer))
+
+(def encode-storage-object
+  (sm/encoder schema:storage-object sm/json-transformer))
+
+(def encode-file-thumbnail
+  (sm/encoder schema:file-thumbnail sm/json-transformer))
+
+;; --- DECODERS
+
+(def decode-manifest
+  (sm/decoder schema:manifest sm/json-transformer))
+
+(def decode-media
+  (sm/decoder ::ctf/media sm/json-transformer))
+
+(def decode-component
+  (sm/decoder ::ctc/component sm/json-transformer))
+
+(def decode-color
+  (sm/decoder ::ctcl/color sm/json-transformer))
+
+(def decode-file
+  (sm/decoder ::ctf/file sm/json-transformer))
+
+(def decode-page
+  (sm/decoder ::ctp/page sm/json-transformer))
+
+(def decode-shape
+  (sm/decoder ::cts/shape sm/json-transformer))
+
+(def decode-typography
+  (sm/decoder ::cty/typography sm/json-transformer))
+
+(def decode-plugin-data
+  (sm/decoder ::ctpg/plugin-data sm/json-transformer))
+
+(def decode-storage-object
+  (sm/decoder schema:storage-object sm/json-transformer))
+
+(def decode-file-thumbnail
+  (sm/decoder schema:file-thumbnail sm/json-transformer))
+
+;; --- VALIDATORS
+
+(def validate-manifest
+  (sm/check-fn schema:manifest))
+
+(def validate-file
+  (sm/check-fn ::ctf/file))
+
+(def validate-page
+  (sm/check-fn ::ctp/page))
+
+(def validate-shape
+  (sm/check-fn ::cts/shape))
+
+(def validate-media
+  (sm/check-fn ::ctf/media))
+
+(def validate-color
+  (sm/check-fn ::ctcl/color))
+
+(def validate-component
+  (sm/check-fn ::ctc/component))
+
+(def validate-typography
+  (sm/check-fn ::cty/typography))
+
+(def validate-plugin-data
+  (sm/check-fn ::ctpg/plugin-data))
+
+(def validate-storage-object
+  (sm/check-fn schema:storage-object))
+
+(def validate-file-thumbnail
+  (sm/check-fn schema:file-thumbnail))
+
+;; --- EXPORT IMPL
+
+(defn- write-entry!
+  [^ZipOutputStream output ^String path data]
+  (.putNextEntry output (ZipEntry. path))
+  (let [writer (OutputStreamWriter. output "UTF-8")]
+    (json/write writer data :indent true :key-fn json/write-camel-key)
+    (.flush writer))
+  (.closeEntry output))
+
+(defn- get-file
+  [{:keys [::embed-assets ::include-libraries] :as cfg} file-id]
+
+  (when (and include-libraries embed-assets)
+    (throw (IllegalArgumentException.
+            "the `include-libraries` and `embed-assets` are mutally excluding options")))
+
+  (let [detach?  (and (not embed-assets) (not include-libraries))]
+    (cond-> (bfc/get-file cfg file-id)
+      detach?
+      (-> (ctf/detach-external-references file-id)
+          (dissoc :libraries))
+
+      embed-assets
+      (update :data #(bfc/embed-assets cfg % file-id)))))
+
+(defn- resolve-extension
+  [mtype]
+  (case mtype
+    "image/png"     ".png"
+    "image/jpeg"    ".jpg"
+    "image/gif"     ".gif"
+    "image/svg+xml" ".svg"
+    "image/webp"    ".webp"
+    "font/woff"     ".woff"
+    "font/woff2"    ".woff2"
+    "font/ttf"      ".ttf"
+    "font/otf"      ".otf"
+    "application/octet-stream" ".bin"))
+
+(defn- export-storage-objects
+  [{:keys [::output] :as cfg}]
+  (let [storage (sto/resolve cfg)]
+    (doseq [id (-> bfc/*state* deref :storage-objects not-empty)]
+      (let [sobject (sto/get-object storage id)
+            smeta   (meta sobject)
+            ext     (resolve-extension (:content-type smeta))
+            path    (str "objects/" id ".json")
+            params  (-> (meta sobject)
+                        (assoc :id (:id sobject))
+                        (assoc :size (:size sobject))
+                        (encode-storage-object))]
+
+        (write-entry! output path params)
+
+        (with-open [input (sto/get-object-data storage sobject)]
+          (.putNextEntry output (ZipEntry. (str "objects/" id ext)))
+          (io/copy! input output (:size sobject))
+          (.closeEntry output))))))
+
+(defn- export-file
+  [{:keys [::file-id ::output] :as cfg}]
+  (let [file         (get-file cfg file-id)
+        media        (->> (bfc/get-file-media cfg file)
+                          (map (fn [media]
+                                 (dissoc media :file-id))))
+
+        data         (:data file)
+        typographies (:typographies data)
+        plugins-data (:plugin-data data)
+        components   (:components data)
+        colors       (:colors data)
+
+        pages        (:pages data)
+        pages-index  (:pages-index data)
+
+        thumbnails   (bfc/get-file-object-thumbnails cfg file-id)]
+
+    (vswap! bfc/*state* update :files assoc file-id
+            {:id file-id
+             :project-id (:project-id file)
+             :name (:name file)})
+
+    (let [file (cond-> (dissoc file :data)
+                 (:options data)
+                 (assoc :options (:options data))
+                 :always
+                 (encode-file))
+          path (str "files/" file-id ".json")]
+      (write-entry! output path file))
+
+    (doseq [[index page-id] (d/enumerate pages)]
+      (let [path    (str "files/" file-id "/pages/" page-id ".json")
+            page    (get pages-index page-id)
+            objects (:objects page)
+            page    (-> page
+                        (dissoc :objects)
+                        (assoc :index index))
+            page    (encode-page page)]
+
+        (write-entry! output path page)
+
+        (doseq [[shape-id shape] objects]
+          (let [path  (str "files/" file-id "/pages/" page-id "/" shape-id ".json")
+                shape (assoc shape :page-id page-id)
+                shape (encode-shape shape)]
+            (write-entry! output path shape)))))
+
+    (vswap! bfc/*state* bfc/collect-storage-objects media)
+    (vswap! bfc/*state* bfc/collect-storage-objects thumbnails)
+
+    (doseq [{:keys [id] :as media} media]
+      (let [path  (str "files/" file-id "/media/" id ".json")
+            media (encode-media media)]
+        (write-entry! output path media)))
+
+    (doseq [thumbnail thumbnails]
+      (let [data (cth/parse-object-id (:object-id thumbnail))
+            path (str "files/" file-id "/thumbnails/" (:page-id data)
+                      "/" (:frame-id data) ".json")
+            data (-> data
+                     (assoc :media-id (:media-id thumbnail))
+                     (encode-file-thumbnail))]
+        (write-entry! output path data)))
+
+    (doseq [[id component] components]
+      (let [path      (str "files/" file-id "/components/" id ".json")
+            component (encode-component component)]
+        (write-entry! output path component)))
+
+    (doseq [[id color] colors]
+      (let [path  (str "files/" file-id "/colors/" id ".json")
+            color (-> (encode-color color)
+                      (dissoc :file-id))
+            color (cond-> color
+                    (and (contains? color :path)
+                         (str/empty? (:path color)))
+                    (dissoc :path))]
+        (write-entry! output path color)))
+
+    (doseq [[id object] typographies]
+      (let [path  (str "files/" file-id "/typographies/" id ".json")
+            color (encode-typography object)]
+        (write-entry! output path color)))
+
+    (when-let [data (not-empty plugins-data)]
+      (let [path (str "files/" file-id "/plugin-data.json")]
+        (write-entry! output path data)))))
+
+(defn- export-files
+  [{:keys [::ids ::include-libraries ::output] :as cfg}]
+  (let [ids  (into ids (when include-libraries (bfc/get-libraries cfg ids)))
+        rels (if include-libraries
+               (->> (bfc/get-files-rels cfg ids)
+                    (mapv (juxt :file-id :library-file-id)))
+               [])]
+
+    (vswap! bfc/*state* assoc :files (d/ordered-map))
+
+    ;; Write all the exporting files
+    (doseq [[index file-id] (d/enumerate ids)]
+      (-> cfg
+          (assoc ::file-id file-id)
+          (assoc ::file-seqn index)
+          (export-file)))
+
+    ;; Write manifest file
+    (let [files  (:files @bfc/*state*)
+          params {:type "penpot/export-files"
+                  :version 1
+                  :generated-by (str "penpot/" (:full cf/version))
+                  :files (vec (vals files))
+                  :relations rels}]
+      (write-entry! output "manifest.json" params))))
+
+;; --- IMPORT IMPL
+
+(defn- read-zip-entries
+  [^ZipFile input]
+  (into #{} (iterator-seq (.entries input))))
+
+(defn- get-zip-entry*
+  [^ZipFile input ^String path]
+  (.getEntry input path))
+
+(defn- get-zip-entry
+  [input path]
+  (let [entry (get-zip-entry* input path)]
+    (when-not entry
+      (ex/raise :type :validation
+                :code :inconsistent-penpot-file
+                :hint "the penpot file seems corrupt, missing underlying zip entry"
+                :path path))
+    entry))
+
+(defn- get-zip-entry-size
+  [^ZipEntry entry]
+  (.getSize entry))
+
+(defn- zip-entry-name
+  [^ZipEntry entry]
+  (.getName entry))
+
+(defn- zip-entry-stream
+  ^InputStream
+  [^ZipFile input ^ZipEntry entry]
+  (.getInputStream input entry))
+
+(defn- zip-entry-reader
+  [^ZipFile input ^ZipEntry entry]
+  (-> (zip-entry-stream input entry)
+      (jio/reader :encoding "UTF-8")))
+
+(defn- zip-entry-storage-content
+  "Wraps a ZipFile and ZipEntry into a penpot storage compatible
+  object and avoid creating temporal objects"
+  [input entry]
+  (let [hash  (delay (->> entry
+                          (zip-entry-stream input)
+                          (sto.impl/calculate-hash)))]
+    (reify
+      sto.impl/IContentObject
+      (get-size [_]
+        (get-zip-entry-size entry))
+
+      sto.impl/IContentHash
+      (get-hash [_]
+        (deref hash))
+
+      jio/IOFactory
+      (make-reader [this opts]
+        (jio/make-reader this opts))
+      (make-writer [_ _]
+        (throw (UnsupportedOperationException. "not implemented")))
+
+      (make-input-stream [_ _]
+        (zip-entry-stream input entry))
+      (make-output-stream [_ _]
+        (throw (UnsupportedOperationException. "not implemented"))))))
+
+(defn- read-manifest
+  [^ZipFile input]
+  (let [entry (get-zip-entry input "manifest.json")]
+    (with-open [reader (zip-entry-reader input entry)]
+      (let [manifest (json/read reader :key-fn json/read-kebab-key)]
+        (decode-manifest manifest)))))
+
+(defn- match-media-entry-fn
+  [file-id]
+  (let [pattern (str "^files/" file-id "/media/([^/]+).json$")
+        pattern (re-pattern pattern)]
+    (fn [entry]
+      (when-let [[_ id] (re-matches pattern (zip-entry-name entry))]
+        {:entry entry
+         :id (parse-uuid id)}))))
+
+(defn- match-color-entry-fn
+  [file-id]
+  (let [pattern (str "^files/" file-id "/colors/([^/]+).json$")
+        pattern (re-pattern pattern)]
+    (fn [entry]
+      (when-let [[_ id] (re-matches pattern (zip-entry-name entry))]
+        {:entry entry
+         :id (parse-uuid id)}))))
+
+(defn- match-component-entry-fn
+  [file-id]
+  (let [pattern (str "^files/" file-id "/components/([^/]+).json$")
+        pattern (re-pattern pattern)]
+    (fn [entry]
+      (when-let [[_ id] (re-matches pattern (zip-entry-name entry))]
+        {:entry entry
+         :id (parse-uuid id)}))))
+
+(defn- match-typography-entry-fn
+  [file-id]
+  (let [pattern (str "^files/" file-id "/typographies/([^/]+).json$")
+        pattern (re-pattern pattern)]
+    (fn [entry]
+      (when-let [[_ id] (re-matches pattern (zip-entry-name entry))]
+        {:entry entry
+         :id (parse-uuid id)}))))
+
+(defn- match-thumbnail-entry-fn
+  [file-id]
+  (let [pattern (str "^files/" file-id "/thumbnails/([^/]+)/([^/]+).json$")
+        pattern (re-pattern pattern)]
+    (fn [entry]
+      (when-let [[_ page-id frame-id] (re-matches pattern (zip-entry-name entry))]
+        {:entry entry
+         :page-id (parse-uuid page-id)
+         :frame-id (parse-uuid frame-id)
+         :file-id file-id}))))
+
+(defn- match-page-entry-fn
+  [file-id]
+  (let [pattern (str "^files/" file-id "/pages/([^/]+).json$")
+        pattern (re-pattern pattern)]
+    (fn [entry]
+      (when-let [[_ id] (re-matches pattern (zip-entry-name entry))]
+        {:entry entry
+         :id (parse-uuid id)}))))
+
+(defn- match-shape-entry-fn
+  [file-id page-id]
+  (let [pattern (str "^files/" file-id "/pages/" page-id "/([^/]+).json$")
+        pattern (re-pattern pattern)]
+    (fn [entry]
+      (when-let [[_ id] (re-matches pattern (zip-entry-name entry))]
+        {:entry entry
+         :page-id page-id
+         :id (parse-uuid id)}))))
+
+(defn- match-storage-entry-fn
+  []
+  (let [pattern (str "^objects/([^/]+).json$")
+        pattern (re-pattern pattern)]
+    (fn [entry]
+      (when-let [[_ id] (re-matches pattern (zip-entry-name entry))]
+        {:entry entry
+         :id (parse-uuid id)}))))
+
+(defn- read-entry
+  [^ZipFile input entry]
+  (with-open [reader (zip-entry-reader input entry)]
+    (json/read reader :key-fn json/read-kebab-key)))
+
+(defn- read-file
+  [{:keys [::input ::file-id]}]
+  (let [path  (str "files/" file-id ".json")
+        entry (get-zip-entry input path)]
+    (-> (read-entry input entry)
+        (decode-file)
+        (validate-file))))
+
+(defn- read-file-plugin-data
+  [{:keys [::input ::file-id]}]
+  (let [path  (str "files/" file-id "/plugin-data.json")
+        entry (get-zip-entry* input path)]
+    (some->> entry
+             (read-entry input)
+             (decode-plugin-data)
+             (validate-plugin-data))))
+
+(defn- read-file-media
+  [{:keys [::input ::file-id ::entries]}]
+  (->> (keep (match-media-entry-fn file-id) entries)
+       (reduce (fn [result {:keys [id entry]}]
+                 (let [object (->> (read-entry input entry)
+                                   (decode-media)
+                                   (validate-media))
+                       object (assoc object :file-id file-id)]
+                   (if (= id (:id object))
+                     (conj result object)
+                     result)))
+               [])
+       (not-empty)))
+
+(defn- read-file-colors
+  [{:keys [::input ::file-id ::entries]}]
+  (->> (keep (match-color-entry-fn file-id) entries)
+       (reduce (fn [result {:keys [id entry]}]
+                 (let [object (->> (read-entry input entry)
+                                   (decode-color)
+                                   (validate-color))]
+                   (if (= id (:id object))
+                     (assoc result id object)
+                     result)))
+               {})
+       (not-empty)))
+
+(defn- read-file-components
+  [{:keys [::input ::file-id ::entries]}]
+  (->> (keep (match-component-entry-fn file-id) entries)
+       (reduce (fn [result {:keys [id entry]}]
+                 (let [object (->> (read-entry input entry)
+                                   (decode-component)
+                                   (validate-component))]
+                   (if (= id (:id object))
+                     (assoc result id object)
+                     result)))
+               {})
+       (not-empty)))
+
+(defn- read-file-typographies
+  [{:keys [::input ::file-id ::entries]}]
+  (->> (keep (match-typography-entry-fn file-id) entries)
+       (reduce (fn [result {:keys [id entry]}]
+                 (let [object (->> (read-entry input entry)
+                                   (decode-typography)
+                                   (validate-typography))]
+                   (if (= id (:id object))
+                     (assoc result id object)
+                     result)))
+               {})
+       (not-empty)))
+
+(defn- read-file-shapes
+  [{:keys [::input ::file-id ::page-id ::entries] :as cfg}]
+  (->> (keep (match-shape-entry-fn file-id page-id) entries)
+       (reduce (fn [result {:keys [id entry]}]
+                 (let [object (->> (read-entry input entry)
+                                   (decode-shape)
+                                   (validate-shape))]
+                   (if (= id (:id object))
+                     (assoc result id object)
+                     result)))
+               {})
+       (not-empty)))
+
+(defn- read-file-pages
+  [{:keys [::input ::file-id ::entries] :as cfg}]
+  (->> (keep (match-page-entry-fn file-id) entries)
+       (keep (fn [{:keys [id entry]}]
+               (let [page (->> (read-entry input entry)
+                               (decode-page))
+                     page (dissoc page :options)]
+                 (when (= id (:id page))
+                   (let [objects (-> (assoc cfg ::page-id id)
+                                     (read-file-shapes))]
+                     (assoc page :objects objects))))))
+       (sort-by :index)
+       (reduce (fn [result {:keys [id] :as page}]
+                 (assoc result id (dissoc page :index)))
+               (d/ordered-map))))
+
+(defn- read-file-thumbnails
+  [{:keys [::input ::file-id ::entries] :as cfg}]
+  (->> (keep (match-thumbnail-entry-fn file-id) entries)
+       (reduce (fn [result {:keys [page-id frame-id entry]}]
+                 (let [object (->> (read-entry input entry)
+                                   (decode-file-thumbnail)
+                                   (validate-file-thumbnail))]
+                   (if (and (= frame-id (:frame-id object))
+                            (= page-id (:page-id object)))
+                     (conj result object)
+                     result)))
+               [])
+       (not-empty)))
+
+(defn- read-file-data
+  [{:keys [] :as cfg}]
+  (let [colors       (read-file-colors cfg)
+        typographies (read-file-typographies cfg)
+        components   (read-file-components cfg)
+        plugin-data  (read-file-plugin-data cfg)
+        pages        (read-file-pages cfg)]
+
+    {:pages (-> pages keys vec)
+     :pages-index (into {} pages)
+     :colors colors
+     :typographies typographies
+     :components components
+     :plugin-data plugin-data}))
+
+(defn- import-file
+  [{:keys [::db/conn ::project-id ::file-id ::file-name] :as cfg}]
+  (let [file-id'   (bfc/lookup-index file-id)
+        file       (read-file cfg)
+        media      (read-file-media cfg)
+        thumbnails (read-file-thumbnails cfg)]
+
+    (l/dbg :hint "processing file"
+           :id (str file-id')
+           :prev-id (str file-id)
+           :features (str/join "," (:features file))
+           :version (:version file)
+           ::l/sync? true)
+
+    (events/tap :progress {:op :import :section :file :name file-name})
+
+    (when media
+      ;; Update index with media
+      (l/dbg :hint "update media index"
+             :file-id (str file-id')
+             :total (count media)
+             ::l/sync? true)
+
+      (vswap! bfc/*state* update :index bfc/update-index (map :id media))
+      (vswap! bfc/*state* update :media into media))
+
+    (when thumbnails
+      (l/dbg :hint "update thumbnails index"
+             :file-id (str file-id')
+             :total (count thumbnails)
+             ::l/sync? true)
+
+      (vswap! bfc/*state* update :index bfc/update-index (map :media-id thumbnails))
+      (vswap! bfc/*state* update :thumbnails into thumbnails))
+
+    (let [data (-> (read-file-data cfg)
+                   (d/without-nils)
+                   (assoc :id file-id')
+                   (cond-> (:options file)
+                     (assoc :options (:options file))))
+
+          file (-> file
+                   (assoc :id file-id')
+                   (assoc :data data)
+                   (assoc :name file-name)
+                   (assoc :project-id project-id)
+                   (dissoc :options)
+                   (bfc/process-file))]
+
+      (->> file
+           (bfc/register-pending-migrations cfg)
+           (bfc/persist-file! cfg))
+
+      (when (::bfc/overwrite cfg)
+        (db/delete! conn :file-thumbnail {:file-id file-id'}))
+
+      file-id')))
+
+(defn- import-file-relations
+  [{:keys [::db/conn ::manifest ::bfc/timestamp] :as cfg}]
+  (events/tap :progress {:op :import :section :relations})
+
+  (doseq [[file-id libr-id] (:relations manifest)]
+
+    (let [file-id (bfc/lookup-index file-id)
+          libr-id (bfc/lookup-index libr-id)]
+
+      (when (and file-id libr-id)
+        (l/dbg :hint "create file library link"
+               :file-id (str file-id)
+               :lib-id (str libr-id)
+               ::l/sync? true)
+        (db/insert! conn :file-library-rel
+                    {:synced-at timestamp
+                     :file-id file-id
+                     :library-file-id libr-id})))))
+
+(defn- import-storage-objects
+  [{:keys [::input ::entries ::bfc/timestamp] :as cfg}]
+  (events/tap :progress {:op :import :section :storage-objects})
+
+  (let [storage (sto/resolve cfg)
+        entries (keep (match-storage-entry-fn) entries)]
+
+    (doseq [{:keys [id entry]} entries]
+      (let [object (->> (read-entry input entry)
+                        (decode-storage-object)
+                        (validate-storage-object))]
+
+        (when (not= id (:id object))
+          (ex/raise :type :validation
+                    :code :inconsistent-penpot-file
+                    :hint "the penpot file seems corrupt, found unexpected uuid (storage-object-id)"
+                    :expected-id (str id)
+                    :found-id (str (:id object))))
+
+        (let [ext     (resolve-extension (:content-type object))
+              path    (str "objects/" id ext)
+              content (->> path
+                           (get-zip-entry input)
+                           (zip-entry-storage-content input))]
+
+          (when (not= (:size object) (sto/get-size content))
+            (ex/raise :type :validation
+                      :code :inconsistent-penpot-file
+                      :hint "found corrupted storage object: size does not match"
+                      :path path
+                      :expected-size (:size object)
+                      :found-size (sto/get-size content)))
+
+          (when (not= (:hash object) (sto/get-hash content))
+            (ex/raise :type :validation
+                      :code :inconsistent-penpot-file
+                      :hint "found corrupted storage object: hash does not match"
+                      :path path
+                      :expected-hash (:hash object)
+                      :found-hash (sto/get-hash content)))
+
+          (let [params  (-> object
+                            (dissoc :id :size)
+                            (assoc ::sto/content content)
+                            (assoc ::sto/deduplicate? true)
+                            (assoc ::sto/touched-at timestamp))
+                sobject (sto/put-object! storage params)]
+
+            (l/dbg :hint "persisted storage object"
+                   :id (str (:id sobject))
+                   :prev-id (str id)
+                   :bucket (:bucket params)
+                   ::l/sync? true)
+
+            (vswap! bfc/*state* update :index assoc id (:id sobject))))))))
+
+(defn- import-file-media
+  [{:keys [::db/conn] :as cfg}]
+  (events/tap :progress {:op :import :section :media})
+
+  (doseq [item (:media @bfc/*state*)]
+    (let [params (-> item
+                     (update :id bfc/lookup-index)
+                     (update :file-id bfc/lookup-index)
+                     (d/update-when :media-id bfc/lookup-index)
+                     (d/update-when :thumbnail-id bfc/lookup-index))]
+
+      (l/dbg :hint "inserting file media object"
+             :id (str (:id params))
+             :file-id (str (:file-id params))
+             ::l/sync? true)
+
+      (db/insert! conn :file-media-object params
+                  {::db/on-conflict-do-nothing? (::bfc/overwrite cfg)}))))
+
+(defn- import-file-thumbnails
+  [{:keys [::db/conn] :as cfg}]
+  (events/tap :progress {:op :import :section :thumbnails})
+  (doseq [item (:thumbnails @bfc/*state*)]
+    (let [file-id   (bfc/lookup-index (:file-id item))
+          media-id  (bfc/lookup-index (:media-id item))
+          object-id (-> (assoc item :file-id file-id)
+                        (cth/fmt-object-id))
+
+          params    {:file-id file-id
+                     :object-id object-id
+                     :tag (:tag item)
+                     :media-id media-id}]
+
+      (l/dbg :hint "inserting file object thumbnail"
+             :file-id (str file-id)
+             :media-id (str media-id)
+             ::l/sync? true)
+
+      (db/insert! conn :file-tagged-object-thumbnail params
+                  {::db/on-conflict-do-nothing? (::bfc/overwrite cfg)}))))
+
+(defn- import-files
+  [{:keys [::bfc/timestamp ::input ::name] :or {timestamp (dt/now)} :as cfg}]
+
+  (dm/assert!
+   "expected zip file"
+   (instance? ZipFile input))
+
+  (dm/assert!
+   "expected valid instant"
+   (dt/instant? timestamp))
+
+  (let [manifest (-> (read-manifest input)
+                     (validate-manifest))
+        entries  (read-zip-entries input)]
+
+    (when-not (= "penpot/export-files" (:type manifest))
+      (ex/raise :type :validation
+                :code :invalid-binfile-v3-manifest
+                :hint "unexpected type on manifest"
+                :manifest manifest))
+
+    ;; Check if all files referenced on manifest are present
+    (doseq [{file-id :id} (:files manifest)]
+      (let [path (str "files/" file-id ".json")]
+        (when-not (get-zip-entry input path)
+          (ex/raise :type :validation
+                    :code :invalid-binfile-v3
+                    :hint "some files referenced on manifest not found"
+                    :path path
+                    :file-id file-id))))
+
+    (events/tap :progress {:op :import :section :manifest})
+
+    (let [index (bfc/update-index (map :id (:files manifest)))
+          state {:media [] :index index}
+          cfg   (-> cfg
+                    (assoc ::entries entries)
+                    (assoc ::manifest manifest)
+                    (assoc ::bfc/timestamp timestamp))]
+
+      (binding [bfc/*state* (volatile! state)]
+        (db/tx-run! cfg (fn [cfg]
+                          (bfc/disable-database-timeouts! cfg)
+                          (let [ids (->> (:files manifest)
+                                         (reduce (fn [result {:keys [id] :as file}]
+                                                   (let [name' (get file :name)
+                                                         name' (if (map? name)
+                                                                 (get name id)
+                                                                 name')]
+                                                     (conj result (-> cfg
+                                                                      (assoc ::file-id id)
+                                                                      (assoc ::file-name name')
+                                                                      (import-file)))))
+                                                 []))]
+                            (import-file-relations cfg)
+                            (import-storage-objects cfg)
+                            (import-file-media cfg)
+                            (import-file-thumbnails cfg)
+
+                            (bfc/apply-pending-migrations! cfg)
+
+                            ids)))))))
+
+;; --- PUBLIC API
+
+(defn export-files!
+  "Do the exportation of a specified file in custom penpot binary
+  format. There are some options available for customize the output:
+
+  `::include-libraries`: additionally to the specified file, all the
+  linked libraries also will be included (including transitive
+  dependencies).
+
+  `::embed-assets`: instead of including the libraries, embed in the
+  same file library all assets used from external libraries."
+
+  [{:keys [::ids] :as cfg} output]
+
+  (dm/assert!
+   "expected a set of uuid's for `::ids` parameter"
+   (and (set? ids)
+        (every? uuid? ids)))
+
+  (dm/assert!
+   "expected instance of jio/IOFactory for `input`"
+   (satisfies? jio/IOFactory output))
+
+  (let [id (uuid/next)
+        tp (dt/tpoint)
+        ab (volatile! false)
+        cs (volatile! nil)]
+    (try
+      (l/info :hint "start exportation" :export-id (str id))
+      (binding [bfc/*state* (volatile! (bfc/initial-state))]
+        (with-open [output (io/output-stream output)]
+          (with-open [output (ZipOutputStream. output)]
+            (let [cfg (assoc cfg ::output output)]
+              (export-files cfg)
+              (export-storage-objects cfg)))))
+
+      (catch java.io.IOException _cause
+        ;; Do nothing, EOF means client closes connection abruptly
+        (vreset! ab true)
+        nil)
+
+      (catch Throwable cause
+        (vreset! cs cause)
+        (vreset! ab true)
+        (throw cause))
+
+      (finally
+        (l/info :hint "exportation finished" :export-id (str id)
+                :elapsed (str (inst-ms (tp)) "ms")
+                :aborted @ab
+                :cause @cs)))))
+
+
+(defn import-files!
+  [{:keys [::input] :as cfg}]
+
+  (dm/assert!
+   "expected valid profile-id and project-id on `cfg`"
+   (and (uuid? (::profile-id cfg))
+        (uuid? (::project-id cfg))))
+
+  (dm/assert!
+   "expected instance of jio/IOFactory for `input`"
+   (satisfies? jio/IOFactory input))
+
+  (let [id (uuid/next)
+        tp (dt/tpoint)
+        cs (volatile! nil)]
+
+    (l/info :hint "import: started" :id (str id))
+    (try
+      (with-open [input (ZipFile. (fs/file input))]
+        (import-files (assoc cfg ::input input)))
+
+      (catch Throwable cause
+        (vreset! cs cause)
+        (throw cause))
+
+      (finally
+        (l/info :hint "import: terminated"
+                :id (str id)
+                :elapsed (dt/format-duration (tp))
+                :error? (some? @cs))))))

backend/src/app/http/debug.clj

@@ -295,8 +295,9 @@
               cfg        (assoc cfg
                                 ::bf.v1/overwrite false
                                 ::bf.v1/profile-id profile-id
-                                ::bf.v1/project-id project-id)]
-          (bf.v1/import-files! cfg path)
+                                ::bf.v1/project-id project-id
+                                ::bf.v1/input path)]
+          (bf.v1/import-files! cfg)
           {::rres/status  200
            ::rres/headers {"content-type" "text/plain"}
            ::rres/body    "OK CLONED"})
@@ -329,8 +330,9 @@
                       ::bf.v1/overwrite overwrite?
                       ::bf.v1/migrate migrate?
                       ::bf.v1/profile-id profile-id
-                      ::bf.v1/project-id project-id)]
-      (bf.v1/import-files! cfg path)
+                      ::bf.v1/project-id project-id
+                      ::bf.v1/input path)]
+      (bf.v1/import-files! cfg)
       {::rres/status  200
        ::rres/headers {"content-type" "text/plain"}
        ::rres/body    "OK"})))

backend/src/app/rpc/commands/binfile.clj

@@ -8,6 +8,7 @@
   (:refer-clojure :exclude [assert])
   (:require
    [app.binfile.v1 :as bf.v1]
+   [app.binfile.v3 :as bf.v3]
    [app.common.logging :as l]
    [app.common.schema :as sm]
    [app.db :as db]
@@ -35,51 +36,103 @@
   [:map {:title "export-binfile"}
    [:name [:string {:max 250}]]
    [:file-id ::sm/uuid]
-   [:include-libraries :boolean]
-   [:embed-assets :boolean]])
+   [:version {:optional true} ::sm/int]
+   [:include-libraries ::sm/boolean]
+   [:embed-assets ::sm/boolean]])
+
+(defn stream-export-v1
+  [cfg {:keys [file-id include-libraries embed-assets] :as params}]
+  (reify rres/StreamableResponseBody
+    (-write-body-to-stream [_ _ output-stream]
+      (try
+        (-> cfg
+            (assoc ::bf.v1/ids #{file-id})
+            (assoc ::bf.v1/embed-assets embed-assets)
+            (assoc ::bf.v1/include-libraries include-libraries)
+            (bf.v1/export-files! output-stream))
+        (catch Throwable cause
+          (l/err :hint "exception on exporting file"
+                 :file-id (str file-id)
+                 :cause cause))))))
+
+(defn stream-export-v3
+  [cfg {:keys [file-id include-libraries embed-assets] :as params}]
+  (reify rres/StreamableResponseBody
+    (-write-body-to-stream [_ _ output-stream]
+      (try
+        (-> cfg
+            (assoc ::bf.v3/ids #{file-id})
+            (assoc ::bf.v3/embed-assets embed-assets)
+            (assoc ::bf.v3/include-libraries include-libraries)
+            (bf.v3/export-files! output-stream))
+        (catch Throwable cause
+          (l/err :hint "exception on exporting file"
+                 :file-id (str file-id)
+                 :cause cause))))))
 
 (sv/defmethod ::export-binfile
   "Export a penpot file in a binary format."
   {::doc/added "1.15"
    ::webhooks/event? true
    ::sm/result schema:export-binfile}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id include-libraries embed-assets] :as params}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id version file-id] :as params}]
   (files/check-read-permissions! pool profile-id file-id)
   (fn [_]
-    {::rres/status 200
-     ::rres/headers {"content-type" "application/octet-stream"}
-     ::rres/body (reify rres/StreamableResponseBody
-                   (-write-body-to-stream [_ _ output-stream]
-                     (try
-                       (-> cfg
-                           (assoc ::bf.v1/ids #{file-id})
-                           (assoc ::bf.v1/embed-assets embed-assets)
-                           (assoc ::bf.v1/include-libraries include-libraries)
-                           (bf.v1/export-files! output-stream))
-                       (catch Throwable cause
-                         (l/err :hint "exception on exporting file"
-                                :file-id (str file-id)
-                                :cause cause)))))}))
+    (let [version (or version 1)
+          body    (case (int version)
+                    1 (stream-export-v1 cfg params)
+                    2 (throw (ex-info "not-implemented" {}))
+                    3 (stream-export-v3 cfg params))]
+
+      {::rres/status 200
+       ::rres/headers {"content-type" "application/octet-stream"}
+       ::rres/body body})))
 
 ;; --- Command: import-binfile
 
+(defn- import-binfile-v1
+  [{:keys [::wrk/executor] :as cfg} {:keys [project-id profile-id name file]}]
+  (let [cfg (-> cfg
+                (assoc ::bf.v1/project-id project-id)
+                (assoc ::bf.v1/profile-id profile-id)
+                (assoc ::bf.v1/name name)
+                (assoc ::bf.v1/input (:path file)))]
+
+    ;; NOTE: the importation process performs some operations that are
+    ;; not very friendly with virtual threads, and for avoid
+    ;; unexpected blocking of other concurrent operations we dispatch
+    ;; that operation to a dedicated executor.
+    (px/invoke! executor (partial bf.v1/import-files! cfg))))
+
+(defn- import-binfile-v3
+  [{:keys [::wrk/executor] :as cfg} {:keys [project-id profile-id name file]}]
+  (let [cfg (-> cfg
+                (assoc ::bf.v3/project-id project-id)
+                (assoc ::bf.v3/profile-id profile-id)
+                (assoc ::bf.v3/name name)
+                (assoc ::bf.v3/input (:path file)))]
+    ;; NOTE: the importation process performs some operations that are
+    ;; not very friendly with virtual threads, and for avoid
+    ;; unexpected blocking of other concurrent operations we dispatch
+    ;; that operation to a dedicated executor.
+    (px/invoke! executor (partial bf.v3/import-files! cfg))))
+
 (defn- import-binfile
-  [{:keys [::wrk/executor ::bf.v1/project-id ::db/pool] :as cfg} input]
-  ;; NOTE: the importation process performs some operations that
-  ;; are not very friendly with virtual threads, and for avoid
-  ;; unexpected blocking of other concurrent operations we
-  ;; dispatch that operation to a dedicated executor.
-  (let [result (px/invoke! executor (partial bf.v1/import-files! cfg input))]
+  [{:keys [::db/pool] :as cfg} {:keys [project-id version] :as params}]
+  (let [result (case (int version)
+                 1 (import-binfile-v1 cfg params)
+                 3 (import-binfile-v3 cfg params))]
     (db/update! pool :project
                 {:modified-at (dt/now)}
                 {:id project-id})
     result))
 
-(def ^:private
-  schema:import-binfile
+(def ^:private schema:import-binfile
   [:map {:title "import-binfile"}
-   [:name [:string {:max 250}]]
+   [:name [:or [:string {:max 250}]
+           [:map-of ::sm/uuid [:string {:max 250}]]]]
    [:project-id ::sm/uuid]
+   [:version {:optional true} ::sm/int]
    [:file ::media/upload]])
 
 (sv/defmethod ::import-binfile
@@ -88,12 +141,11 @@
    ::webhooks/event? true
    ::sse/stream? true
    ::sm/params schema:import-binfile}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id name project-id file] :as params}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id version] :as params}]
   (projects/check-edition-permissions! pool profile-id project-id)
-  (let [cfg (-> cfg
-                (assoc ::bf.v1/project-id project-id)
-                (assoc ::bf.v1/profile-id profile-id)
-                (assoc ::bf.v1/name name))]
+  (let [params (-> params
+                   (assoc :profile-id profile-id)
+                   (assoc :version (or version 1)))]
     (with-meta
-      (sse/response #(import-binfile cfg (:path file)))
+      (sse/response (partial import-binfile cfg params))
       {::audit/props {:file nil}})))

backend/src/app/rpc/commands/management.clj

@@ -176,7 +176,7 @@
 
   (binding [bfc/*state* (volatile! {:index {team-id (uuid/next)}})]
     (let [projs (bfc/get-team-projects cfg team-id)
-          files (bfc/get-team-files cfg team-id)
+          files (bfc/get-team-files-ids cfg team-id)
           frels (bfc/get-files-rels cfg files)
 
           team  (-> (db/get-by-id conn :team team-id)
@@ -396,14 +396,15 @@
 (defn clone-template
   [cfg {:keys [project-id profile-id] :as params} template]
   (db/tx-run! cfg (fn [{:keys [::db/conn ::wrk/executor] :as cfg}]
-                    ;; NOTE: the importation process performs some operations that
-                    ;; are not very friendly with virtual threads, and for avoid
-                    ;; unexpected blocking of other concurrent operations we
-                    ;; dispatch that operation to a dedicated executor.
+                    ;; NOTE: the importation process performs some operations
+                    ;; that are not very friendly with virtual threads, and for
+                    ;; avoid unexpected blocking of other concurrent operations
+                    ;; we dispatch that operation to a dedicated executor.
                     (let [cfg    (-> cfg
                                      (assoc ::bf.v1/project-id project-id)
-                                     (assoc ::bf.v1/profile-id profile-id))
-                          result (px/invoke! executor (partial bf.v1/import-files! cfg template))]
+                                     (assoc ::bf.v1/profile-id profile-id)
+                                     (assoc ::bf.v1/input template))
+                          result (px/invoke! executor (partial bf.v1/import-files! cfg))]
 
                       (db/update! conn :project
                                   {:modified-at (dt/now)}