🎉 Add inplace binfile import support

backend/src/app/binfile/common.clj

@@ -15,13 +15,14 @@
    [app.common.files.migrations :as fmg]
    [app.common.files.validate :as fval]
    [app.common.logging :as l]
+   [app.common.schema :as sm]
    [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
-   [app.features.fdata :as feat.fdata]
+   [app.features.fdata :as fdata]
-   [app.features.file-migrations :as feat.fmigr]
+   [app.features.file-migrations :as fmigr]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.storage :as sto]
@@ -32,12 +33,14 @@
    [clojure.set :as set]
    [cuerdas.core :as str]
    [datoteka.fs :as fs]
-   [datoteka.io :as io]))
+   [datoteka.io :as io]
+   [promesa.exec :as px]))
 
 (set! *warn-on-reflection* true)
 
 (def ^:dynamic *state* nil)
 (def ^:dynamic *options* nil)
+(def ^:dynamic *reference-file* nil)
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; DEFAULTS
@@ -53,17 +56,12 @@
   (* 1024 1024 100))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
 (declare get-resolved-file-libraries)
+(declare update-file!)
 
 (def file-attrs
-  #{:id
+  (sm/keys ctf/schema:file))
-    :name
-    :migrations
-    :features
-    :project-id
-    :is-shared
-    :version
-    :data})
 
 (defn parse-file-format
   [template]
@@ -151,22 +149,33 @@
       changes  (assoc :changes (blob/decode changes))
       data     (assoc :data (blob/decode data)))))
 
+(def sql:get-minimal-file
+  "SELECT f.id,
+          f.revn,
+          f.modified_at,
+          f.deleted_at
+     FROM file AS f
+    WHERE f.id = ?")
+
+(defn get-minimal-file
+  [cfg id & {:as opts}]
+  (db/get-with-sql cfg [sql:get-minimal-file id] opts))
 
 (defn decode-file
   "A general purpose file decoding function that resolves all external
   pointers, run migrations and return plain vanilla file map"
   [cfg {:keys [id] :as file} & {:keys [migrate?] :or {migrate? true}}]
-  (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
+  (binding [pmap/*load-fn* (partial fdata/load-pointer cfg id)]
     (let [file (->> file
-                    (feat.fmigr/resolve-applied-migrations cfg)
+                    (fmigr/resolve-applied-migrations cfg)
-                    (feat.fdata/resolve-file-data cfg))
+                    (fdata/resolve-file-data cfg))
           libs (delay (get-resolved-file-libraries cfg file))]
 
       (-> file
           (update :features db/decode-pgarray #{})
           (update :data blob/decode)
-          (update :data feat.fdata/process-pointers deref)
+          (update :data fdata/process-pointers deref)
-          (update :data feat.fdata/process-objects (partial into {}))
+          (update :data fdata/process-objects (partial into {}))
           (update :data assoc :id id)
           (cond-> migrate? (fmg/migrate-file libs))))))
 
@@ -421,6 +430,27 @@
     (db/exec-one! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"])
     (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])))
 
+(defn invalidate-thumbnails
+  [cfg file-id]
+  (let [storage (sto/resolve cfg)
+
+        sql-1
+        (str "update file_tagged_object_thumbnail "
+             "   set deleted_at = now() "
+             " where file_id=? returning media_id")
+
+        sql-2
+        (str "update file_thumbnail "
+             "   set deleted_at = now() "
+             " where file_id=? returning media_id")]
+
+    (run! #(sto/touch-object! storage %)
+          (sequence
+           (keep :media-id)
+           (concat
+            (db/exec! cfg [sql-1 file-id])
+            (db/exec! cfg [sql-2 file-id]))))))
+
 (defn process-file
   [cfg {:keys [id] :as file}]
   (let [libs (delay (get-resolved-file-libraries cfg file))]
@@ -445,77 +475,79 @@
         (vary-meta dissoc ::fmg/migrated))))
 
 (defn encode-file
-  [{:keys [::db/conn] :as cfg} {:keys [id features] :as file}]
+  [{:keys [::wrk/executor] :as cfg} {:keys [id features] :as file}]
-  (let [file (if (contains? features "fdata/objects-map")
+  (let [file (if (and (contains? features "fdata/objects-map")
-               (feat.fdata/enable-objects-map file)
+                      (:data file))
+               (fdata/enable-objects-map file)
                file)
 
-        file (if (contains? features "fdata/pointer-map")
+        file (if (and (contains? features "fdata/pointer-map")
-               (binding [pmap/*tracked* (pmap/create-tracked)]
+                      (:data file))
-                 (let [file (feat.fdata/enable-pointer-map file)]
+
-                   (feat.fdata/persist-pointers! cfg id)
+               (binding [pmap/*tracked* (pmap/create-tracked :inherit true)]
+                 (let [file (fdata/enable-pointer-map file)]
+                   (fdata/persist-pointers! cfg id)
                    file))
                file)]
 
     (-> file
-        (update :features db/encode-pgarray conn "text")
+        (d/update-when :features into-array)
-        (update :data blob/encode))))
+        (d/update-when :data (fn [data] (px/invoke! executor #(blob/encode data)))))))
 
-(defn get-params-from-file
+(defn- file->params
   [file]
-  (let [params {:has-media-trimmed (:has-media-trimmed file)
+  (-> (select-keys file file-attrs)
-                :ignore-sync-until (:ignore-sync-until file)
+      (dissoc :team-id)
-                :project-id (:project-id file)
+      (dissoc :migrations)))
-                :features (:features file)
-                :name (:name file)
-                :is-shared (:is-shared file)
-                :version (:version file)
-                :data (:data file)
-                :id (:id file)
-                :deleted-at (:deleted-at file)
-                :created-at (:created-at file)
-                :modified-at (:modified-at file)
-                :revn (:revn file)
-                :vern (:vern file)}]
-
-    (-> (d/without-nils params)
-        (assoc :data-backend nil)
-        (assoc :data-ref-id nil))))
 
 (defn insert-file!
-  "Insert a new file into the database table"
+  "Insert a new file into the database table. Expectes a not-encoded file.
+  Returns nil."
   [{:keys [::db/conn] :as cfg} file & {:as opts}]
-  (feat.fmigr/upsert-migrations! conn file)
+
-  (let [params (-> (encode-file cfg file)
+  (when (:migrations file)
-                   (get-params-from-file))]
+    (fmigr/upsert-migrations! conn file))
-    (db/insert! conn :file params opts)))
+
+  (let [file (encode-file cfg file)]
+    (db/insert! conn :file
+                (file->params file)
+                {::db/return-keys false})
+    nil))
 
 (defn update-file!
-  "Update an existing file on the database."
+  "Update an existing file on the database. Expects not encoded file."
-  [{:keys [::db/conn ::sto/storage] :as cfg} {:keys [id] :as file} & {:as opts}]
+  [{:keys [::db/conn] :as cfg} {:keys [id] :as file} & {:as opts}]
-  (let [file   (encode-file cfg file)
-        params (-> (get-params-from-file file)
-                   (dissoc :id))]
 
-    ;; If file was already offloaded, we touch the underlying storage
+  (if (::reset-migrations opts false)
-    ;; object for properly trigger storage-gc-touched task
+    (fmigr/reset-migrations! conn file)
-    (when (feat.fdata/offloaded? file)
+    (fmigr/upsert-migrations! conn file))
-      (some->> (:data-ref-id file) (sto/touch-object! storage)))
 
-    (feat.fmigr/upsert-migrations! conn file)
+  (let [file
-    (db/update! conn :file params {:id id} opts)))
+        (encode-file cfg file)
+
+        params
+        (file->params (dissoc file :id))]
+
+    (db/update! conn :file params
+                {:id id}
+                {::db/return-keys false})
+
+    nil))
 
 (defn save-file!
   "Applies all the final validations and perist the file, binfile
-  specific, should not be used outside of binfile domain"
+  specific, should not be used outside of binfile domain.
+
+  Returns nil"
   [{:keys [::timestamp] :as cfg} file & {:as opts}]
 
   (assert (dt/instant? timestamp) "expected valid timestamp")
-
   (let [file (-> file
                  (assoc :created-at timestamp)
                  (assoc :modified-at timestamp)
-                 (assoc :ignore-sync-until (dt/plus timestamp (dt/duration {:seconds 5})))
+                 (cond-> (not (::overwrite cfg))
+                   (assoc :ignore-sync-until (dt/plus timestamp (dt/duration {:seconds 5}))))
+                 (update :revn inc)
                  (update :features
                          (fn [features]
                            (-> (::features cfg #{})
@@ -532,8 +564,9 @@
         (when (ex/exception? result)
           (l/error :hint "file schema validation error" :cause result))))
 
-    (insert-file! cfg file opts)))
+    (if (::overwrite cfg)
-
+      (update-file! cfg file (assoc opts ::reset-migrations true))
+      (insert-file! cfg file opts))))
 
 (def ^:private sql:get-file-libraries
   "WITH RECURSIVE libs AS (
@@ -558,7 +591,8 @@
           l.revn,
           l.vern,
           l.synced_at,
-          l.is_shared
+          l.is_shared,
+          l.version
      FROM libs AS l
     INNER JOIN project AS p ON (p.id = l.project_id)
     WHERE l.deleted_at IS NULL OR l.deleted_at > now();")
@@ -573,6 +607,8 @@
          (map decode-row))
         (db/exec! conn [sql:get-file-libraries file-id])))
 
+;; FIXME: this will use a lot of memory if file uses too many big
+;; libraries, we should load required libraries on demand
 (defn get-resolved-file-libraries
   "A helper for preload file libraries"
   [{:keys [::db/conn] :as cfg} file]

backend/src/app/binfile/v3.clj

@@ -284,10 +284,12 @@
                  (assoc :options (:options data))
 
                  :always
-                 (dissoc :data)
+                 (dissoc :data))
 
+          file (cond-> file
                  :always
                  (encode-file))
+
           path (str "files/" file-id ".json")]
       (write-entry! output path file))
 
@@ -544,15 +546,18 @@
     (json/read reader)))
 
 (defn- read-file
-  [{:keys [::bfc/input ::file-id]}]
+  [{:keys [::bfc/input ::bfc/timestamp]} file-id]
   (let [path  (str "files/" file-id ".json")
         entry (get-zip-entry input path)]
     (-> (read-entry input entry)
         (decode-file)
+        (update :revn d/nilv 1)
+        (update :created-at d/nilv timestamp)
+        (update :modified-at d/nilv timestamp)
         (validate-file))))
 
 (defn- read-file-plugin-data
-  [{:keys [::bfc/input ::file-id]}]
+  [{:keys [::bfc/input]} file-id]
   (let [path  (str "files/" file-id "/plugin-data.json")
         entry (get-zip-entry* input path)]
     (some->> entry
@@ -561,7 +566,7 @@
              (validate-plugin-data))))
 
 (defn- read-file-media
-  [{:keys [::bfc/input ::file-id ::entries]}]
+  [{:keys [::bfc/input ::entries]} file-id]
   (->> (keep (match-media-entry-fn file-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -581,7 +586,7 @@
        (not-empty)))
 
 (defn- read-file-colors
-  [{:keys [::bfc/input ::file-id ::entries]}]
+  [{:keys [::bfc/input ::entries]} file-id]
   (->> (keep (match-color-entry-fn file-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -594,7 +599,7 @@
        (not-empty)))
 
 (defn- read-file-components
-  [{:keys [::bfc/input ::file-id ::entries]}]
+  [{:keys [::bfc/input ::entries]} file-id]
   (let [clean-component-post-decode
         (fn [component]
           (d/update-when component :objects
@@ -625,7 +630,7 @@
          (not-empty))))
 
 (defn- read-file-typographies
-  [{:keys [::bfc/input ::file-id ::entries]}]
+  [{:keys [::bfc/input ::entries]} file-id]
   (->> (keep (match-typography-entry-fn file-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -638,14 +643,14 @@
        (not-empty)))
 
 (defn- read-file-tokens-lib
-  [{:keys [::bfc/input ::file-id ::entries]}]
+  [{:keys [::bfc/input ::entries]} file-id]
   (when-let [entry (d/seek (match-tokens-lib-entry-fn file-id) entries)]
     (->> (read-plain-entry input entry)
          (decode-tokens-lib)
          (validate-tokens-lib))))
 
 (defn- read-file-shapes
-  [{:keys [::bfc/input ::file-id ::page-id ::entries] :as cfg}]
+  [{:keys [::bfc/input ::entries] :as cfg} file-id page-id]
   (->> (keep (match-shape-entry-fn file-id page-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -659,15 +664,14 @@
        (not-empty)))
 
 (defn- read-file-pages
-  [{:keys [::bfc/input ::file-id ::entries] :as cfg}]
+  [{:keys [::bfc/input ::entries] :as cfg} file-id]
   (->> (keep (match-page-entry-fn file-id) entries)
        (keep (fn [{:keys [id entry]}]
                (let [page (->> (read-entry input entry)
                                (decode-page))
                      page (dissoc page :options)]
                  (when (= id (:id page))
-                   (let [objects (-> (assoc cfg ::page-id id)
+                   (let [objects (read-file-shapes cfg file-id id)]
-                                     (read-file-shapes))]
                      (assoc page :objects objects))))))
        (sort-by :index)
        (reduce (fn [result {:keys [id] :as page}]
@@ -675,7 +679,7 @@
                (d/ordered-map))))
 
 (defn- read-file-thumbnails
-  [{:keys [::bfc/input ::file-id ::entries] :as cfg}]
+  [{:keys [::bfc/input ::entries] :as cfg} file-id]
   (->> (keep (match-thumbnail-entry-fn file-id) entries)
        (reduce (fn [result {:keys [page-id frame-id tag entry]}]
                  (let [object (->> (read-entry input entry)
@@ -690,13 +694,13 @@
        (not-empty)))
 
 (defn- read-file-data
-  [cfg]
+  [cfg file-id]
-  (let [colors       (read-file-colors cfg)
+  (let [colors       (read-file-colors cfg file-id)
-        typographies (read-file-typographies cfg)
+        typographies (read-file-typographies cfg file-id)
-        tokens-lib   (read-file-tokens-lib cfg)
+        tokens-lib   (read-file-tokens-lib cfg file-id)
-        components   (read-file-components cfg)
+        components   (read-file-components cfg file-id)
-        plugin-data  (read-file-plugin-data cfg)
+        plugin-data  (read-file-plugin-data cfg file-id)
-        pages        (read-file-pages cfg)]
+        pages        (read-file-pages cfg file-id)]
     {:pages (-> pages keys vec)
      :pages-index (into {} pages)
      :colors colors
@@ -706,11 +710,11 @@
      :plugin-data plugin-data}))
 
 (defn- import-file
-  [{:keys [::bfc/project-id ::file-id ::file-name] :as cfg}]
+  [{:keys [::bfc/project-id] :as cfg} {file-id :id file-name :name}]
   (let [file-id'   (bfc/lookup-index file-id)
-        file       (read-file cfg)
+        file       (read-file cfg file-id)
-        media      (read-file-media cfg)
+        media      (read-file-media cfg file-id)
-        thumbnails (read-file-thumbnails cfg)]
+        thumbnails (read-file-thumbnails cfg file-id)]
 
     (l/dbg :hint "processing file"
            :id (str file-id')
@@ -740,7 +744,7 @@
       (vswap! bfc/*state* update :index bfc/update-index (map :media-id thumbnails))
       (vswap! bfc/*state* update :thumbnails into thumbnails))
 
-    (let [data (-> (read-file-data cfg)
+    (let [data (-> (read-file-data cfg file-id)
                    (d/without-nils)
                    (assoc :id file-id')
                    (cond-> (:options file)
@@ -757,7 +761,7 @@
           file  (ctf/check-file file)]
 
       (bfm/register-pending-migrations! cfg file)
-      (bfc/save-file! cfg file ::db/return-keys false)
+      (bfc/save-file! cfg file)
 
       file-id')))
 
@@ -853,7 +857,8 @@
              :file-id (str (:file-id params))
              ::l/sync? true)
 
-      (db/insert! conn :file-media-object params))))
+      (db/insert! conn :file-media-object params
+                  ::db/on-conflict-do-nothing? (::bfc/overwrite cfg)))))
 
 (defn- import-file-thumbnails
   [{:keys [::db/conn] :as cfg}]
@@ -873,17 +878,77 @@
              :media-id (str media-id)
              ::l/sync? true)
 
-      (db/insert! conn :file-tagged-object-thumbnail params))))
+      (db/insert! conn :file-tagged-object-thumbnail params
+                  {::db/on-conflict-do-nothing? true}))))
+
+(defn- import-files*
+  [{:keys [::manifest] :as cfg}]
+  (bfc/disable-database-timeouts! cfg)
+
+  (vswap! bfc/*state* update :index bfc/update-index (:files manifest) :id)
+
+  (let [files  (get manifest :files)
+        result (reduce (fn [result {:keys [id] :as file}]
+                         (let [name' (get file :name)
+                               name' (if (map? name)
+                                       (get name id)
+                                       name')
+                               file (assoc file :name name')]
+                           (conj result (import-file cfg file))))
+                       []
+                       files)]
+
+    (import-file-relations cfg)
+    (import-storage-objects cfg)
+    (import-file-media cfg)
+    (import-file-thumbnails cfg)
+
+    (bfm/apply-pending-migrations! cfg)
+
+    result))
+
+(defn- import-file-and-overwrite*
+  [{:keys [::manifest ::bfc/file-id] :as cfg}]
+
+  (when (not= 1 (count (:files manifest)))
+    (ex/raise :type :validation
+              :code :invalid-condition
+              :hint "unable to perform in-place update with binfile containing more than 1 file"
+              :manifest manifest))
+
+  (bfc/disable-database-timeouts! cfg)
+
+  (let [ref-file (bfc/get-minimal-file cfg file-id ::db/for-update true)
+        file     (first (get manifest :files))
+        cfg      (assoc cfg ::bfc/overwrite true)]
+
+    (vswap! bfc/*state* update :index assoc (:id file) file-id)
+
+    (binding [bfc/*options* cfg
+              bfc/*reference-file* ref-file]
+
+      (import-file cfg file)
+      (import-storage-objects cfg)
+      (import-file-media cfg)
+
+      (bfc/invalidate-thumbnails cfg file-id)
+      (bfm/apply-pending-migrations! cfg)
+
+      [file-id])))
 
 (defn- import-files
-  [{:keys [::bfc/timestamp ::bfc/input ::bfc/name] :or {timestamp (dt/now)} :as cfg}]
+  [{:keys [::bfc/timestamp ::bfc/input] :or {timestamp (dt/now)} :as cfg}]
 
   (assert (instance? ZipFile input) "expected zip file")
   (assert (dt/instant? timestamp) "expected valid instant")
 
   (let [manifest (-> (read-manifest input)
                      (validate-manifest))
-        entries  (read-zip-entries input)]
+        entries  (read-zip-entries input)
+        cfg      (-> cfg
+                     (assoc ::entries entries)
+                     (assoc ::manifest manifest)
+                     (assoc ::bfc/timestamp timestamp))]
 
     (when-not (= "penpot/export-files" (:type manifest))
       (ex/raise :type :validation
@@ -891,7 +956,6 @@
                 :hint "unexpected type on manifest"
                 :manifest manifest))
 
-
     ;; Check if all files referenced on manifest are present
     (doseq [{file-id :id features :features} (:files manifest)]
       (let [path (str "files/" file-id ".json")]
@@ -907,35 +971,10 @@
 
     (events/tap :progress {:section :manifest})
 
-    (let [index (bfc/update-index (map :id (:files manifest)))
+    (binding [bfc/*state* (volatile! {:media [] :index {}})]
-          state {:media [] :index index}
+      (if (::bfc/file-id cfg)
-          cfg   (-> cfg
+        (db/tx-run! cfg import-file-and-overwrite*)
-                    (assoc ::entries entries)
+        (db/tx-run! cfg import-files*)))))
-                    (assoc ::manifest manifest)
-                    (assoc ::bfc/timestamp timestamp))]
-
-      (binding [bfc/*state* (volatile! state)]
-        (db/tx-run! cfg (fn [cfg]
-                          (bfc/disable-database-timeouts! cfg)
-                          (let [ids (->> (:files manifest)
-                                         (reduce (fn [result {:keys [id] :as file}]
-                                                   (let [name' (get file :name)
-                                                         name' (if (map? name)
-                                                                 (get name id)
-                                                                 name')]
-                                                     (conj result (-> cfg
-                                                                      (assoc ::file-id id)
-                                                                      (assoc ::file-name name')
-                                                                      (import-file)))))
-                                                 []))]
-                            (import-file-relations cfg)
-                            (import-storage-objects cfg)
-                            (import-file-media cfg)
-                            (import-file-thumbnails cfg)
-
-                            (bfm/apply-pending-migrations! cfg)
-
-                            ids)))))))
 
 ;; --- PUBLIC API
 

backend/src/app/features/fdata.clj

@@ -18,7 +18,9 @@
    [app.storage :as sto]
    [app.util.blob :as blob]
    [app.util.objects-map :as omap]
-   [app.util.pointer-map :as pmap]))
+   [app.util.pointer-map :as pmap]
+   [app.worker :as wrk]
+   [promesa.exec :as px]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; OFFLOAD
@@ -81,6 +83,12 @@
   (let [data (get-file-data system file)]
     (assoc file :data data)))
 
+(defn decode-file-data
+  [{:keys [::wrk/executor]} {:keys [data] :as file}]
+  (cond-> file
+    (bytes? data)
+    (assoc :data (px/invoke! executor #(blob/decode data)))))
+
 (defn load-pointer
   "A database loader pointer helper"
   [system file-id id]

backend/src/app/features/file_migrations.clj

@@ -8,6 +8,7 @@
   "Backend specific code for file migrations. Implemented as permanent feature of files."
   (:require
    [app.common.data :as d]
+   [app.common.exceptions :as ex]
    [app.common.files.migrations :as fmg :refer [xf:map-name]]
    [app.db :as db]
    [app.db.sql :as-alias sql]))
@@ -26,12 +27,19 @@
 (defn upsert-migrations!
   "Persist or update file migrations. Return the updated/inserted number
   of rows"
-  [conn {:keys [id] :as file}]
+  [cfg {:keys [id] :as file}]
-  (let [migrations (or (-> file meta ::fmg/migrated)
+  (let [conn       (db/get-connection cfg)
-                       (-> file :migrations not-empty)
+        migrations (or (-> file meta ::fmg/migrated)
-                       fmg/available-migrations)
+                       (-> file :migrations))
         columns    [:file-id :name]
-        rows       (mapv (fn [name] [id name]) migrations)]
+        rows       (->> migrations
+                        (mapv (fn [name] [id name]))
+                        (not-empty))]
+
+    (when-not rows
+      (ex/raise :type :internal
+                :code :missing-migrations
+                :hint "no migrations available on file"))
 
     (-> (db/insert-many! conn :file-migration columns rows
                          {::db/return-keys false
@@ -40,6 +48,6 @@
 
 (defn reset-migrations!
   "Replace file migrations"
-  [conn {:keys [id] :as file}]
+  [cfg {:keys [id] :as file}]
-  (db/delete! conn :file-migration {:file-id id})
+  (db/delete! cfg :file-migration {:file-id id})
-  (upsert-migrations! conn file))
+  (upsert-migrations! cfg file))

backend/src/app/rpc/commands/binfile.clj

@@ -125,21 +125,35 @@
    [:name [:or [:string {:max 250}]
            [:map-of ::sm/uuid [:string {:max 250}]]]]
    [:project-id ::sm/uuid]
+   [:file-id {:optional true} ::sm/uuid]
    [:version {:optional true} ::sm/int]
    [:file ::media/upload]])
 
 (sv/defmethod ::import-binfile
-  "Import a penpot file in a binary format."
+  "Import a penpot file in a binary format. If `file-id` is provided,
+  an in-place import will be performed instead of creating a new file.
+
+  The in-place imports are only supported for binfile-v3 and when a
+  .penpot file only contains one penpot file.
+  "
   {::doc/added "1.15"
+   ::doc/changes ["1.20" "Add file-id param for in-place import"
+                  "1.20" "Set default version to 3"]
+
    ::webhooks/event? true
    ::sse/stream? true
    ::sm/params schema:import-binfile}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id version file] :as params}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id version file-id file] :as params}]
   (projects/check-edition-permissions! pool profile-id project-id)
-  (let [version  (or version 1)
+  (let [version  (or version 3)
         params   (-> params
                      (assoc :profile-id profile-id)
                      (assoc :version version))
+
+        cfg      (cond-> cfg
+                   (uuid? file-id)
+                   (assoc ::bfc/file-id file-id))
+
         manifest (case (int version)
                    1 nil
                    3 (bf.v3/get-manifest (:path file)))]
@@ -147,5 +161,6 @@
     (with-meta
       (sse/response (partial import-binfile cfg params))
       {::audit/props {:file nil
+                      :file-id file-id
                       :generated-by (:generated-by manifest)
                       :referer (:referer manifest)}})))

backend/src/app/rpc/commands/files_create.clj

@@ -29,6 +29,7 @@
   [conn {:keys [file-id profile-id role]}]
   (let [params {:file-id file-id
                 :profile-id profile-id}]
+
     (->> (perms/assign-role-flags params role)
          (db/insert! conn :file-profile-rel))))
 
@@ -51,12 +52,12 @@
                                :is-shared is-shared
                                :features features
                                :ignore-sync-until ignore-sync-until
-                               :modified-at modified-at
+                               :created-at modified-at
                                :deleted-at deleted-at}
                               {:create-page create-page
-                               :page-id page-id})
+                               :page-id page-id})]
-          file (-> (bfc/insert-file! cfg file)
+
-                   (bfc/decode-row))]
+      (bfc/insert-file! cfg file)
 
       (->> (assoc params :file-id (:id file) :role :owner)
            (create-file-role! conn))

backend/src/app/util/pointer_map.clj

@@ -61,8 +61,10 @@
 (declare create)
 
 (defn create-tracked
-  []
+  [& {:keys [inherit]}]
-  (atom {}))
+  (if inherit
+    (atom (if *tracked* @*tracked* {}))
+    (atom {})))
 
 (defprotocol IPointerMap
   (get-id [_])

common/src/app/common/types/file.cljc

@@ -102,6 +102,7 @@
    [:project-id {:optional true} ::sm/uuid]
    [:team-id {:optional true} ::sm/uuid]
    [:is-shared {:optional true} ::sm/boolean]
+   [:has-media-trimmed {:optional true} ::sm/boolean]
    [:data {:optional true} schema:data]
    [:version :int]
    [:features ::cfeat/features]
@@ -188,6 +189,7 @@
           :features features
           :migrations migrations
           :ignore-sync-until ignore-sync-until
+          :has-media-trimmed false
           :created-at created-at
           :modified-at modified-at
           :deleted-at deleted-at})]