Merge pull request #2075 from penpot/niwinz-export-embed-assets

Embed assets and multiple files support for binfile export

backend/resources/log4j2-devenv.xml

@@ -24,12 +24,12 @@
     <Logger name="com.zaxxer.hikari" level="error"/>
     <Logger name="org.postgresql" level="error" />
 
-    <Logger name="app.rpc.commands.binfile" level="trace" />
+    <Logger name="app.rpc.commands.binfile" level="debug" />
     <Logger name="app.storage.tmp" level="trace" />
-    <Logger name="app.worker" level="trace" />
+    <Logger name="app.worker" level="info" />
-    <Logger name="app.msgbus" level="trace" />
+    <Logger name="app.msgbus" level="info" />
-    <Logger name="app.http.websocket" level="trace" />
+    <Logger name="app.http.websocket" level="info" />
-    <Logger name="app.util.websocket" level="trace" />
+    <Logger name="app.util.websocket" level="info" />
 
     <Logger name="app.cli" level="debug" additivity="false">
       <AppenderRef ref="console"/>

backend/resources/templates/debug.tmpl

@@ -50,13 +50,21 @@ Debug Main Page
       file.</desc>
 
       <form method="get" action="/dbg/file/export">
-        <div class="row">
+        <div class="row set-of-inputs">
-          <input type="text" style="width:300px" name="file-id" placeholder="file-id" />
+          <input type="text" style="width:300px" name="file-ids" placeholder="file-id" />
+          <input type="text" style="width:300px" name="file-ids" placeholder="file-id" />
+          <input type="text" style="width:300px" name="file-ids" placeholder="file-id" />
+          <input type="text" style="width:300px" name="file-ids" placeholder="file-id" />
         </div>
 
         <div class="row">
           <label>Include libraries?</label>
-          <input type="checkbox" name="includelibs" checked/>
+          <input type="checkbox" name="includelibs" />
+        </div>
+
+        <div class="row">
+          <label>Embed assets?</label>
+          <input type="checkbox" name="embedassets" checked/>
         </div>
 
         <div class="row">
@@ -100,7 +108,7 @@ Debug Main Page
 
         <div class="row">
           <label>Ignore index errors?</label>
-          <input type="checkbox" name="ignore-index-errors" />
+          <input type="checkbox" name="ignore-index-errors" checked/>
           <br />
           <small>
             Do not break on index lookup erros (remap operation).

backend/resources/templates/styles.css

@@ -168,3 +168,12 @@ form .row {
   padding: 5px 0;
 }
 
+.set-of-inputs {
+  flex-direction: column;
+  display: flex;
+}
+
+.set-of-inputs input:not(:last-child) {
+  margin-bottom: 3px;
+}
+

backend/src/app/http/debug.clj

@@ -265,16 +265,21 @@
 
 (defn export-handler
   [{:keys [pool] :as cfg} {:keys [params profile-id] :as request}]
-  (let [file-id (some-> params :file-id parse-uuid)
-        libs?   (contains? params :includelibs)
-        clone?  (contains? params :clone)]
 
-    (when-not file-id
+  (let [file-ids (->> (:file-ids params)
+                      (remove empty?)
+                      (map parse-uuid))
+        libs?    (contains? params :includelibs)
+        clone?   (contains? params :clone)
+        embed?   (contains? params :embedassets)]
+
+    (when-not (seq file-ids)
       (ex/raise :type :validation
                 :code :missing-arguments))
 
     (let [path (-> cfg
-                   (assoc ::binf/file-id file-id)
+                   (assoc ::binf/file-ids file-ids)
+                   (assoc ::binf/embed-assets? embed?)
                    (assoc ::binf/include-libraries? libs?)
                    (binf/export!))]
       (if clone?
@@ -283,6 +288,7 @@
            (assoc cfg
                   ::binf/input path
                   ::binf/overwrite? false
+                  ::binf/ignore-index-errors? true
                   ::binf/profile-id profile-id
                   ::binf/project-id project-id))
 
@@ -294,7 +300,7 @@
         (yrs/response
          :status  200
          :headers {"content-type" "application/octet-stream"
-                   "content-disposition" (str "attachmen; filename=" file-id ".penpot")}
+                   "content-disposition" (str "attachmen; filename=" (first file-ids) ".penpot")}
          :body    (io/input-stream path))))))
 
 

backend/src/app/rpc/commands/binfile.clj

@@ -306,7 +306,7 @@
      SELECT fl.id, fl.deleted_at
        FROM file AS fl
        JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
-      WHERE flr.file_id = ?::uuid
+      WHERE flr.file_id = ANY(?)
     UNION
      SELECT fl.id, fl.deleted_at
        FROM file AS fl
@@ -318,8 +318,10 @@
     WHERE l.deleted_at IS NULL OR l.deleted_at > now();")
 
 (defn- retrieve-libraries
-  [pool file-id]
+  [pool ids]
-  (map :id (db/exec! pool [sql:file-libraries file-id])))
+  (with-open [^AutoCloseable conn (db/open pool)]
+    (let [ids (db/create-array conn "uuid" ids)]
+      (map :id (db/exec! pool [sql:file-libraries ids])))))
 
 (def ^:private sql:file-library-rels
   "SELECT * FROM file_library_rel
@@ -330,6 +332,58 @@
   (with-open [^AutoCloseable conn (db/open pool)]
     (db/exec! conn [sql:file-library-rels (db/create-array conn "uuid" ids)])))
 
+(defn- embed-file-assets
+  [pool {:keys [id] :as file}]
+  (letfn [(walk-map-form [state form]
+            (cond
+              (uuid? (:fill-color-ref-file form))
+              (do
+                (vswap! state conj [(:fill-color-ref-file form) :colors (:fill-color-ref-id form)])
+                (assoc form :fill-color-ref-file id))
+
+              (uuid? (:stroke-color-ref-file form))
+              (do
+                (vswap! state conj [(:stroke-color-ref-file form) :colors (:stroke-color-ref-id form)])
+                (assoc form :stroke-color-ref-file id))
+
+              (uuid? (:typography-ref-file form))
+              (do
+                (vswap! state conj [(:typography-ref-file form) :typographies (:typography-ref-id form)])
+                (assoc form :typography-ref-file id))
+
+              (uuid? (:component-file form))
+              (do
+                (vswap! state conj [(:component-file form) :components (:component-id form)])
+                (assoc form :component-file id))
+
+              :else
+              form))
+
+          (process-group-of-assets [data [lib-id items]]
+            ;; NOTE: there are a posibility that shape refers to a not
+            ;; existing file because the file was removed. In this
+            ;; case we just ignore the asset.
+            (if-let [lib (retrieve-file pool lib-id)]
+              (reduce #(process-asset %1 lib %2) data items)
+              data))
+
+          (process-asset [data lib [bucket asset-id]]
+            (let [asset (get-in lib [:data bucket asset-id])
+                  ;; Add a special case for colors that need to have
+                  ;; correctly set the :file-id prop (pending of the
+                  ;; refactor that will remove it).
+                  asset (cond-> asset
+                          (= bucket :colors) (assoc :file-id id))]
+              (update data bucket assoc asset-id asset)))]
+
+    (update file :data (fn [data]
+                         (let [assets (volatile! [])]
+                           (walk/postwalk #(cond->> % (map? %) (walk-map-form assets)) data)
+                           (->> (deref assets)
+                                (filter #(as-> (first %) $ (and (uuid? $) (not= $ id))))
+                                (d/group-by first rest)
+                                (reduce process-group-of-assets data)))))))
+
 (defn write-export!
   "Do the exportation of a speficied file in custom penpot binary
   format. There are some options available for customize the output:
@@ -337,20 +391,47 @@
   `::include-libraries?`: additionaly to the specified file, all the
   linked libraries also will be included (including transitive
   dependencies).
+
+  `::embed-assets?`: instead of including the libraryes, embedd in the
+  same file library all assets used from external libraries.
   "
 
-  [{:keys [pool storage ::output ::file-id ::include-libraries?]}]
+  [{:keys [pool storage ::output ::file-ids ::include-libraries? ::embed-assets?] :as options}]
-  (let [libs  (when include-libraries?
+
-                (retrieve-libraries pool file-id))
+  (us/assert! :spec ::db/pool :val pool)
-        rels  (when include-libraries?
+  (us/assert! :spec ::sto/storage :val storage)
-                (retrieve-library-relations pool (cons file-id libs)))
+
-        files (into [file-id] libs)
+  (us/assert!
-        sids  (atom #{})]
+   :expr (every? uuid? file-ids)
+   :hint "`files` should be a vector of uuid")
+
+  (us/assert!
+   :expr (bs/data-output-stream? output)
+   :hint "`output` should be an instance of OutputStream")
+
+  (us/assert!
+   :expr (d/boolean-or-nil? include-libraries?)
+   :hint "invalid value provided for `include-libraries?` option, expected boolean")
+
+  (us/assert!
+   :expr (d/boolean-or-nil? embed-assets?)
+   :hint "invalid value provided for `embed-assets?` option, expected boolean")
+
+  (us/assert!
+   :always? true
+   :expr (not (and include-libraries? embed-assets?))
+   :hint "the `include-libraries?` and `embed-assets?` are mutally excluding options")
+
+  (let [libs  (when include-libraries? (retrieve-libraries pool file-ids))
+        files (into file-ids libs)
+        rels  (when include-libraries? (retrieve-library-relations pool file-ids))
+        sids  (volatile! #{})]
 
     ;; Write header with metadata
     (l/debug :hint "exportation summary"
              :files (count files)
              :rels  (count rels)
+             :embed-assets? embed-assets?
              :include-libs? include-libraries?
              ::l/async false)
 
@@ -363,12 +444,13 @@
     (l/debug :hint "write section" :section :v1/files :total (count files) ::l/async false)
     (write-label! output :v1/files)
     (doseq [file-id files]
-      (let [file  (retrieve-file pool file-id)
+      (let [file  (cond->> (retrieve-file pool file-id)
+                    embed-assets? (embed-file-assets pool))
             media (retrieve-file-media pool file)]
 
         ;; Collect all storage ids for later write them all under
         ;; specific storage objects section.
-        (swap! sids into (sequence storage-object-id-xf media))
+        (vswap! sids into (sequence storage-object-id-xf media))
 
         (l/trace :hint "write penpot file"
                  :id file-id

backend/src/app/util/bytes.clj

@@ -28,6 +28,18 @@
 (def ^:const default-buffer-size
   (:xnio/buffer-size yt/defaults))
 
+(defn input-stream?
+  [s]
+  (instance? InputStream s))
+
+(defn output-stream?
+  [s]
+  (instance? OutputStream s))
+
+(defn data-output-stream?
+  [s]
+  (instance? DataOutputStream s))
+
 (defn copy!
   [src dst & {:keys [offset size buffer-size]
               :or {offset 0 buffer-size default-buffer-size}}]

common/src/app/common/data.cljc

@@ -23,6 +23,9 @@
   #?(:clj
      (:import linked.set.LinkedSet)))
 
+(def boolean-or-nil?
+  (some-fn nil? boolean?))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Data Structures
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

common/src/app/common/spec.cljc

@@ -261,6 +261,33 @@
         message (str "spec verify: '" (pr-str spec) "'")]
     `(spec-assert* ~spec ~x ~message ~context)))
 
+(defmacro assert!
+  "General purpose assertion macro."
+  [& {:keys [expr spec always? hint val]}]
+  (cond
+    (some? spec)
+    (let [context (if-let [nsdata (:ns &env)]
+                    {:ns (str (:name nsdata))
+                     :name (pr-str spec)
+                     :line (:line &env)
+                     :file (:file (:meta nsdata))}
+                    {:ns   (str (ns-name *ns*))
+                     :name (pr-str spec)
+                     :line (:line (meta &form))})
+          message (or hint (str "spec assert: " (pr-str spec)))]
+      (when (or always? *assert*)
+        `(spec-assert* ~spec ~val ~message ~context)))
+
+    (some? expr)
+    (let [message (or hint (str "expr assert: " (pr-str expr)))]
+      (when (or always? *assert*)
+        `(when-not ~expr
+           (ex/raise :type :assertion
+                     :code :expr-validation
+                     :hint ~message))))
+
+    :else nil))
+
 ;; --- Public Api
 
 (defn conform

frontend/src/app/main/data/workspace/colors.cljs

@@ -14,34 +14,10 @@
    [app.main.data.workspace.layout :as layout]
    [app.main.data.workspace.state-helpers :as wsh]
    [app.main.data.workspace.texts :as dwt]
-   [app.main.repo :as rp]
    [app.util.color :as uc]
    [beicon.core :as rx]
    [potok.core :as ptk]))
 
-(def clear-color-for-rename
-  (ptk/reify ::clear-color-for-rename
-    ptk/UpdateEvent
-    (update [_ state]
-      (assoc-in state [:workspace-global :color-for-rename] nil))))
-
-(declare rename-color-result)
-
-(defn rename-color
-  [file-id color-id name]
-  (ptk/reify ::rename-color
-    ptk/WatchEvent
-    (watch [_ _ _]
-      (->> (rp/mutation! :rename-color {:id color-id :name name})
-           (rx/map (partial rename-color-result file-id))))))
-
-(defn rename-color-result
-  [_file-id color]
-  (ptk/reify ::rename-color-result
-    ptk/UpdateEvent
-    (update [_ state]
-      (update-in state [:workspace-file :colors] #(d/replace-by-id % color)))))
-
 (defn change-palette-selected
   "Change the library used by the general palette tool"
   [selected]

frontend/src/app/main/ui/workspace/sidebar/assets.cljs

@@ -47,14 +47,10 @@
    [potok.core :as ptk]
    [rumext.alpha :as mf]))
 
-;; TODO: refactor to remove duplicate code and less parameter passing.
+;; NOTE: TODO: for avoid too many arguments, I think we can use react
-;;  - Move all state to [:workspace-local :assets-bar file-id :open-boxes {}
+;; context variables for pass to the down tree all the common
-;;                                                            :open-groups {}
+;; variables that are defined on the MAIN container/box component.
-;;                                                            :reverse-sort?
+
-;;                                                            :listing-thumbs?
-;;                                                            :selected-assets {}]
-;;  - Move selection code to independent functions that receive the state as a parameter.
-;;
 ;; TODO: change update operations to admit multiple ids, thus avoiding the need of
 ;;       emitting many events and opening an undo transaction. Also move the logic
 ;;       of grouping, deleting, etc. to events in the data module, since now the
@@ -205,8 +201,6 @@
               create-typed-assets-group (partial create-typed-assets-group components-to-group)]
           (modal/show! :name-group-dialog {:accept create-typed-assets-group}))))))
 
-
-
 (defn- on-drag-enter-asset
   [event asset dragging? selected-assets selected-assets-paths]
   (when (and
@@ -275,8 +269,6 @@
           (:id target-asset)
           (cph/merge-path-item prefix (:name target-asset))))))))
 
-
-
 ;; ---- Common blocks ----
 
 (def auto-pos-menu-state {:open? false
@@ -1090,6 +1082,7 @@
                        :else (:value color))
 
         ;; TODO: looks like the first argument is not necessary
+        ;; TODO: this code should be out of this UI component
         apply-color
         (fn [_ event]
           (let [objects  (wsh/lookup-page-objects @st/state)