🎉 Add blob encoding v3.

Safer approach (uses json instead of custom binary format) than v2.

CHANGES.md

@@ -5,6 +5,7 @@
 ### :sparkles: New features
 
 - Add more chinese translations [#726](https://github.com/penpot/penpot/pull/726)
+- Add blob-encoding v3 (uses ZSTD+transit) [#738](https://github.com/penpot/penpot/pull/738)
 - Duplicate and move files and projects [Taiga #267](https://tree.taiga.io/project/penpot/us/267)
 - Import SVG will create Penpot's shapes
 - Improve french translations [#731](https://github.com/penpot/penpot/pull/731)

backend/dev/user.clj

@@ -12,11 +12,12 @@
    [app.common.exceptions :as ex]
    [app.config :as cfg]
    [app.main :as main]
+   [app.util.blob :as blob]
+   [app.util.json :as json]
    [app.util.time :as dt]
    [app.util.transit :as t]
-   [app.util.json :as json]
    [clojure.java.io :as io]
-   [clojure.pprint :refer [pprint]]
+   [clojure.pprint :refer [pprint print-table]]
    [clojure.repl :refer :all]
    [clojure.spec.alpha :as s]
    [clojure.spec.gen.alpha :as sgen]
@@ -25,8 +26,7 @@
    [clojure.tools.namespace.repl :as repl]
    [clojure.walk :refer [macroexpand-all]]
    [criterium.core :refer [quick-bench bench with-progress-reporting]]
-   [integrant.core :as ig]
+   [integrant.core :as ig]))
-   [taoensso.nippy :as nippy]))
 
 (repl/disable-reload! (find-ns 'integrant.core))
 
@@ -91,3 +91,10 @@
   []
   (stop)
   (repl/refresh-all :after 'user/start))
+
+(defn compression-bench
+  [data]
+  (print-table
+   [{:v1 (alength (blob/encode data {:version 1}))
+     :v2 (alength (blob/encode data {:version 2}))
+     :v3 (alength (blob/encode data {:version 3}))}]))

backend/src/app/util/blob.clj

@@ -28,8 +28,10 @@
 
 (declare decode-v1)
 (declare decode-v2)
+(declare decode-v3)
 (declare encode-v1)
 (declare encode-v2)
+(declare encode-v3)
 
 (def default-version
   (:default-blob-version cfg/config 1))
@@ -40,6 +42,7 @@
    (case  (long version)
      1 (encode-v1 data)
      2 (encode-v2 data)
+     3 (encode-v3 data)
      (throw (ex-info "unsupported version" {:version version})))))
 
 (defn decode
@@ -52,6 +55,7 @@
       (case version
         1 (decode-v1 data ulen)
         2 (decode-v2 data ulen)
+        3 (decode-v3 data ulen)
         (throw (ex-info "unsupported version" {:version version}))))))
 
 ;; --- IMPL
@@ -100,3 +104,26 @@
     (Zstd/decompressByteArray ^bytes udata 0 ulen
                               ^bytes cdata 6 (- (alength cdata) 6))
     (n/fast-thaw udata)))
+
+(defn- encode-v3
+  [data]
+  (let [data  (t/encode data {:type :json})
+        dlen  (alength ^bytes data)
+        mlen  (Zstd/compressBound dlen)
+        cdata (byte-array mlen)
+        clen  (Zstd/compressByteArray ^bytes cdata 0 mlen
+                                      ^bytes data 0 dlen
+                                      4)]
+    (with-open [^ByteArrayOutputStream baos (ByteArrayOutputStream. (+ (alength cdata) 2 4))
+                ^DataOutputStream dos (DataOutputStream. baos)]
+      (.writeShort dos (short 3)) ;; version number
+      (.writeInt dos (int dlen))
+      (.write dos ^bytes cdata (int 0) clen)
+      (.toByteArray baos))))
+
+(defn- decode-v3
+  [^bytes cdata ^long ulen]
+  (let [udata (byte-array ulen)]
+    (Zstd/decompressByteArray ^bytes udata 0 ulen
+                              ^bytes cdata 6 (- (alength cdata) 6))
+    (t/decode udata {:type :json})))