diff --git a/.gitignore b/.gitignore index 04e1da56b..7b5ea6e61 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ node_modules /common/target /common/coverage /.clj-kondo/.cache +clj-profiler/ /bundle* /media /deploy diff --git a/backend/deps.edn b/backend/deps.edn index ae1750a22..5a18c8c8c 100644 --- a/backend/deps.edn +++ b/backend/deps.edn @@ -13,6 +13,7 @@ com.taoensso/nippy {:mvn/version "3.1.1"} com.github.luben/zstd-jni {:mvn/version "1.5.0-4"} + org.clojure/data.fressian {:mvn/version "1.0.0"} ;; NOTE: don't upgrade to latest version, breaking change is ;; introduced on 0.10.0 that suffixes counters with _total if they @@ -63,6 +64,7 @@ org.clojure/test.check {:mvn/version "RELEASE"} org.clojure/data.csv {:mvn/version "1.0.0"} com.clojure-goes-fast/clj-async-profiler {:mvn/version "0.5.1"} + clojure-humanize/clojure-humanize {:mvn/version "0.2.2"} criterium/criterium {:mvn/version "RELEASE"} mockery/mockery {:mvn/version "RELEASE"}} diff --git a/backend/dev/user.clj b/backend/dev/user.clj index 4f47934a8..ad51f5771 100644 --- a/backend/dev/user.clj +++ b/backend/dev/user.clj @@ -6,15 +6,18 @@ (ns user (:require + [datoteka.core] [app.common.exceptions :as ex] [app.config :as cfg] [app.main :as main] [app.util.blob :as blob] [app.util.json :as json] + [app.util.fressian :as fres] [app.util.time :as dt] - [app.util.transit :as t] + [app.common.transit :as t] [clojure.java.io :as io] [clojure.pprint :refer [pprint print-table]] + [clojure.contrib.humanize :as hum] [clojure.repl :refer :all] [clojure.spec.alpha :as s] [clojure.spec.gen.alpha :as sgen] @@ -22,10 +25,12 @@ [clojure.test :as test] [clojure.tools.namespace.repl :as repl] [clojure.walk :refer [macroexpand-all]] + [clj-async-profiler.core :as prof] [criterium.core :refer [quick-bench bench with-progress-reporting]] [integrant.core :as ig])) (repl/disable-reload! (find-ns 'integrant.core)) +(set! *warn-on-reflection* true) (defonce system nil) @@ -91,7 +96,15 @@ (defn compression-bench [data] - (print-table - [{:v1 (alength (blob/encode data {:version 1})) - :v2 (alength (blob/encode data {:version 2})) - :v3 (alength (blob/encode data {:version 3}))}])) + (let [humanize (fn [v] (hum/filesize v :binary true :format " %.4f "))] + (print-table + [{:v1 (humanize (alength (blob/encode data {:version 1}))) + :v2 (humanize (alength (blob/encode data {:version 2}))) + :v3 (humanize (alength (blob/encode data {:version 3}))) + :v4 (humanize (alength (blob/encode data {:version 4}))) + }]))) + + +;; ;; (def contents (read-string (slurp (io/resource "bool-contents-1.edn")))) +;; (def pre-data (datoteka.core/slurp-bytes (io/resource "file-data-sample"))) +;; (def data (blob/decode pre-data)) diff --git a/backend/scripts/repl b/backend/scripts/repl index bf63eeb7d..db86c66db 100755 --- a/backend/scripts/repl +++ b/backend/scripts/repl @@ -9,7 +9,6 @@ export OPTIONS="-A:jmx-remote:dev \ -J-Dclojure.tools.logging.factory=clojure.tools.logging.impl/log4j2-factory \ -J-XX:+UseShenandoahGC -J-XX:-OmitStackTraceInFastThrow -J-Xms50m -J-Xmx512m"; -# export OPTIONS="$OPTIONS -J-XX:+UnlockDiagnosticVMOptions"; # export OPTIONS="$OPTIONS -J-XX:-TieredCompilation -J-XX:CompileThreshold=10000"; export OPTIONS_EVAL="nil" diff --git a/backend/src/app/db.clj b/backend/src/app/db.clj index 2e53029cb..e56b9bcb5 100644 --- a/backend/src/app/db.clj +++ b/backend/src/app/db.clj @@ -27,14 +27,16 @@ com.zaxxer.hikari.HikariConfig com.zaxxer.hikari.HikariDataSource com.zaxxer.hikari.metrics.prometheus.PrometheusMetricsTrackerFactory + java.io.InputStream + java.io.OutputStream java.lang.AutoCloseable java.sql.Connection java.sql.Savepoint org.postgresql.PGConnection org.postgresql.geometric.PGpoint + org.postgresql.jdbc.PgArray org.postgresql.largeobject.LargeObject org.postgresql.largeobject.LargeObjectManager - org.postgresql.jdbc.PgArray org.postgresql.util.PGInterval org.postgresql.util.PGobject)) diff --git a/backend/src/app/msgbus.clj b/backend/src/app/msgbus.clj index f80b8334c..a8ad395fe 100644 --- a/backend/src/app/msgbus.clj +++ b/backend/src/app/msgbus.clj @@ -179,18 +179,18 @@ ;; Add a unique listener to connection (.addListener sub-conn (reify RedisPubSubListener - (message [it pattern topic message]) - (message [it topic message] + (message [_it _pattern _topic _message]) + (message [_it topic message] ;; There are no back pressure, so we use a sliding ;; buffer for cases when the pubsub broker sends ;; more messages that we can process. (let [val {:topic topic :message (blob/decode message)}] (when-not (a/offer! rcv-ch val) (l/warn :msg "dropping message on subscription loop")))) - (psubscribed [it pattern count]) - (punsubscribed [it pattern count]) - (subscribed [it topic count]) - (unsubscribed [it topic count]))) + (psubscribed [_it _pattern _count]) + (punsubscribed [_it _pattern _count]) + (subscribed [_it _topic _count]) + (unsubscribed [_it _topic _count]))) (letfn [(subscribe-to-single-topic [nsubs topic chan] (let [nsubs (if (nil? nsubs) #{chan} (conj nsubs chan))] diff --git a/backend/src/app/rpc/mutations/files.clj b/backend/src/app/rpc/mutations/files.clj index 4153ac62a..d9dd5ff12 100644 --- a/backend/src/app/rpc/mutations/files.clj +++ b/backend/src/app/rpc/mutations/files.clj @@ -409,7 +409,6 @@ [conn project-id] (:team-id (db/get-by-id conn :project project-id {:columns [:team-id]}))) - ;; TEMPORARY FILE CREATION (s/def ::create-temp-file ::create-file) diff --git a/backend/src/app/storage/impl.clj b/backend/src/app/storage/impl.clj index 4c3a61900..3c9c6a7d0 100644 --- a/backend/src/app/storage/impl.clj +++ b/backend/src/app/storage/impl.clj @@ -117,11 +117,11 @@ io/IOFactory (make-reader [_ opts] (io/make-reader path opts)) - (make-writer [_ opts] + (make-writer [_ _] (throw (UnsupportedOperationException. "not implemented"))) (make-input-stream [_ opts] (io/make-input-stream path opts)) - (make-output-stream [_ opts] + (make-output-stream [_ _] (throw (UnsupportedOperationException. "not implemented"))) clojure.lang.Counted (count [_] size) @@ -138,11 +138,11 @@ io/IOFactory (make-reader [_ opts] (io/make-reader bais opts)) - (make-writer [_ opts] + (make-writer [_ _] (throw (UnsupportedOperationException. "not implemented"))) (make-input-stream [_ opts] (io/make-input-stream bais opts)) - (make-output-stream [_ opts] + (make-output-stream [_ _] (throw (UnsupportedOperationException. "not implemented"))) clojure.lang.Counted @@ -159,11 +159,11 @@ io/IOFactory (make-reader [_ opts] (io/make-reader is opts)) - (make-writer [_ opts] + (make-writer [_ _] (throw (UnsupportedOperationException. "not implemented"))) (make-input-stream [_ opts] (io/make-input-stream is opts)) - (make-output-stream [_ opts] + (make-output-stream [_ _] (throw (UnsupportedOperationException. "not implemented"))) clojure.lang.Counted diff --git a/backend/src/app/util/blob.clj b/backend/src/app/util/blob.clj index 42539b934..f4daebc35 100644 --- a/backend/src/app/util/blob.clj +++ b/backend/src/app/util/blob.clj @@ -10,6 +10,7 @@ (:require [app.common.transit :as t] [app.config :as cf] + [app.util.fressian :as fres] [taoensso.nippy :as n]) (:import java.io.ByteArrayInputStream @@ -21,23 +22,28 @@ net.jpountz.lz4.LZ4FastDecompressor net.jpountz.lz4.LZ4Compressor)) +(set! *warn-on-reflection* true) + (def lz4-factory (LZ4Factory/fastestInstance)) (declare decode-v1) (declare decode-v2) (declare decode-v3) +(declare decode-v4) (declare encode-v1) (declare encode-v2) (declare encode-v3) +(declare encode-v4) (defn encode ([data] (encode data nil)) ([data {:keys [version]}] - (let [version (or version (cf/get :default-blob-version 1))] + (let [version (or version (cf/get :default-blob-version 3))] (case (long version) 1 (encode-v1 data) 2 (encode-v2 data) 3 (encode-v3 data) + 4 (encode-v4 data) (throw (ex-info "unsupported version" {:version version})))))) (defn decode @@ -51,6 +57,7 @@ 1 (decode-v1 data ulen) 2 (decode-v2 data ulen) 3 (decode-v3 data ulen) + 4 (decode-v4 data ulen) (throw (ex-info "unsupported version" {:version version})))))) ;; --- IMPL @@ -122,3 +129,26 @@ (Zstd/decompressByteArray ^bytes udata 0 ulen ^bytes cdata 6 (- (alength cdata) 6)) (t/decode udata {:type :json}))) + +(defn- encode-v4 + [data] + (let [data (fres/encode data) + dlen (alength ^bytes data) + mlen (Zstd/compressBound dlen) + cdata (byte-array mlen) + clen (Zstd/compressByteArray ^bytes cdata 0 mlen + ^bytes data 0 dlen + 0)] + (with-open [^ByteArrayOutputStream baos (ByteArrayOutputStream. (+ (alength cdata) 2 4)) + ^DataOutputStream dos (DataOutputStream. baos)] + (.writeShort dos (short 4)) ;; version number + (.writeInt dos (int dlen)) + (.write dos ^bytes cdata (int 0) clen) + (.toByteArray baos)))) + +(defn- decode-v4 + [^bytes cdata ^long ulen] + (let [udata (byte-array ulen)] + (Zstd/decompressByteArray ^bytes udata 0 ulen + ^bytes cdata 6 (- (alength cdata) 6)) + (fres/decode udata))) diff --git a/backend/src/app/util/fressian.clj b/backend/src/app/util/fressian.clj new file mode 100644 index 000000000..08a7b5cf7 --- /dev/null +++ b/backend/src/app/util/fressian.clj @@ -0,0 +1,259 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) UXBOX Labs SL + +(ns app.util.fressian + (:require + [app.common.geom.matrix :as gmt] + [app.common.geom.point :as gpt] + [clojure.data.fressian :as fres]) + (:import + clojure.lang.Ratio + org.fressian.handlers.WriteHandler + org.fressian.handlers.ReadHandler + org.fressian.Writer + org.fressian.Reader + org.fressian.StreamingWriter + app.common.geom.matrix.Matrix + app.common.geom.point.Point + java.io.ByteArrayInputStream + java.io.ByteArrayOutputStream)) + +;; --- MISC + +(set! *warn-on-reflection* true) + +(defn str->bytes + ([^String s] + (str->bytes s "UTF-8")) + ([^String s, ^String encoding] + (.getBytes s encoding))) + +(defn write-named + [tag ^Writer w s] + (.writeTag w tag 2) + (.writeObject w (namespace s) true) + (.writeObject w (name s) true)) + +(defn write-list-like + ([^Writer w tag o] + (.writeTag w tag 1) + (.writeList w o))) + +(defn read-list-like + [^Reader rdr build-fn] + (build-fn (.readObject rdr))) + +(defn write-map-like + "Writes a map as Fressian with the tag 'map' and all keys cached." + [^Writer w tag m] + (.writeTag w tag 1) + (.beginClosedList ^StreamingWriter w) + (loop [items (seq m)] + (when-let [^clojure.lang.MapEntry item (first items)] + (.writeObject w (.key item) true) + (.writeObject w (.val item)) + (recur (rest items)))) + (.endList ^StreamingWriter w)) + +(defn read-map-like + [^Reader rdr] + (let [kvs ^java.util.List (.readObject rdr)] + (if (< (.size kvs) 16) + (clojure.lang.PersistentArrayMap. (.toArray kvs)) + (clojure.lang.PersistentHashMap/create (seq kvs))))) + +(def write-handlers + { Character + {"char" + (reify WriteHandler + (write [_ w ch] + (.writeTag w "char" 1) + (.writeInt w (int ch))))} + + app.common.geom.point.Point + {"penpot/point" + (reify WriteHandler + (write [_ w o] + (.writeTag ^Writer w "penpot/point" 1) + (.writeList ^Writer w (java.util.List/of (.-x ^Point o) (.-y ^Point o)))))} + + app.common.geom.matrix.Matrix + {"penpot/matrix" + (reify WriteHandler + (write [_ w o] + (.writeTag ^Writer w "penpot/matrix" 1) + (.writeList ^Writer w (java.util.List/of (.-a ^Matrix o) + (.-b ^Matrix o) + (.-c ^Matrix o) + (.-d ^Matrix o) + (.-e ^Matrix o) + (.-f ^Matrix o)))))} + + Ratio + {"ratio" + (reify WriteHandler + (write [_ w n] + (.writeTag w "ratio" 2) + (.writeObject w (.numerator ^Ratio n)) + (.writeObject w (.denominator ^Ratio n))))} + + clojure.lang.IPersistentMap + {"clj/map" + (reify WriteHandler + (write [_ w d] + (write-map-like w "clj/map" d)))} + + clojure.lang.Keyword + {"clj/keyword" + (reify WriteHandler + (write [_ w s] + (write-named "clj/keyword" w s)))} + + clojure.lang.BigInt + {"bigint" + (reify WriteHandler + (write [_ w d] + (let [^BigInteger bi (if (instance? clojure.lang.BigInt d) + (.toBigInteger ^clojure.lang.BigInt d) + d)] + (.writeTag w "bigint" 1) + (.writeBytes w (.toByteArray bi)))))} + + ;; Persistent set + clojure.lang.IPersistentSet + {"clj/set" + (reify WriteHandler + (write [_ w o] + (write-list-like w "clj/set" o)))} + + ;; Persistent vector + clojure.lang.IPersistentVector + {"clj/vector" + (reify WriteHandler + (write [_ w o] + (write-list-like w "clj/vector" o)))} + + ;; Persistent list + clojure.lang.IPersistentList + {"clj/list" + (reify WriteHandler + (write [_ w o] + (write-list-like w "clj/list" o)))} + + ;; Persistent seq & lazy seqs + clojure.lang.ISeq + {"clj/seq" + (reify WriteHandler + (write [_ w o] + (write-list-like w "clj/seq" o)))} + }) + + +(def read-handlers + {"bigint" + (reify ReadHandler + (read [_ rdr _ _] + (let [^bytes bibytes (.readObject rdr)] + (bigint (BigInteger. bibytes))))) + + "byte" + (reify ReadHandler + (read [_ rdr _ _] + (byte (.readObject rdr)))) + + "penpot/matrix" + (reify ReadHandler + (read [_ rdr _ _] + (let [^java.util.List x (.readObject rdr)] + (Matrix. (.get x 0) (.get x 1) (.get x 2) (.get x 3) (.get x 4) (.get x 5))))) + + "penpot/point" + (reify ReadHandler + (read [_ rdr _ _] + (let [^java.util.List x (.readObject rdr)] + (Point. (.get x 0) (.get x 1))))) + + "char" + (reify ReadHandler + (read [_ rdr _ _] + (char (.readObject rdr)))) + + "clj/ratio" + (reify ReadHandler + (read [_ rdr _ _] + (Ratio. (biginteger (.readObject rdr)) + (biginteger (.readObject rdr))))) + + "clj/keyword" + (reify ReadHandler + (read [_ rdr _ _] + (keyword (.readObject rdr) (.readObject rdr)))) + + "clj/map" + (reify ReadHandler + (read [_ rdr _ _] + (read-map-like rdr))) + + "clj/set" + (reify ReadHandler + (read [_ rdr _ _] + (read-list-like rdr set))) + + "clj/vector" + (reify ReadHandler + (read [_ rdr _ _] + (read-list-like rdr vec))) + + "clj/list" + (reify ReadHandler + (read [_ rdr _ _] + (read-list-like rdr #(apply list %)))) + + "clj/seq" + (reify ReadHandler + (read [_ rdr _ _] + (read-list-like rdr sequence))) + }) + +(def write-handler-lookup + (-> write-handlers + fres/associative-lookup + fres/inheritance-lookup)) + +(def read-handler-lookup + (-> read-handlers + (fres/associative-lookup))) + +;; --- Low-Level Api + +(defn reader + [istream] + (fres/create-reader istream :handlers read-handler-lookup)) + +(defn writer + [ostream] + (fres/create-writer ostream :handlers write-handler-lookup)) + +(defn read! + [reader] + (fres/read-object reader)) + +(defn write! + [writer data] + (fres/write-object writer data)) + +;; --- High-Level Api + +(defn encode + [data] + (with-open [out (ByteArrayOutputStream.)] + (write! (writer out) data) + (.toByteArray out))) + +(defn decode + [data] + (with-open [input (ByteArrayInputStream. ^bytes data)] + (read! (reader input)))) diff --git a/common/src/app/common/data.cljc b/common/src/app/common/data.cljc index 47f75a2c3..ad167a321 100644 --- a/common/src/app/common/data.cljc +++ b/common/src/app/common/data.cljc @@ -6,7 +6,7 @@ (ns app.common.data "Data manipulation and query helper functions." - (:refer-clojure :exclude [read-string hash-map merge name]) + (:refer-clojure :exclude [read-string hash-map merge name parse-double]) #?(:cljs (:require-macros [app.common.data])) (:require diff --git a/common/src/app/common/transit.cljc b/common/src/app/common/transit.cljc index 7f1eddda5..e6d90e18d 100644 --- a/common/src/app/common/transit.cljc +++ b/common/src/app/common/transit.cljc @@ -144,19 +144,28 @@ ;; --- Low-Level Api +#?(:clj + (def read-handlers + (t/read-handler-map +read-handlers+))) + +#?(:clj + (def write-handlers + (t/write-handler-map +write-handlers+))) + #?(:clj (defn reader ([istream] (reader istream nil)) ([istream {:keys [type] :or {type :json}}] - (t/reader istream type {:handlers +read-handlers+})))) + (t/reader istream type {:handlers read-handlers})))) #?(:clj (defn writer ([ostream] (writer ostream nil)) ([ostream {:keys [type] :or {type :json}}] - (t/writer ostream type {:handlers +write-handlers+})))) + (t/writer ostream type {:handlers write-handlers})))) + #?(:clj (defn read! [reader]