♻️ Move library to its own directory

.gitignore

@@ -68,6 +68,8 @@
 /vendor/**/target
 /vendor/svgclean/bundle*.js
 /web
+/library/target/
+
 clj-profiler/
 node_modules
 /test-results/

frontend/package.json

@@ -14,8 +14,8 @@
     "url": "https://github.com/penpot/penpot"
   },
   "resolutions": {
-    "@vitejs/plugin-react": "^4.2.0",
-    "@zip.js/zip.js@npm:^2.7.44": "patch:@zip.js/zip.js@npm%3A2.7.60#~/.yarn/patches/@zip.js-zip.js-npm-2.7.60-b6b814410b.patch"
+    "@zip.js/zip.js@npm:^2.7.44": "patch:@zip.js/zip.js@npm%3A2.7.60#~/.yarn/patches/@zip.js-zip.js-npm-2.7.60-b6b814410b.patch",
+    "@vitejs/plugin-react": "^4.2.0"
   },
   "scripts": {
     "build:app:assets": "node ./scripts/build-app-assets.js",
@@ -26,7 +26,6 @@
     "build:app:libs": "node ./scripts/build-libs.js",
     "build:app:main": "clojure -M:dev:shadow-cljs release main worker",
     "build:app": "yarn run clear:shadow-cache && yarn run build:app:main && yarn run build:app:libs",
-    "build:library": "yarn run clear:shadow-cache && clojure -M:dev:shadow-cljs release library",
     "e2e:server": "node ./scripts/e2e-server.js",
     "fmt:clj": "cljfmt fix --parallel=true src/ test/",
     "fmt:clj:check": "cljfmt check --parallel=false src/ test/",
@@ -46,7 +45,6 @@
     "watch:app:main": "clojure -M:dev:shadow-cljs watch main worker storybook",
     "clear:shadow-cache": "rm -rf .shadow-cljs",
     "watch:app": "yarn run clear:shadow-cache && concurrently \"yarn run watch:app:main\" \"yarn run watch:app:libs\"",
-    "watch:library": "yarn run clear:shadow-cache && clojure -M:dev:shadow-cljs watch library",
     "watch": "yarn run watch:app:assets",
     "watch:storybook": "concurrently \"storybook dev -p 6006 --no-open\" \"yarn run watch:storybook:assets\"",
     "watch:storybook:assets": "node ./scripts/watch-storybook.js"

frontend/shadow-cljs.edn

@@ -149,32 +149,6 @@
    {:test {:init-fn frontend-tests.runner/init
            :prepend-js ";if (typeof globalThis.navigator?.userAgent === 'undefined') { globalThis.navigator = {userAgent: ''}; };"}}}
 
-  :library
-  {:target :esm
-   :runtime :custom
-   :output-dir "target/library"
-   :devtools {:autoload false}
-
-   :modules
-   {:penpot
-    {:exports {BuilderError lib.file-builder/BuilderError
-               createFile lib.file-builder/create-file}}}
-
-   :compiler-options
-   {:output-feature-set :es2020
-    :output-wrapper false
-    :warnings {:fn-deprecated false}}
-
-   :release
-   {:compiler-options
-    {:fn-invoke-direct true
-     :optimizations #shadow/env ["PENPOT_BUILD_OPTIMIZATIONS" :as :keyword :default :advanced]
-     :pretty-print false
-     :source-map true
-     :elide-asserts true
-     :anon-fn-naming-policy :off
-     :source-map-detail-level :all}}}
-
   :bench
   {:target :node-script
    :output-to "target/bench.js"

frontend/src/lib/playground/sample1.js

@@ -1,30 +0,0 @@
-import * as penpot from "../../../target/library/penpot.js";
-
-console.log(penpot);
-
-try {
-  const file = penpot.createFile({name: "Test"});
-  file.addPage({name: "Foo Page"})
-  const boardId = file.addArtboard({name: "Foo Board"})
-  const rectId  = file.addRect({name: "Foo Rect", width:100, height: 200})
-
-  file.addLibraryColor({color: "#fabada", opacity: 0.5})
-
-  console.log("created board", boardId);
-  console.log("created rect", rectId);
-
-  const board = file.getShape(boardId);
-  console.log("=========== BOARD =============")
-  console.dir(board, {depth: 10});
-
-  const rect = file.getShape(rectId);
-  console.log("=========== RECT =============")
-  console.dir(rect, {depth: 10});
-
-  // console.dir(file.toMap(), {depth:10});
-} catch (e) {
-  console.log(e);
-  // console.log(e.data);
-}
-
-process.exit(0);

library/.yarn/patches/@zip.js-zip.js-npm-2.7.60-b6b814410b.patch

@@ -0,0 +1,18 @@
+diff --git a/lib/zip-fs.js b/lib/zip-fs.js
+index 1444c0f00e5f1ad6c13521f90a7f3c6659d81116..90e38baef5365c2abbcb9337f7ab37f800e883a4 100644
+--- a/lib/zip-fs.js
++++ b/lib/zip-fs.js
+@@ -33,12 +33,7 @@ import { initShimAsyncCodec } from "./core/util/stream-codec-shim.js";
+ import { terminateWorkers } from "./core/codec-pool.js";
+ 
+ let baseURL;
+-try {
+-	baseURL = import.meta.url;
+-	// eslint-disable-next-line no-unused-vars
+-} catch (_) {
+-	// ignored
+-}
++
+ configure({ baseURL });
+ configureWebWorker(configure);
+ 

library/deps.edn

@@ -0,0 +1,33 @@
+{:paths ["src" "vendor" "resources" "test"]
+ :deps
+ {penpot/common
+  {:local/root "../common"}
+
+  penpot/frontend
+  {:local/root "../frontend"}
+  }
+
+ :aliases
+ {:outdated
+  {:extra-deps {com.github.liquidz/antq {:mvn/version "RELEASE"}}
+   :main-opts ["-m" "antq.core"]}
+
+  :jvm-repl
+  {:extra-deps
+   {com.bhauman/rebel-readline {:mvn/version "RELEASE"}}
+   :main-opts ["-m" "rebel-readline.main"]
+   :jvm-opts ["--sun-misc-unsafe-memory-access=allow"]}
+
+  :dev
+  {:extra-paths ["dev"]
+   :extra-deps
+   {thheller/shadow-cljs {:mvn/version "3.0.5"}
+    com.bhauman/rebel-readline {:mvn/version "RELEASE"}
+    org.clojure/tools.namespace {:mvn/version "RELEASE"}
+    criterium/criterium {:mvn/version "RELEASE"}
+    cider/cider-nrepl {:mvn/version "0.48.0"}}}
+
+  :shadow-cljs
+  {:main-opts ["-m" "shadow.cljs.devtools.cli"]
+   :jvm-opts ["--sun-misc-unsafe-memory-access=allow"]}
+  }}

library/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@penpotapp/library",
+  "version": "1.0.0",
+  "license": "MPL-2.0",
+  "author": "Kaleidos INC",
+  "private": true,
+  "packageManager": "yarn@4.9.1+sha512.f95ce356460e05be48d66401c1ae64ef84d163dd689964962c6888a9810865e39097a5e9de748876c2e0bf89b232d583c33982773e9903ae7a76257270986538",
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/penpot/penpot"
+  },
+  "resolutions": {
+    "@zip.js/zip.js@npm:^2.7.44": "patch:@zip.js/zip.js@npm%3A2.7.60#~/.yarn/patches/@zip.js-zip.js-npm-2.7.60-b6b814410b.patch"
+  },
+  "scripts": {
+    "clear:shadow-cache": "rm -rf .shadow-cljs",
+    "build": "yarn run clear:shadow-cache && clojure -M:dev:shadow-cljs release library",
+    "fmt:clj": "cljfmt fix --parallel=true src/ test/",
+    "fmt:clj:check": "cljfmt check --parallel=false src/ test/",
+    "lint:clj": "clj-kondo --parallel --lint src/",
+    "build:test": "clojure -M:dev:shadow-cljs compile test",
+    "test": "yarn run build:test && node target/tests/test.js",
+    "watch:test": "mkdir -p target/tests && concurrently \"clojure -M:dev:shadow-cljs watch test\" \"nodemon -C -d 2 -w target/tests --exec 'node target/tests/test.js'\"",
+    "watch": "yarn run clear:shadow-cache && clojure -M:dev:shadow-cljs watch library"
+  },
+  "devDependencies": {
+    "@types/node": "^22.12.0",
+    "concurrently": "^9.1.2",
+    "nodemon": "^3.1.9",
+    "shadow-cljs": "3.0.5"
+  },
+  "dependencies": {
+    "@zip.js/zip.js": "patch:@zip.js/zip.js@npm%3A2.7.60#~/.yarn/patches/@zip.js-zip.js-npm-2.7.60-b6b814410b.patch",
+    "luxon": "^3.6.1",
+    "sax": "^1.4.1",
+    "source-map-support": "^0.5.21"
+  }
+}

library/playground/sample1.js

@@ -0,0 +1,48 @@
+import * as penpot from "../target/library/penpot.js";
+import { writeFile } from 'fs/promises';
+import { createWriteStream } from 'fs';
+import { Writable } from "stream";
+
+console.log(penpot);
+
+(async function() {
+  const file = penpot.createFile({name: "Test"});
+
+  file.addPage({name: "Foo Page"})
+  const boardId = file.addArtboard({name: "Foo Board"})
+  const rectId  = file.addRect({name: "Foo Rect", width:100, height: 200})
+
+  file.addLibraryColor({color: "#fabada", opacity: 0.5})
+
+  // console.log("created board", boardId);
+  // console.log("created rect", rectId);
+
+  // const board = file.getShape(boardId);
+  // console.log("=========== BOARD =============")
+  // console.dir(board, {depth: 10});
+
+  // const rect = file.getShape(rectId);
+  // console.log("=========== RECT =============")
+  // console.dir(rect, {depth: 10});
+
+  {
+    let result = await penpot.exportAsBytes(file)
+    await writeFile("sample-sync.zip", result);
+  }
+
+  {
+    // Create a file stream to write the zip to
+    const output = createWriteStream('sample-stream.zip');
+
+    // Wrap Node's stream in a WHATWG WritableStream
+    const writable = Writable.toWeb(output);
+
+    await penpot.exportStream(file, writable);
+  }
+
+})().catch((cause) => {
+  console.log(cause);
+  process.exit(-1);
+}).finally(() => {
+  process.exit(0);
+})

library/scripts/repl

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+export OPTIONS="-A:dev -J-XX:-OmitStackTraceInFastThrow";
+
+set -ex
+exec clojure $OPTIONS -M -m rebel-readline.main

library/shadow-cljs.edn

@@ -0,0 +1,55 @@
+{:deps {:aliases [:dev]}
+ :http {:port #shadow/env ["HTTP_PORT" :as :int :default 4448]}
+ :dev-http {#shadow/env ["DEV_PORT" :as :int :default 8889] "classpath:public"}
+ :nrepl false
+ :socket-repl false
+ :cache-dir #shadow/env ["CACHE" :default ".shadow-cljs"]
+
+ :builds
+ {:test
+  {:target :esm
+   :output-dir "target/tests"
+   :runtime :custom
+   :js-options {:js-provider :import}
+
+   :modules
+   {:test {:init-fn lib.tests.runner/init
+           :prepend-js ";if (typeof globalThis.navigator?.userAgent === 'undefined') { globalThis.navigator = {userAgent: ''}; };"}}}
+
+  :library
+  {:target :esm
+   :runtime :custom
+   :output-dir "target/library"
+   :devtools {:autoload false}
+
+   :modules
+   {:penpot
+    {:exports {BuilderError lib.builder/BuilderError
+               createFile lib.builder/create-file
+               exportAsBytes lib.export/export-bytes
+               exportAsBlob lib.export/export-blob
+               exportStream lib.export/export-stream
+               }}}
+
+   :js-options
+   {:entry-keys ["module" "browser" "main"]
+    :export-conditions ["module" "import", "browser" "require" "default"]
+    :js-provider :import
+    ;; :external-index "target/library/dependencies.js"
+    ;; :external-index-format :esm
+    }
+
+   :compiler-options
+   {:output-feature-set :es2020
+    :output-wrapper false
+    :warnings {:fn-deprecated false}}
+
+   :release
+   {:compiler-options
+    {:fn-invoke-direct true
+     :optimizations #shadow/env ["PENPOT_BUILD_OPTIMIZATIONS" :as :keyword :default :advanced]
+     :pretty-print false
+     :source-map true
+     :elide-asserts true
+     :anon-fn-naming-policy :off
+     :source-map-detail-level :all}}}}}

library/src/lib/builder.cljs

@@ -4,7 +4,7 @@
 ;;
 ;; Copyright (c) KALEIDOS INC
 
-(ns lib.file-builder
+(ns lib.builder
   (:require
    [app.common.data :as d]
    [app.common.data.macros :as dm]

library/src/lib/export.cljs

@@ -0,0 +1,199 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns lib.export
+  "A .penpot export implementation"
+  (:require
+   [app.common.data :as d]
+   [app.common.data.macros :as dm]
+   [app.common.files.builder :as fb]
+   [app.common.json :as json]
+   [app.common.schema :as sm]
+   [app.common.uuid :as uuid]
+   [app.util.object :as obj]
+   [app.common.types.color :as types.color]
+   [app.common.types.component :as types.component]
+   [app.common.types.file :as types.file]
+   [app.common.types.page :as types.page]
+   [app.common.types.plugins :as ctpg]
+   [app.common.types.shape :as types.shape]
+   [app.common.types.tokens-lib :as types.tokens-lib]
+   [app.common.types.typography :as types.typography]
+   [cuerdas.core :as str]
+   [app.util.zip :as zip]
+   [promesa.core :as p]))
+
+(def ^:private schema:file
+  [:merge
+   types.file/schema:file
+   [:map [:options {:optional true} types.file/schema:options]]])
+
+(def ^:private encode-file
+  (sm/encoder schema:file sm/json-transformer))
+
+(def ^:private encode-page
+  (sm/encoder types.page/schema:page sm/json-transformer))
+
+(def ^:private encode-shape
+  (sm/encoder types.shape/schema:shape sm/json-transformer))
+
+(def ^:private encode-component
+  (sm/encoder types.component/schema:component sm/json-transformer))
+
+;; (def encode-media
+;;   (sm/encoder ::ctf/media sm/json-transformer))
+
+(def encode-color
+  (sm/encoder types.color/schema:color sm/json-transformer))
+
+(def encode-typography
+  (sm/encoder types.typography/schema:typography sm/json-transformer))
+
+(def encode-tokens-lib
+  (sm/encoder types.tokens-lib/schema:tokens-lib sm/json-transformer))
+
+(def encode-plugin-data
+  (sm/encoder ::ctpg/plugin-data sm/json-transformer))
+
+(def ^:private valid-buckets
+  #{"file-media-object"
+    "team-font-variant"
+    "file-object-thumbnail"
+    "file-thumbnail"
+    "profile"
+    "file-data"
+    "file-data-fragment"
+    "file-change"})
+
+;; FIXME: move to types
+(def ^:private schema:storage-object
+  [:map {:title "StorageObject"}
+   [:id ::sm/uuid]
+   [:size ::sm/int]
+   [:content-type :string]
+   [:bucket [::sm/one-of {:format :string} valid-buckets]]
+   [:hash :string]])
+
+(def encode-storage-object
+  (sm/encoder schema:storage-object sm/json-transformer))
+
+;; (def encode-file-thumbnail
+;;   (sm/encoder schema:file-thumbnail sm/json-transformer))
+
+
+;; FIXME: naming
+
+(def ^:private file-attrs
+  #{:id
+    :name
+    :migrations
+    :features
+    :is-shared
+    :version})
+
+(defn- generate-file-export-procs
+  [{:keys [id data] :as file}]
+  ;; (prn "generate-file-export-procs")
+  ;; (app.common.pprint/pprint file)
+  (cons
+   (let [file (cond-> (select-keys file file-attrs)
+                (:options data)
+                (assoc :options (:options data)))]
+     [(str "files/" id ".json")
+      (delay (-> file encode-file json/encode))])
+
+   (concat
+    (let [pages       (get data :pages)
+          pages-index (get data :pages-index)]
+      (->> (d/enumerate pages)
+           (mapcat
+            (fn [[index page-id]]
+              (let [path    (str "files/" id "/pages/" page-id ".json")
+                    page    (get pages-index page-id)
+                    objects (:objects page)
+                    page    (-> page
+                                (dissoc :objects)
+                                (assoc :index index))]
+                (cons
+                 [(str "files/" id "/pages/" page-id ".json")
+                  (delay (-> page encode-page json/encode))]
+                 (map (fn [[shape-id shape]]
+                        (let [shape (assoc shape :page-id page-id)]
+                          [(str "files/" id "/pages/" page-id "/" shape-id ".json")
+                           (delay (-> shape encode-shape json/encode))]))
+                      objects)))))))
+
+    (->> (get data :components)
+         (map (fn [[component-id component]]
+                [(str "files/" id "/components/" component-id ".json")
+                 (delay (-> component encode-component json/encode))])))
+
+    (->> (get data :colors)
+         (map (fn [[color-id color]]
+                [(str "files/" id "/colors/" color-id ".json")
+                 (delay (let [color (-> color
+                                        encode-color
+                                        (dissoc :file-id))]
+                          (cond-> color
+                            (and (contains? color :path)
+                                 (str/empty? (:path color)))
+                            (dissoc :path)
+
+                            :always
+                            (json/encode))))])))
+
+    (->> (get data :typographies)
+         (map (fn [[typography-id typography]]
+                [(str "files/" id "/typographies/" typography-id ".json")
+                 (delay (-> typography
+                            encode-typography
+                            json/encode))])))
+
+    (when-let [tokens-lib (get data :tokens-lib)]
+      (list [(str "files/" id "/tokens.json")
+             (delay (-> tokens-lib
+                        (encode-tokens-lib tokens-lib)
+                        json/encode))])))))
+
+(defn generate-manifest-procs
+  [file]
+  (let [mdata  {:id (:id file)
+                :name (:name file)
+                :features (:features file)}
+        params {:type "penpot/export-files"
+                :version 1
+                :generated-by "penpot-lib/develop"
+                :files [mdata]
+                :relations []}]
+    (list
+     ["manifest.json" (delay (json/encode params))])))
+
+(defn- export
+  [file writer]
+  (->> (p/reduce (fn [writer [path proc]]
+                   (let [data (deref proc)]
+                     (js/console.log "export" path)
+                     (->> (zip/add writer path data)
+                          (p/fmap (constantly writer)))))
+
+                 writer
+                 (concat
+                  (generate-manifest-procs @file)
+                  (generate-file-export-procs @file)))
+       (p/mcat (fn [writer]
+                 (zip/close writer)))))
+
+(defn export-bytes
+  [file]
+  (export file (zip/writer (zip/bytes-writer))))
+
+(defn export-blob
+  [file]
+  (export file (zip/writer (zip/blob-writer))))
+
+(defn export-stream
+  [file stream]
+  (export file (zip/writer stream)))