✨ Allows initial data to be extracted/loaded to file

backend/src/app/config.clj

@@ -70,8 +70,8 @@
    :ldap-auth-fullname-attribute "displayName"
    :ldap-auth-avatar-attribute "jpegPhoto"
 
-   ;;:initial-data-project-id "5761a890-3b81-11eb-9e7d-556a2f641513"
+   ;; :initial-data-project-name "Penpot Onboarding"
-   ;;:initial-data-project-name "Penpot Onboarding"
+   ;; :initial-data-file "/internal/initial-data.json"
    })
 
 (s/def ::http-server-port ::us/integer)

backend/src/app/db/profile_initial_data.clj

@@ -14,8 +14,11 @@
    [app.db :as db]
    [app.rpc.mutations.projects :as projects]
    [app.storage :as storage]
+   [app.util.transit :as tr]
    [clojure.tools.logging :as log]
-   [cuerdas.core :as str]))
+   [cuerdas.core :as str])
+  (:import java.io.FileOutputStream
+           java.io.FileInputStream))
 
 (def sql:file
   "select * from file where project_id = ?")
@@ -39,15 +42,6 @@
      from file_media_thumbnail
      where media_object_id in (select id from media_ids)")
 
-(def sql:storage-object
-  "with file_ids as (select id from file where project_id = ?),
-        media_ids as (select media_id as id from file_media_object where file_id in (select id from file_ids)),
-        thumbs_ids as (select thumbnail_id as id from file_media_object where file_id in (select id from file_ids)),
-        storage_ids as (select id from media_ids union select id from thumbs_ids)
-   select *
-     from storage_object
-     where id in (select id from storage_ids)")
-
 (defn change-ids
   "Given a collection and a map from ID to ID. Changes all the `keys` properties
   so they point to the new ID existing in `map-ids`"
@@ -71,26 +65,34 @@
 
     [new-map-ids (map (partial change-id new-map-ids) coll)]))
 
-(defn allocate-storage-objects
+(defn create-initial-data-dump
-  "Copies the storage data to a new object and stores the new id into `map-ids`"
+  [conn project-id output-file]
-  [storage map-ids objects]
+  (let [ ;; Retrieve data from templates
-  (let [clone-object
+        file                 (db/exec! conn [sql:file, project-id])
-        (fn [map-ids object]
+        file-library-rel     (db/exec! conn [sql:file-library-rel, project-id])
-          (try
+        file-media-object    (db/exec! conn [sql:file-media-object, project-id])
-            (let [object (storage/row->storage-object object)
+        file-media-thumbnail (db/exec! conn [sql:file-media-thumbnail, project-id])
-                  new-obj (storage/clone-object storage object)]
+
-              (assoc map-ids (:id object) (:id new-obj)))
+        data {:file file
-            (catch Exception err
+              :file-library-rel file-library-rel
-              (log/errorf "Error cloning store object %s" (:id object))
+              :file-media-object file-media-object
-              map-ids)))]
+              :file-media-thumbnail file-media-thumbnail}]
-    (->> objects
+
-         (reduce clone-object map-ids))))
+    (with-open [output (FileOutputStream. output-file)]
+      (tr/encode-stream data output))))
 
 (defn create-profile-initial-data
   [conn storage profile]
 
-  (when-let [sample-project-id (get cfg/config :initial-data-project-id)]
+  (let [initial-data-file (get cfg/config :initial-data-file)
-    (let [sample-project-name (get cfg/config :initial-data-project-name "Penpot Onboarding")
+        initial-data (when initial-data-file
+                       (with-open [input (FileInputStream. initial-data-file)]
+                         (tr/decode-stream input)))]
+    (when initial-data
+      (let [{:keys [file file-library-rel file-media-object file-media-thumbnail]} initial-data
+
+            sample-project-name (get cfg/config :initial-data-project-name "Penpot Onboarding")
+
 
             proj (projects/create-project conn {:profile-id (:id profile)
                                                 :team-id (:default-team-id profile)
@@ -103,20 +105,11 @@
                                                           :project-id (:id proj)
                                                           :profile-id (:id profile)})
 
-          ;; Retrieve data from templates
-          file                 (db/exec! conn [sql:file, sample-project-id])
-          file-library-rel     (db/exec! conn [sql:file-library-rel, sample-project-id])
-          file-media-object    (db/exec! conn [sql:file-media-object, sample-project-id])
-          file-media-thumbnail (db/exec! conn [sql:file-media-thumbnail, sample-project-id])
-          storage-object       (db/exec! conn [sql:storage-object, sample-project-id])
-
             map-ids {}
 
             ;; Create new ID's and change the references
             [map-ids file]           (change-ids map-ids file #{:id})
 
-          map-ids                  (allocate-storage-objects storage map-ids storage-object)
-
             [map-ids file-library-rel]     (change-ids map-ids file-library-rel #{:file-id :library-file-id})
             [map-ids file-media-object]    (change-ids map-ids file-media-object #{:id :file-id :media-id :thumbnail-id})
             [map-ids file-media-thumbnail] (change-ids map-ids file-media-thumbnail #{:id :media-object-id})
@@ -134,5 +127,4 @@
         (db/insert-multi! conn :file-profile-rel file-profile-rel)
         (db/insert-multi! conn :file-library-rel file-library-rel)
         (db/insert-multi! conn :file-media-object file-media-object)
-      (db/insert-multi! conn :file-media-thumbnail file-media-thumbnail))
+        (db/insert-multi! conn :file-media-thumbnail file-media-thumbnail)))))
-    {:result "OK"}))

backend/src/app/srepl/main.clj

@@ -5,6 +5,7 @@
    [app.common.pages :as cp]
    [app.common.pages.migrations :as pmg]
    [app.db :as db]
+   [app.db.profile-initial-data :as pid]
    [app.main :refer [system]]
    [app.srepl.dev :as dev]
    [app.util.blob :as blob]
@@ -46,3 +47,12 @@
 ;; Examples
 ;; (def backup (update-file  #uuid "1586e1f0-3e02-11eb-b1d2-556a2f641513" identity))
 ;; (def x (update-file #uuid "1586e1f0-3e02-11eb-b1d2-556a2f641513" (fn [{:keys [data] :as file}] (update-in data [:pages-index #uuid "878278c0-3ef0-11eb-9d67-8551e7624f43" :objects] dissoc nil))))
+
+(defn initial-data-dump
+  ([system file]
+   (let [default-project-id #uuid "5761a890-3b81-11eb-9e7d-556a2f641513"]
+     (initial-data-dump system default-project-id file)))
+
+  ([system project-id file]
+   (db/with-atomic [conn (:app.db/pool system)]
+     (pid/create-initial-data-dump conn  file))))

backend/src/app/util/transit.clj

@@ -94,21 +94,33 @@
 (declare str->bytes)
 (declare bytes->str)
 
+(defn decode-stream
+  ([input]
+   (decode-stream input nil))
+  ([input opts]
+   (read! (reader input opts))))
+
 (defn decode
   ([data]
    (decode data nil))
   ([data opts]
    (with-open [input (ByteArrayInputStream. ^bytes data)]
-     (read! (reader input opts)))))
+     (decode-stream input opts))))
+
+(defn encode-stream
+  ([data out]
+   (encode-stream data out nil))
+  ([data out opts]
+   (let [w (writer out opts)]
+     (write! w data))))
 
 (defn encode
   ([data]
    (encode data nil))
   ([data opts]
    (with-open [out (ByteArrayOutputStream.)]
-     (let [w (writer out opts)]
+     (encode-stream data out opts)
-       (write! w data)
+     (.toByteArray out))))
-       (.toByteArray out)))))
 
 (defn decode-str
   [message]