🎉 Add onboarding data to the database.

backend/src/app/config.clj

@@ -76,8 +76,8 @@
    :ldap-attrs-fullname "cn"
    :ldap-attrs-photo "jpegPhoto"
 
-   ;; :initial-data-file "resources/initial-data.json"
+   ;; a server prop key where initial project is stored.
-   ;; :initial-data-project-name "Penpot Oboarding"
+   :initial-project-skey "initial-project"
    })
 
 (s/def ::allow-demo-users ::us/boolean)
@@ -103,8 +103,7 @@
 (s/def ::http-session-idle-max-age ::dt/duration)
 (s/def ::http-session-updater-batch-max-age ::dt/duration)
 (s/def ::http-session-updater-batch-max-size ::us/integer)
-(s/def ::initial-data-file ::us/string)
+(s/def ::initial-project-skey ::us/string)
-(s/def ::initial-data-project-name ::us/string)
 (s/def ::ldap-attrs-email ::us/string)
 (s/def ::ldap-attrs-fullname ::us/string)
 (s/def ::ldap-attrs-photo ::us/string)
@@ -161,8 +160,8 @@
                    ::database-username
                    ::default-blob-version
                    ::error-report-webhook
-                   ::feedback-enabled
                    ::feedback-destination
+                   ::feedback-enabled
                    ::github-client-id
                    ::github-client-secret
                    ::gitlab-base-uri
@@ -170,33 +169,37 @@
                    ::gitlab-client-secret
                    ::google-client-id
                    ::google-client-secret
+                   ::host
                    ::http-server-port
+                   ::http-session-idle-max-age
                    ::http-session-updater-batch-max-age
                    ::http-session-updater-batch-max-size
-                   ::http-session-idle-max-age
+                   ::initial-project-skey
-                   ::host
-                   ::ldap-attrs-username
                    ::ldap-attrs-email
                    ::ldap-attrs-fullname
                    ::ldap-attrs-photo
+                   ::ldap-attrs-username
+                   ::ldap-base-dn
                    ::ldap-bind-dn
                    ::ldap-bind-password
-                   ::ldap-base-dn
                    ::ldap-host
                    ::ldap-port
                    ::ldap-ssl
                    ::ldap-starttls
                    ::ldap-user-query
-                   ::public-uri
+                   ::local-assets-uri
-                   ::profile-complaint-threshold
+                   ::loggers-loki-uri
+                   ::loggers-zmq-uri
+                   ::profile-bounce-max-age
                    ::profile-bounce-threshold
                    ::profile-complaint-max-age
-                   ::profile-bounce-max-age
+                   ::profile-complaint-threshold
+                   ::public-uri
                    ::redis-uri
                    ::registration-domain-whitelist
                    ::registration-enabled
-                   ::rlimits-password
                    ::rlimits-image
+                   ::rlimits-password
                    ::smtp-default-from
                    ::smtp-default-reply-to
                    ::smtp-enabled
@@ -206,23 +209,18 @@
                    ::smtp-ssl
                    ::smtp-tls
                    ::smtp-username
-                   ::storage-backend
-                   ::storage-fs-directory
                    ::srepl-host
                    ::srepl-port
-                   ::local-assets-uri
+                   ::storage-backend
-                   ::loggers-loki-uri
+                   ::storage-fs-directory
-                   ::loggers-zmq-uri
                    ::storage-s3-bucket
                    ::storage-s3-region
                    ::telemetry-enabled
-                   ::telemetry-with-taiga
                    ::telemetry-server-enabled
                    ::telemetry-server-port
                    ::telemetry-uri
-                   ::tenant
+                   ::telemetry-with-taiga
-                   ::initial-data-file
+                   ::tenant]))
-                   ::initial-data-project-name]))
 
 (defn- env->config
   [env]

backend/src/app/db.clj

@@ -200,11 +200,6 @@
               (sql/insert table params opts)
               (assoc opts :return-keys true))))
 
-(defn insert-multi!
-  [ds table param-list]
-  (doseq [params param-list]
-    (insert! ds table params)))
-
 (defn update!
   ([ds table params where] (update! ds table params where nil))
   ([ds table params where opts]

backend/src/app/db/profile_initial_data.clj

@@ -1,117 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020-2021 UXBOX Labs SL
-
-(ns app.db.profile-initial-data
-  (:require
-   [app.common.uuid :as uuid]
-   [app.config :as cfg]
-   [app.db :as db]
-   [app.rpc.mutations.projects :as projects]
-   [app.util.transit :as tr]
-   [clojure.java.io :as io]
-   [datoteka.core :as fs]))
-
-(def sql:file
-  "select * from file where project_id = ?")
-
-(def sql:file-library-rel
-  "with file_ids as (select id from file where project_id = ?)
-   select *
-     from file_library_rel
-     where file_id in (select id from file_ids)")
-
-(def sql:file-media-object
-  "with file_ids as (select id from file where project_id = ?)
-   select *
-     from file_media_object
-     where file_id in (select id from file_ids)")
-
-(defn change-ids
-  "Given a collection and a map from ID to ID. Changes all the `keys` properties
-  so they point to the new ID existing in `map-ids`"
-  [map-ids coll keys]
-  (let [generate-id
-        (fn [map-ids {:keys [id]}]
-          (assoc map-ids id (uuid/next)))
-
-        remap-key
-        (fn [obj map-ids key]
-          (cond-> obj
-            (contains? obj key)
-            (assoc key (get map-ids (get obj key) (get obj key)))))
-
-        change-id
-        (fn [map-ids obj]
-          (reduce #(remap-key %1 map-ids %2) obj keys))
-
-        new-map-ids (reduce generate-id map-ids coll)]
-
-    [new-map-ids (map (partial change-id new-map-ids) coll)]))
-
-(defn create-initial-data-dump
-  [conn project-id output-path]
-  (let [ ;; Retrieve data from templates
-        opath                (fs/path output-path)
-        file                 (db/exec! conn [sql:file, project-id])
-        file-library-rel     (db/exec! conn [sql:file-library-rel, project-id])
-        file-media-object    (db/exec! conn [sql:file-media-object, project-id])
-
-        data {:file file
-              :file-library-rel file-library-rel
-              :file-media-object file-media-object}]
-    (with-open [output (io/output-stream opath)]
-      (tr/encode-stream data output)
-      nil)))
-
-(defn read-initial-data
-  [path]
-  (when (fs/exists? path)
-    (with-open [input (io/input-stream (fs/path path))]
-      (tr/decode-stream input))))
-
-(defn create-profile-initial-data
-  ([conn profile]
-   (when-let [initial-data-path (:initial-data-file cfg/config)]
-     (create-profile-initial-data conn initial-data-path profile)))
-
-  ([conn file profile]
-   (when-let [{:keys [file file-library-rel file-media-object]} (read-initial-data file)]
-     (let [sample-project-name (:initial-data-project-name cfg/config "Penpot Onboarding")
-
-           proj (projects/create-project conn {:profile-id (:id profile)
-                                               :team-id (:default-team-id profile)
-                                               :name sample-project-name})
-
-           map-ids {}
-
-           ;; Create new ID's and change the references
-           [map-ids file]                 (change-ids map-ids file #{:id})
-           [map-ids file-library-rel]     (change-ids map-ids file-library-rel #{:file-id :library-file-id})
-           [_       file-media-object]    (change-ids map-ids file-media-object #{:id :file-id :media-id :thumbnail-id})
-
-           file             (map #(assoc % :project-id (:id proj)) file)
-           file-profile-rel (map #(array-map :file-id (:id %)
-                                             :profile-id (:id profile)
-                                             :is-owner true
-                                             :is-admin true
-                                             :can-edit true)
-                                 file)]
-
-       (projects/create-project-profile conn {:project-id (:id proj)
-                                              :profile-id (:id profile)})
-
-       (projects/create-team-project-profile conn {:team-id (:default-team-id profile)
-                                                   :project-id (:id proj)
-                                                   :profile-id (:id profile)})
-
-       ;; Re-insert into the database
-       (db/insert-multi! conn :file file)
-       (db/insert-multi! conn :file-profile-rel file-profile-rel)
-       (db/insert-multi! conn :file-library-rel file-library-rel)
-       (db/insert-multi! conn :file-media-object file-media-object)))))

backend/src/app/migrations.clj

@@ -160,6 +160,9 @@
 
    {:name "0049-mod-http-session-table"
     :fn (mg/resource "app/migrations/sql/0049-mod-http-session-table.sql")}
+
+   {:name "0050-mod-server-prop-table"
+    :fn (mg/resource "app/migrations/sql/0050-mod-server-prop-table.sql")}
    ])
 
 

backend/src/app/migrations/sql/0050-mod-server-prop-table.sql

@@ -0,0 +1,4 @@
+ALTER TABLE server_prop
+  ADD COLUMN preload boolean DEFAULT false;
+
+UPDATE server_prop SET preload = true;

backend/src/app/rpc/mutations/demo.clj

@@ -5,7 +5,7 @@
 ;; This Source Code Form is "Incompatible With Secondary Licenses", as
 ;; defined by the Mozilla Public License, v. 2.0.
 ;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) 2020-2021 UXBOX Labs SL
 
 (ns app.rpc.mutations.demo
   "A demo specific mutations."
@@ -14,8 +14,8 @@
    [app.common.uuid :as uuid]
    [app.config :as cfg]
    [app.db :as db]
-   [app.db.profile-initial-data :refer [create-profile-initial-data]]
    [app.rpc.mutations.profile :as profile]
+   [app.setup.initial-data :as sid]
    [app.tasks :as tasks]
    [app.util.services :as sv]
    [buddy.core.codecs :as bc]
@@ -48,7 +48,7 @@
     (db/with-atomic [conn pool]
       (->> (#'profile/create-profile conn params)
            (#'profile/create-profile-relations conn)
-           (create-profile-initial-data conn))
+           (sid/load-initial-project! conn))
 
       ;; Schedule deletion of the demo profile
       (tasks/submit! conn {:name "delete-profile"

backend/src/app/rpc/mutations/profile.clj

@@ -14,12 +14,12 @@
    [app.common.uuid :as uuid]
    [app.config :as cfg]
    [app.db :as db]
-   [app.db.profile-initial-data :refer [create-profile-initial-data]]
    [app.emails :as emails]
    [app.media :as media]
    [app.rpc.mutations.projects :as projects]
    [app.rpc.mutations.teams :as teams]
    [app.rpc.queries.profile :as profile]
+   [app.setup.initial-data :as sid]
    [app.storage :as sto]
    [app.tasks :as tasks]
    [app.util.services :as sv]
@@ -81,7 +81,8 @@
   (let [profile (->> (create-profile conn params)
                      (create-profile-relations conn))
         profile (assoc profile ::created true)]
-    (create-profile-initial-data conn profile)
+
+    (sid/load-initial-project! conn profile)
 
     (if-let [token (:invitation-token params)]
       ;; If invitation token comes in params, this is because the
@@ -309,7 +310,7 @@
           (register-profile [conn params]
             (let [profile (->> (create-profile conn params)
                                (create-profile-relations conn))]
-              (create-profile-initial-data conn profile)
+              (sid/load-initial-project! conn profile)
               (assoc profile ::created true)))]
 
     (let [profile (profile/retrieve-profile-data-by-email conn email)

backend/src/app/setup.clj

@@ -37,20 +37,25 @@
   (let [key (-> (bn/random-bytes 64)
                 (bc/bytes->b64u)
                 (bc/bytes->str))]
-    (db/exec-one! conn ["insert into server_prop (id, content)
+    (db/insert! conn :server-prop
-                         values ('secret-key', ?) on conflict do nothing"
+                {:id "secret-key"
-                        (db/tjson key)])))
+                 :preload true
+                 :content (db/tjson key)}
+                {:on-conflict-do-nothing true})))
 
 (defn- initialize-instance-id!
   [{:keys [conn] :as cfg}]
   (let [iid (uuid/random)]
-    (db/exec-one! conn ["insert into server_prop (id, content)
+
-                         values ('instance-id', ?::jsonb) on conflict do nothing"
+    (db/insert! conn :server-prop
-                        (db/tjson iid)])))
+                {:id "instance-id"
+                 :preload true
+                 :content (db/tjson iid)}
+                {:on-conflict-do-nothing true})))
 
 (defn- retrieve-all
   [{:keys [conn] :as cfg}]
   (reduce (fn [acc row]
             (assoc acc (keyword (:id row)) (db/decode-transit-pgobject (:content row))))
           {}
-          (db/exec! conn ["select * from server_prop;"])))
+          (db/query conn :server-prop {:preload true})))

backend/src/app/setup/initial_data.clj

@@ -0,0 +1,137 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; This Source Code Form is "Incompatible With Secondary Licenses", as
+;; defined by the Mozilla Public License, v. 2.0.
+;;
+;; Copyright (c) 2021 UXBOX Labs SL
+
+(ns app.setup.initial-data
+  (:refer-clojure :exclude [load])
+  (:require
+   [app.common.uuid :as uuid]
+   [app.config :as cfg]
+   [app.db :as db]
+   [app.rpc.mutations.projects :as projects]
+   [app.rpc.queries.profile :as profile]))
+
+;; --- DUMP GENERATION
+
+(def sql:file
+  "select * from file where project_id = ?")
+
+(def sql:file-library-rel
+  "with file_ids as (select id from file where project_id = ?)
+   select *
+     from file_library_rel
+     where file_id in (select id from file_ids)")
+
+(def sql:file-media-object
+  "with file_ids as (select id from file where project_id = ?)
+   select *
+     from file_media_object
+     where file_id in (select id from file_ids)")
+
+(defn dump
+  ([system project-id] (dump system project-id nil))
+  ([system project-id {:keys [skey project-name]
+                       :or {project-name "Penpot Onboarding"}}]
+   (db/with-atomic [conn (:app.db/pool system)]
+     (let [skey              (or skey (cfg/get :initial-project-skey))
+           file              (db/exec! conn [sql:file project-id])
+           file-library-rel  (db/exec! conn [sql:file-library-rel project-id])
+           file-media-object (db/exec! conn [sql:file-media-object project-id])
+           data              {:project-name project-name
+                              :file file
+                              :file-library-rel file-library-rel
+                              :file-media-object file-media-object}]
+
+       (db/delete! conn :server-prop
+                   {:id skey})
+       (db/insert! conn :server-prop
+                   {:id skey
+                    :preload false
+                    :content (db/tjson data)})
+       nil))))
+
+
+;; --- DUMP LOADING
+
+(defn- remap-ids
+  "Given a collection and a map from ID to ID. Changes all the `keys`
+  properties so they point to the new ID existing in `map-ids`"
+  [map-ids coll keys]
+  (let [generate-id
+        (fn [map-ids {:keys [id]}]
+          (assoc map-ids id (uuid/next)))
+
+        remap-key
+        (fn [obj map-ids key]
+          (cond-> obj
+            (contains? obj key)
+            (assoc key (get map-ids (get obj key) (get obj key)))))
+
+        change-id
+        (fn [map-ids obj]
+          (reduce #(remap-key %1 map-ids %2) obj keys))
+
+        new-map-ids (reduce generate-id map-ids coll)]
+
+    [new-map-ids (map (partial change-id new-map-ids) coll)]))
+
+(defn- retrieve-data
+  [conn skey]
+  (when-let [row (db/exec-one! conn ["select content from server_prop where id = ?" skey])]
+    (when-let [content (:content row)]
+      (when (db/pgobject? content)
+        (db/decode-transit-pgobject content)))))
+
+(defn load-initial-project!
+  ([conn profile] (load-initial-project! conn profile nil))
+  ([conn profile opts]
+   (let [skey (or (:skey opts) (cfg/get :initial-project-skey))
+         data (retrieve-data conn skey)]
+     (when data
+       (let [project (projects/create-project conn {:profile-id (:id profile)
+                                                    :team-id (:default-team-id profile)
+                                                    :name (:project-name data)})
+
+             map-ids {}
+
+             [map-ids file]                 (remap-ids map-ids (:file data) #{:id})
+             [map-ids file-library-rel]     (remap-ids map-ids (:file-library-rel data) #{:file-id :library-file-id})
+             [_       file-media-object]    (remap-ids map-ids (:file-media-object data) #{:id :file-id :media-id :thumbnail-id})
+
+             file             (map #(assoc % :project-id (:id project)) file)
+             file-profile-rel (map #(array-map :file-id (:id %)
+                                               :profile-id (:id profile)
+                                               :is-owner true
+                                               :is-admin true
+                                               :can-edit true)
+                                   file)]
+
+       (projects/create-project-profile conn {:project-id (:id project)
+                                              :profile-id (:id profile)})
+
+       (projects/create-team-project-profile conn {:team-id (:default-team-id profile)
+                                                   :project-id (:id project)
+                                                   :profile-id (:id profile)})
+
+       ;; Re-insert into the database
+       (doseq [params file]
+         (db/insert! conn :file params))
+       (doseq [params file-profile-rel]
+         (db/insert! conn :file-profile-rel params))
+       (doseq [params file-library-rel]
+         (db/insert! conn :file-library-rel params))
+       (doseq [params file-media-object]
+         (db/insert! conn :file-media-object params)))))))
+
+(defn load
+  [system {:keys [email] :as opts}]
+  (db/with-atomic [conn (:app.db/pool system)]
+    (when-let [profile (profile/retrieve-profile-data-by-email conn email)]
+      (load-initial-project! conn profile opts)
+      true)))
+

backend/src/app/srepl/main.clj

@@ -6,7 +6,6 @@
    [app.common.pages.migrations :as pmg]
    [app.config :as cfg]
    [app.db :as db]
-   [app.db.profile-initial-data :as pid]
    [app.main :refer [system]]
    [app.rpc.queries.profile :as prof]
    [app.srepl.dev :as dev]
@@ -53,27 +52,6 @@
 ;;         (fn [{:keys [data] :as file}]
 ;;           (update-in data [:pages-index #uuid "878278c0-3ef0-11eb-9d67-8551e7624f43" :objects] dissoc nil))))
 
-(def default-project-id #uuid "5761a890-3b81-11eb-9e7d-556a2f641513")
-
-(defn initial-data-dump
-  ([system file] (initial-data-dump system default-project-id file))
-  ([system project-id path]
-   (db/with-atomic [conn (:app.db/pool system)]
-     (pid/create-initial-data-dump conn project-id path))))
-
-(defn load-data-into-user
-  ([system user-email]
-   (if-let [file (:initial-data-file cfg/config)]
-     (load-data-into-user system file user-email)
-     (prn "Data file not found in configuration")))
-
-  ([system file user-email]
-   (db/with-atomic [conn (:app.db/pool system)]
-     (let [profile (prof/retrieve-profile-data-by-email conn user-email)
-           profile (merge profile (prof/retrieve-additional-data conn (:id profile)))]
-       (pid/create-profile-initial-data conn file profile)))))
-
-
 ;; Migrate
 
 (defn update-file-data-blob-format