✨ Make builtin templates download ondemand if cache is not present

backend/src/app/main.clj

@@ -29,6 +29,7 @@
    [app.redis :as-alias rds]
    [app.rpc :as-alias rpc]
    [app.rpc.doc :as-alias rpc.doc]
+   [app.setup :as-alias setup]
    [app.srepl :as-alias srepl]
    [app.storage :as-alias sto]
    [app.storage.fs :as-alias sto.fs]
@@ -322,11 +323,10 @@
 
     ::rpc/climit         (ig/ref ::rpc/climit)
     ::rpc/rlimit         (ig/ref ::rpc/rlimit)
-
+    ::setup/templates    (ig/ref ::setup/templates)
     ::props              (ig/ref :app.setup/props)
 
     :pool                (ig/ref ::db/pool)
-    :templates           (ig/ref :app.setup/builtin-templates)
     }
 
    :app.rpc.doc/routes
@@ -400,8 +400,7 @@
    {::srepl/port (cf/get :prepl-port 6063)
     ::srepl/host (cf/get :prepl-host "localhost")}
 
-   :app.setup/builtin-templates
+   ::setup/templates {}
-   {::http.client/client (ig/ref ::http.client/client)}
 
    :app.setup/props
    {::db/pool    (ig/ref ::db/pool)

backend/src/app/rpc/commands/binfile.clj

@@ -592,7 +592,7 @@
                   (let [options (-> options
                                     (assoc ::section section)
                                     (assoc ::input input)
-                                    (assoc :conn conn))]
+                                    (assoc ::db/conn conn))]
                     (binding [*options* options]
                       (read-section options))))
                 [:v1/metadata :v1/files :v1/rels :v1/sobjects])
@@ -620,7 +620,7 @@
         (update :components pmap-wrap))))
 
 (defmethod read-section :v1/files
-  [{:keys [conn ::input ::migrate? ::project-id ::timestamp ::overwrite?]}]
+  [{:keys [::db/conn ::input ::migrate? ::project-id ::timestamp ::overwrite?]}]
   (doseq [expected-file-id (-> *state* deref :files)]
     (let [file     (read-obj! input)
           media'   (read-obj! input)
@@ -678,7 +678,7 @@
             (db/delete! conn :file-thumbnail {:file-id file-id'})))))))
 
 (defmethod read-section :v1/rels
-  [{:keys [conn ::input ::timestamp]}]
+  [{:keys [::db/conn ::input ::timestamp]}]
   (let [rels (read-obj! input)]
     ;; Insert all file relations
     (doseq [rel rels]
@@ -693,7 +693,7 @@
         (db/insert! conn :file-library-rel rel)))))
 
 (defmethod read-section :v1/sobjects
-  [{:keys [::sto/storage conn ::input ::overwrite?]}]
+  [{:keys [::sto/storage ::db/conn ::input ::overwrite?]}]
   (let [storage (media/configure-assets-storage storage)
         ids     (read-obj! input)]
 

backend/src/app/rpc/commands/management.clj

@@ -10,6 +10,7 @@
    [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.pages.migrations :as pmg]
+   [app.common.schema :as sm]
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
    [app.db :as db]
@@ -20,6 +21,8 @@
    [app.rpc.commands.projects :as proj]
    [app.rpc.commands.teams :as teams :refer [create-project-role create-project]]
    [app.rpc.doc :as-alias doc]
+   [app.setup :as-alias setup]
+   [app.setup.templates :as tmpl]
    [app.util.blob :as blob]
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
@@ -361,7 +364,6 @@
 
     nil))
 
-
 (s/def ::move-project
   (s/keys :req [::rpc/profile-id]
           :req-un [::team-id ::project-id]))
@@ -376,41 +378,42 @@
 
 ;; --- COMMAND: Clone Template
 
-(declare clone-template)
+(defn- clone-template!
-
+  [{:keys [::db/conn] :as cfg} {:keys [profile-id template-id project-id]}]
-(s/def ::template-id ::us/not-empty-string)
+  (let [template (tmpl/get-template-stream cfg template-id)
-(s/def ::clone-template
-  (s/keys :req [::rpc/profile-id]
-          :req-un [::project-id ::template-id]))
-
-(sv/defmethod ::clone-template
-  "Clone into the specified project the template by its id."
-  {::doc/added "1.16"
-   ::webhooks/event? true}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
-  (db/with-atomic [conn pool]
-    (-> (assoc cfg :conn conn)
-        (clone-template (assoc params :profile-id profile-id)))))
-
-(defn- clone-template
-  [{:keys [conn templates] :as cfg} {:keys [profile-id template-id project-id]}]
-  (let [template (d/seek #(= (:id %) template-id) templates)
         project  (db/get-by-id conn :project project-id {:columns [:id :team-id]})]
 
-    (teams/check-edition-permissions! conn profile-id (:team-id project))
-
     (when-not template
       (ex/raise :type :not-found
                 :code :template-not-found
                 :hint "template not found"))
 
+    (teams/check-edition-permissions! conn profile-id (:team-id project))
+
     (-> cfg
-        (assoc ::binfile/input (:path template))
+        ;; FIXME: maybe reuse the conn instead of creating more
+        ;; connections in the import process?
+        (dissoc ::db/conn)
+        (assoc ::binfile/input template)
         (assoc ::binfile/project-id (:id project))
         (assoc ::binfile/ignore-index-errors? true)
         (assoc ::binfile/migrate? true)
         (binfile/import!))))
 
+(def schema:clone-template
+  [:map {:title "clone-template"}
+   [:project-id ::sm/uuid]
+   [:template-id ::sm/word-string]])
+
+(sv/defmethod ::clone-template
+  "Clone into the specified project the template by its id."
+  {::doc/added "1.16"
+   ::webhooks/event? true
+   ::sm/params schema:clone-template}
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
+  (db/with-atomic [conn pool]
+    (-> (assoc cfg ::db/conn conn)
+        (clone-template! (assoc params :profile-id profile-id)))))
 
 ;; --- COMMAND: Get list of builtin templates
 
@@ -420,9 +423,9 @@
   {::doc/added "1.10"
    ::doc/deprecated "1.19"}
   [cfg _params]
-  (mapv #(select-keys % [:id :name :thumbnail-uri]) (:templates cfg)))
+  (mapv #(select-keys % [:id :name :thumbnail-uri]) (::setup/templates cfg)))
 
 (sv/defmethod ::get-builtin-templates
   {::doc/added "1.19"}
   [cfg _params]
-  (mapv #(select-keys % [:id :name :thumbnail-uri]) (:templates cfg)))
+  (mapv #(select-keys % [:id :name :thumbnail-uri]) (::setup/templates cfg)))

backend/src/app/setup.clj

@@ -12,8 +12,8 @@
    [app.common.uuid :as uuid]
    [app.db :as db]
    [app.main :as-alias main]
-   [app.setup.builtin-templates]
    [app.setup.keys :as keys]
+   [app.setup.templates]
    [buddy.core.codecs :as bc]
    [buddy.core.nonce :as bn]
    [clojure.spec.alpha :as s]

backend/src/app/setup/builtin_templates.clj

@@ -1,72 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.setup.builtin-templates
-  "A service/module that is responsible for download, load & internally
-  expose a set of builtin penpot file templates."
-  (:require
-   [app.common.logging :as l]
-   [app.common.spec :as us]
-   [app.http.client :as http]
-   [clojure.edn :as edn]
-   [clojure.java.io :as io]
-   [clojure.spec.alpha :as s]
-   [datoteka.fs :as fs]
-   [integrant.core :as ig]))
-
-(declare download-all!)
-
-(s/def ::id ::us/not-empty-string)
-(s/def ::name ::us/not-empty-string)
-(s/def ::thumbnail-uri ::us/not-empty-string)
-(s/def ::file-uri ::us/not-empty-string)
-(s/def ::path fs/path?)
-
-(s/def ::template
-  (s/keys :req-un [::id ::name ::thumbnail-uri ::file-uri]
-          :opt-un [::path]))
-
-(defmethod ig/pre-init-spec :app.setup/builtin-templates [_]
-  (s/keys :req [::http/client]))
-
-(defmethod ig/init-key :app.setup/builtin-templates
-  [_ cfg]
-  (let [presets (-> "app/onboarding.edn" io/resource slurp edn/read-string)]
-    (l/info :hint "loading template files" :total (count presets))
-    (let [result (download-all! cfg presets)]
-      (us/conform (s/coll-of ::template) result))))
-
-(defn- download-preset!
-  [cfg {:keys [path file-uri] :as preset}]
-  (let [response (http/req! cfg
-                            {:method :get
-                             :uri file-uri}
-                            {:response-type :input-stream
-                             :sync? true})]
-    (us/verify! (= 200 (:status response)) "unexpected response found on fetching preset")
-    (with-open [output (io/output-stream path)]
-      (with-open [input (io/input-stream (:body response))]
-        (io/copy input output)))))
-
-(defn- download-all!
-  "Download presets to the default directory, if preset is already
-  downloaded, no action will be performed."
-  [cfg presets]
-  (let [dest (fs/join fs/*cwd* "builtin-templates")]
-    (when-not (fs/exists? dest)
-      (fs/create-dir dest))
-
-    (doall
-     (map (fn [item]
-            (let [path (fs/join dest (:id item))
-                  item (assoc item :path path)]
-              (if (fs/exists? path)
-                (l/trace :hint "template file already present" :id (:id item))
-                (do
-                  (l/trace :hint "downloading template file" :id (:id item) :dest (str path))
-                  (download-preset! cfg item)))
-              item))
-          presets))))

backend/src/app/setup/templates.clj

@@ -0,0 +1,65 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.setup.templates
+  "A service/module that is responsible for download, load & internally
+  expose a set of builtin penpot file templates."
+  (:require
+   [app.common.data :as d]
+   [app.common.data.macros :as dm]
+   [app.common.logging :as l]
+   [app.common.schema :as sm]
+   [app.http.client :as http]
+   [app.setup :as-alias setup]
+   [clojure.edn :as edn]
+   [clojure.java.io :as io]
+   [datoteka.fs :as fs]
+   [integrant.core :as ig]))
+
+(def ^:private schema:template
+  [:map {:title "Template"}
+   [:id ::sm/word-string]
+   [:name ::sm/word-string]
+   [:thumbnail-uri ::sm/word-string]
+   [:file-uri ::sm/word-string]])
+
+(def ^:private schema:templates
+  [:vector schema:template])
+
+(defmethod ig/init-key ::setup/templates
+  [_ _]
+  (let [templates (-> "app/onboarding.edn" io/resource slurp edn/read-string)
+        dest      (fs/join fs/*cwd* "builtin-templates")]
+
+    (dm/verify!
+     "expected a valid templates file"
+     (sm/valid? schema:templates templates))
+
+    (doseq [{:keys [id path] :as template} templates]
+      (let [path (or path (fs/join dest id))]
+        (if (fs/exists? path)
+          (l/debug :hint "template file" :id id :state "present" :path (dm/str path))
+          (l/debug :hint "template file" :id id :state "absent"))))
+
+    templates))
+
+(defn get-template-stream
+  [cfg template-id]
+  (when-let [template (d/seek #(= (:id %) template-id)
+                              (::setup/templates cfg))]
+    (let [dest (fs/join fs/*cwd* "builtin-templates")
+          path (or (:path template) (fs/join dest template-id))]
+      (if (fs/exists? path)
+        (io/input-stream path)
+        (let [resp (http/req! cfg
+                              {:method :get :uri (:file-uri template)}
+                              {:response-type :input-stream :sync? true})]
+
+          (dm/verify!
+           "unexpected response found on fetching template"
+           (= 200 (:status resp)))
+
+          (io/input-stream (:body resp)))))))

backend/test/backend_tests/helpers.clj

@@ -128,7 +128,7 @@
                      (assoc-in [::db/pool ::db/uri] (:database-uri config))
                      (assoc-in [::db/pool ::db/username] (:database-username config))
                      (assoc-in [::db/pool ::db/password] (:database-password config))
-                     (assoc-in [:app.rpc/methods :templates] templates)
+                     (assoc-in [:app.rpc/methods :app.setup/templates] templates)
                      (dissoc :app.srepl/server
                              :app.http/server
                              :app.http/router
@@ -136,7 +136,7 @@
                              :app.auth.oidc/gitlab-provider
                              :app.auth.oidc/github-provider
                              :app.auth.oidc/generic-provider
-                             :app.setup/builtin-templates
+                             :app.setup/templates
                              :app.auth.oidc/routes
                              :app.worker/monitor
                              :app.http.oauth/handler