✨ Move the dashboard grid thumbnails to backend cache

backend/src/app/migrations.clj

@@ -214,6 +214,9 @@
 
    {:name "0068-mod-storage-object-table"
     :fn (mg/resource "app/migrations/sql/0068-mod-storage-object-table.sql")}
+
+   {:name "0069-add-file-thumbnail-table"
+    :fn (mg/resource "app/migrations/sql/0069-add-file-thumbnail-table.sql")}
    ])
 
 

backend/src/app/migrations/sql/0066-add-frame-thumbnail-table.sql

@@ -8,3 +8,6 @@ CREATE TABLE file_frame_thumbnail (
 
   PRIMARY KEY(file_id, frame_id)
 );
+
+ALTER TABLE file_frame_thumbnail
+  ALTER COLUMN data SET STORAGE external;

backend/src/app/migrations/sql/0069-add-file-thumbnail-table.sql

@@ -0,0 +1,14 @@
+CREATE TABLE file_thumbnail (
+  file_id uuid NOT NULL REFERENCES file(id) ON DELETE CASCADE,
+  revn bigint NOT NULL,
+  created_at timestamptz NOT NULL DEFAULT now(),
+  updated_at timestamptz NOT NULL DEFAULT now(),
+  deleted_at timestamptz NULL,
+  data text NULL,
+  props jsonb NULL,
+  PRIMARY KEY(file_id, revn)
+);
+
+ALTER TABLE file_thumbnail
+  ALTER COLUMN data SET STORAGE external,
+  ALTER COLUMN props SET STORAGE external;

backend/src/app/rpc/helpers.clj

@@ -0,0 +1,16 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.rpc.helpers
+  "General purpose RPC helpers."
+  (:require [app.common.data.macros :as dm]))
+
+(defn http-cache
+  [{:keys [max-age]}]
+  (fn [_ response]
+    (let [exp (if (integer? max-age) max-age (inst-ms max-age))
+          val (dm/fmt "max-age=%" (int (/ exp 1000.0)))]
+      (update response :headers assoc "cache-control" val))))

backend/src/app/rpc/mutations/files.clj

@@ -487,14 +487,34 @@
           update set data = ?;")
 
 (s/def ::data ::us/string)
-(s/def ::upsert-frame-thumbnail
+(s/def ::upsert-file-frame-thumbnail
   (s/keys :req-un [::profile-id ::file-id ::frame-id ::data]))
 
-(sv/defmethod ::upsert-frame-thumbnail
+(sv/defmethod ::upsert-file-frame-thumbnail
   [{:keys [pool] :as cfg} {:keys [profile-id file-id frame-id data]}]
   (db/with-atomic [conn pool]
     (files/check-edition-permissions! conn profile-id file-id)
     (db/exec-one! conn [sql:upsert-frame-thumbnail file-id frame-id data data])
     nil))
 
+;; --- Mutation: Upsert file thumbnail
 
+(def sql:upsert-file-thumbnail
+  "insert into file_thumbnail(file_id, revn, data, props)
+   values (?, ?, ?, ?)
+       on conflict(file_id, revn) do
+          update set data = ?, updated_at=now();")
+
+(s/def ::revn  ::us/integer)
+(s/def ::props (s/map-of ::us/keyword any?))
+(s/def ::upsert-file-thumbnail
+  (s/keys :req-un [::profile-id ::file-id ::revn ::data ::props]))
+
+(sv/defmethod ::upsert-file-thumbnail
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id revn data props]}]
+  (db/with-atomic [conn pool]
+    (files/check-edition-permissions! conn profile-id file-id)
+    (let [props (db/tjson (or props {}))]
+      (db/exec-one! conn [sql:upsert-file-thumbnail
+                          file-id revn data props data])
+      nil)))

backend/src/app/rpc/queries/files.clj

@@ -7,11 +7,14 @@
 (ns app.rpc.queries.files
   (:require
    [app.common.data :as d]
+   [app.common.exceptions :as ex]
    [app.common.pages.helpers :as cph]
    [app.common.pages.migrations :as pmg]
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
    [app.db :as db]
+   [app.db.sql :as sql]
+   [app.rpc.helpers :as rpch]
    [app.rpc.permissions :as perms]
    [app.rpc.queries.projects :as projects]
    [app.rpc.queries.share-link :refer [retrieve-share-link]]
@@ -267,7 +270,9 @@
   [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as props}]
   (check-read-permissions! pool profile-id file-id)
   (let [file (retrieve-file cfg file-id)]
-    (get-thumbnail-data file props)))
+    {:data (get-thumbnail-data file props)
+     :file-id file-id
+     :revn (:revn file)}))
 
 (defn get-thumbnail-data
   [{:keys [data] :as file} props]
@@ -325,7 +330,6 @@
 
     (update data :objects update-objects)))
 
-
 ;; --- Query: Shared Library Files
 
 (def ^:private sql:team-shared-files
@@ -424,22 +428,48 @@
   (teams/check-read-permissions! pool profile-id team-id)
   (db/exec! pool [sql:team-recent-files team-id]))
 
+;; --- QUERY: get all file frame thumbnails
 
-;; --- QUERY: get the thumbnail for an frame
+(s/def ::file-frame-thumbnails
+  (s/keys :req-un [::profile-id ::file-id]
+          :opt-un [::frame-id]))
 
-(def ^:private sql:file-frame-thumbnail
+(sv/defmethod ::file-frame-thumbnails
-  "select data
-     from file_frame_thumbnail
-    where file_id = ?
-      and frame_id = ?")
-
-(s/def ::file-frame-thumbnail
-  (s/keys :req-un [::profile-id ::file-id ::frame-id]))
-
-(sv/defmethod ::file-frame-thumbnail
   [{:keys [pool]} {:keys [profile-id file-id frame-id]}]
   (check-read-permissions! pool profile-id file-id)
-  (db/exec-one! pool [sql:file-frame-thumbnail file-id frame-id]))
+  (let [params (cond-> {:file-id file-id}
+                 frame-id (assoc :frame-id frame-id))
+        rows   (db/query pool :file-frame-thumbnail params)]
+    (d/group-by :frame-id :data rows)))
+
+;; --- QUERY: get file thumbnail
+
+(s/def ::revn ::us/integer)
+
+(s/def ::file-thumbnail
+  (s/keys :req-un [::profile-id ::file-id]
+          :opt-un [::revn]))
+
+(sv/defmethod ::file-thumbnail
+  [{:keys [pool]} {:keys [profile-id file-id revn]}]
+  (check-read-permissions! pool profile-id file-id)
+  (let [sql (sql/select :file-thumbnail
+                        (cond-> {:file-id file-id}
+                          revn (assoc :revn revn))
+                        {:limit 1
+                         :order-by [[:revn :desc]]})
+
+        row (db/exec-one! pool sql)]
+
+    (when-not row
+      (ex/raise :type :not-found
+                :code :file-thumbnail-not-found))
+
+    (with-meta {:data (:data row)
+                :props (some-> (:props row) db/decode-transit-pgobject)
+                :revn (:revn row)
+                :file-id (:file-id row)}
+      {:transform-response (rpch/http-cache {:max-age (* 1000 60 60)})})))
 
 ;; --- Helpers
 

frontend/src/app/main/ui/dashboard/grid.cljs

@@ -27,8 +27,6 @@
    [app.util.timers :as ts]
    [app.util.webapi :as wapi]
    [beicon.core :as rx]
-   [cuerdas.core :as str]
-   [promesa.core :as p]
    [rumext.alpha :as mf]))
 
 (log/set-level! :warn)
@@ -38,57 +36,15 @@
 (def ^:const CACHE-NAME "penpot")
 (def ^:const CACHE-URL "https://penpot.app/cache/")
 
-
 (defn use-thumbnail-cache
   "Creates some hooks to handle the files thumbnails cache"
   [file]
-
+  (mf/use-fn
-  (let [cache-url (str CACHE-URL (:id file) "/" (:revn file) ".svg")
-        get-thumbnail
-        (mf/use-callback
-         (mf/deps cache-url)
-         (fn []
-           (p/let [response (.match js/caches cache-url)]
-             (when (some? response)
-               (p/let [blob         (.blob response)
-                       svg-content  (.text blob)
-                       headers      (.-headers response)
-                       fonts-header (or (.get headers "X-PENPOT-FONTS") "")
-                       fonts        (into #{}
-                                          (remove #(= "" %))
-                                          (str/split fonts-header ","))]
-                 {:svg   svg-content
-                  :fonts fonts})))))
-
-        cache-thumbnail
-        (mf/use-callback
-         (mf/deps cache-url)
-         (fn [{:keys [svg fonts]}]
-           (p/let [cache    (.open js/caches CACHE-NAME)
-                   blob     (js/Blob. #js [svg] #js {:type "image/svg"})
-                   fonts    (str/join "," fonts)
-                   headers  (js/Headers. #js {"X-PENPOT-FONTS" fonts})
-                   response (js/Response. blob #js {:headers headers})]
-             (.put cache cache-url response))))]
-
-    (mf/use-callback
    (mf/deps (:id file) (:revn file))
    (fn []
-       (->> (rx/from (get-thumbnail))
+     (wrk/ask! {:cmd :thumbnails/generate
-            (rx/merge-map
+                :revn (:revn file)
-             (fn [thumb-data]
+                :file-id (:id file)}))))
-               (log/debug :msg "retrieve thumbnail" :file (:id file) :revn (:revn file)
-                          :cache (if (some? thumb-data) :hit :miss))
-
-               (if (some? thumb-data)
-                 (rx/of thumb-data)
-                 (->> (wrk/ask! {:cmd :thumbnails/generate
-                                 :file-id (:id file)})
-                      (rx/tap cache-thumbnail)))))
-
-            ;; If we have a problem we delegate to the thumbnail generation
-            (rx/catch #(wrk/ask! {:cmd :thumbnails/generate
-                                  :file-id (:id file)})))))))
 
 (mf/defc grid-item-thumbnail
   {::mf/wrap [mf/memo]}
@@ -100,10 +56,10 @@
      (mf/deps file)
      (fn []
        (->> (generate)
-            (rx/subs (fn [{:keys [svg fonts]}]
+            (rx/subs (fn [{:keys [data fonts] :as params}]
                        (run! fonts/ensure-loaded! fonts)
                        (when-let [node (mf/ref-val container)]
-                         (dom/set-html! node svg)))))))
+                         (dom/set-html! node data)))))))
 
     [:div.grid-item-th {:style {:background-color (get-in file [:data :options :background])}
                         :ref container}

frontend/src/app/render.cljs

@@ -29,7 +29,6 @@
           :version (:full @cf/version)
           :public-uri (str cf/public-uri))
 
-
 (defn- parse-params
   [loc]
   (let [href (unchecked-get loc "href")]

frontend/src/app/worker/thumbnails.cljs

@@ -16,6 +16,10 @@
    [beicon.core :as rx]
    [rumext.alpha :as mf]))
 
+(defn- not-found?
+  [{:keys [type]}]
+  (= :not-found type))
+
 (defn- handle-response
   [response]
   (cond
@@ -29,30 +33,70 @@
     (rx/throw {:type :unexpected
                :code (:error response)})))
 
-(defn- request-thumbnail
+(defn- request-data-for-thumbnail
-  [file-id]
+  [file-id revn]
-  (let [uri     (u/join (cfg/get-public-uri) "api/rpc/query/file-data-for-thumbnail")
+  (let [path    "api/rpc/query/file-data-for-thumbnail"
         params  {:file-id file-id
+                 :revn revn
                  :strip-frames-with-thumbnails true}
         request {:method :get
-                 :uri uri
+                 :uri (u/join (cfg/get-public-uri) path)
                  :credentials "include"
                  :query params}]
     (->> (http/send! request)
          (rx/map http/conditional-decode-transit)
          (rx/mapcat handle-response))))
 
-(defn render-frame
+(defn- request-thumbnail
-  [data]
+  [file-id revn]
+  (let [path    "api/rpc/query/file-thumbnail"
+        params  {:file-id file-id
+                 :revn revn}
+        request {:method :get
+                 :uri (u/join (cfg/get-public-uri) path)
+                 :credentials "include"
+                 :query params}]
+    (->> (http/send! request)
+         (rx/map http/conditional-decode-transit)
+         (rx/mapcat handle-response))))
+
+(defn- render-thumbnail
+  [{:keys [data file-id revn] :as params}]
   (let [elem (if-let [frame (:thumbnail-frame data)]
                (mf/element render/frame-svg #js {:objects (:objects data) :frame frame})
                (mf/element render/page-svg #js {:data data :width "290" :height "150" :thumbnails? true}))]
-    (rds/renderToStaticMarkup elem)))
+    {:data (rds/renderToStaticMarkup elem)
+     :fonts @fonts/loaded
+     :file-id file-id
+     :revn revn}))
+
+(defn- persist-thumbnail
+  [{:keys [file-id data revn fonts]}]
+  (let [path    "api/rpc/mutation/upsert-file-thumbnail"
+        params  {:file-id file-id
+                 :revn revn
+                 :props {:fonts fonts}
+                 :data data}
+        request {:method :post
+                 :uri (u/join (cfg/get-public-uri) path)
+                 :credentials "include"
+                 :body (http/transit-data params)}]
+    (->> (http/send! request)
+         (rx/map http/conditional-decode-transit)
+         (rx/mapcat handle-response)
+         (rx/map (constantly params)))))
 
 (defmethod impl/handler :thumbnails/generate
-  [{:keys [file-id] :as message}]
+  [{:keys [file-id revn] :as message}]
-  (->> (request-thumbnail file-id)
+  (letfn [(on-result [{:keys [data props]}]
-       (rx/map
+            {:data data
-        (fn [data]
+             :fonts (:fonts props)})
-          {:svg (render-frame data)
+
-           :fonts @fonts/loaded}))))
+          (on-cache-miss [_]
+            (->> (request-data-for-thumbnail file-id revn)
+                 (rx/map render-thumbnail)
+                 (rx/mapcat persist-thumbnail)))]
+
+    (->> (request-thumbnail file-id revn)
+         (rx/catch not-found? on-cache-miss)
+         (rx/map on-result))))