Merge pull request #3193 from penpot/niwinz-thumbnails-1

🎉 Allow submit thumbnails using multipart

backend/src/app/main.clj

@@ -381,7 +381,8 @@
     ::sto/storage (ig/ref ::sto/storage)}
 
    :app.tasks.file-gc/handler
-   {::db/pool (ig/ref ::db/pool)}
+   {::db/pool     (ig/ref ::db/pool)
+    ::sto/storage (ig/ref ::sto/storage)}
 
    :app.tasks.file-xlog-gc/handler
    {::db/pool (ig/ref ::db/pool)}

backend/src/app/media.clj

@@ -29,6 +29,9 @@
    org.im4java.core.IMOperation
    org.im4java.core.Info))
 
+(def default-max-file-size
+  (* 1024 1024 30)) ; 30 MiB
+
 (s/def ::path fs/path?)
 (s/def ::filename string?)
 (s/def ::size integer?)
@@ -54,6 +57,16 @@
 
    upload))
 
+(defn validate-media-size!
+  [upload]
+  (when (> (:size upload) (cf/get :media-max-file-size default-max-file-size))
+    (ex/raise :type :restriction
+              :code :media-max-file-size-reached
+              :hint (str/ffmt "the uploaded file size % is greater than the maximum %"
+                              (:size upload)
+                              default-max-file-size)))
+  upload)
+
 (defmulti process :cmd)
 (defmulti process-error class)
 

backend/src/app/migrations.clj

@@ -316,7 +316,15 @@
     :fn (mg/resource "app/migrations/sql/0101-mod-server-error-report-table.sql")}
 
    {:name "0102-mod-access-token-table"
-    :fn (mg/resource "app/migrations/sql/0102-mod-access-token-table.sql")}])
+    :fn (mg/resource "app/migrations/sql/0102-mod-access-token-table.sql")}
+
+   {:name "0103-mod-file-object-thumbnail-table"
+    :fn (mg/resource "app/migrations/sql/0103-mod-file-object-thumbnail-table.sql")}
+
+   {:name "0104-mod-file-thumbnail-table"
+    :fn (mg/resource "app/migrations/sql/0104-mod-file-thumbnail-table.sql")}
+
+   ])
 
 (defn apply-migrations!
   [pool name migrations]

backend/src/app/migrations/sql/0103-mod-file-object-thumbnail-table.sql

@@ -0,0 +1,2 @@
+ALTER TABLE file_object_thumbnail
+  ADD COLUMN media_id uuid NULL REFERENCES storage_object(id) ON DELETE CASCADE DEFERRABLE;

backend/src/app/migrations/sql/0104-mod-file-thumbnail-table.sql

@@ -0,0 +1,2 @@
+ALTER TABLE file_thumbnail
+  ADD COLUMN media_id uuid NULL REFERENCES storage_object(id) ON DELETE CASCADE DEFERRABLE;

backend/src/app/rpc.clj

@@ -170,6 +170,7 @@
           'app.rpc.commands.files-share
           'app.rpc.commands.files-temp
           'app.rpc.commands.files-update
+          'app.rpc.commands.files-thumbnails
           'app.rpc.commands.ldap
           'app.rpc.commands.management
           'app.rpc.commands.media

backend/src/app/rpc/commands/files.clj

@@ -9,16 +9,13 @@
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
-   [app.common.geom.shapes :as gsh]
    [app.common.pages.helpers :as cph]
    [app.common.pages.migrations :as pmg]
    [app.common.spec :as us]
    [app.common.types.components-list :as ctkl]
    [app.common.types.file :as ctf]
-   [app.common.types.shape-tree :as ctt]
    [app.config :as cf]
    [app.db :as db]
-   [app.db.sql :as sql]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.rpc :as-alias rpc]
@@ -332,41 +329,6 @@
       (-> (get-file-fragment conn file-id fragment-id)
           (rph/with-http-cache long-cache-duration)))))
 
-;; --- COMMAND QUERY: get-file-object-thumbnails
-
-(defn get-object-thumbnails
-  ([conn file-id]
-   (let [sql (str/concat
-              "select object_id, data "
-              "  from file_object_thumbnail"
-              " where file_id=?")]
-     (->> (db/exec! conn [sql file-id])
-          (d/index-by :object-id :data))))
-
-  ([conn file-id object-ids]
-   (let [sql (str/concat
-              "select object_id, data "
-              "  from file_object_thumbnail"
-              " where file_id=? and object_id = ANY(?)")
-         ids (db/create-array conn "text" (seq object-ids))]
-     (->> (db/exec! conn [sql file-id ids])
-          (d/index-by :object-id :data)))))
-
-(s/def ::get-file-object-thumbnails
-  (s/keys :req [::rpc/profile-id] :req-un [::file-id]))
-
-(sv/defmethod ::get-file-object-thumbnails
-  "Retrieve a file object thumbnails."
-  {::doc/added "1.17"
-   ::cond/get-object #(get-minimal-file %1 (:file-id %2))
-   ::cond/reuse-key? true
-   ::cond/key-fn get-file-etag}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
-  (dm/with-open [conn (db/open pool)]
-    (check-read-permissions! conn profile-id file-id)
-    (get-object-thumbnails conn file-id)))
-
-
 ;; --- COMMAND QUERY: get-project-files
 
 (def ^:private sql:project-files
@@ -662,161 +624,6 @@
     (teams/check-read-permissions! conn profile-id team-id)
     (get-team-recent-files conn team-id)))
 
-
-;; --- COMMAND QUERY: get-file-thumbnail
-
-(defn get-file-thumbnail
-  [conn file-id revn]
-  (let [sql (sql/select :file-thumbnail
-                        (cond-> {:file-id file-id}
-                          revn (assoc :revn revn))
-                        {:limit 1
-                         :order-by [[:revn :desc]]})
-        row (db/exec-one! conn sql)]
-    (when-not row
-      (ex/raise :type :not-found
-                :code :file-thumbnail-not-found))
-
-    {:data (:data row)
-     :props (some-> (:props row) db/decode-transit-pgobject)
-     :revn (:revn row)
-     :file-id (:file-id row)}))
-
-(s/def ::revn ::us/integer)
-
-(s/def ::get-file-thumbnail
-  (s/keys :req [::rpc/profile-id]
-          :req-un [::file-id]
-          :opt-un [::revn]))
-
-(sv/defmethod ::get-file-thumbnail
-  {::doc/added "1.17"}
-  [{:keys [::db/pool]} {:keys [::rpc/profile-id file-id revn]}]
-  (dm/with-open [conn (db/open pool)]
-    (check-read-permissions! conn profile-id file-id)
-    (-> (get-file-thumbnail conn file-id revn)
-        (rph/with-http-cache long-cache-duration))))
-
-
-;; --- COMMAND QUERY: get-file-data-for-thumbnail
-
-;; FIXME: performance issue
-;;
-;; We need to improve how we set frame for thumbnail in order to avoid
-;; loading all pages into memory for find the frame set for thumbnail.
-
-(defn get-file-data-for-thumbnail
-  [conn {:keys [data id] :as file}]
-  (letfn [;; function responsible on finding the frame marked to be
-          ;; used as thumbnail; the returned frame always have
-          ;; the :page-id set to the page that it belongs.
-
-          (get-thumbnail-frame [data]
-            ;; NOTE: this is a hack for avoid perform blocking
-            ;; operation inside the for loop, clojure lazy-seq uses
-            ;; synchronized blocks that does not plays well with
-            ;; virtual threads, so we need to perform the load
-            ;; operation first. This operation forces all pointer maps
-            ;; load into the memory.
-            (->> (-> data :pages-index vals)
-                 (filter pmap/pointer-map?)
-                 (run! pmap/load!))
-
-            ;; Then proceed to find the frame set for thumbnail
-
-            (d/seek :use-for-thumbnail?
-                    (for [page  (-> data :pages-index vals)
-                          frame (-> page :objects ctt/get-frames)]
-                      (assoc frame :page-id (:id page)))))
-
-          ;; function responsible to filter objects data structure of
-          ;; all unneeded shapes if a concrete frame is provided. If no
-          ;; frame, the objects is returned untouched.
-          (filter-objects [objects frame-id]
-            (d/index-by :id (cph/get-children-with-self objects frame-id)))
-
-          ;; function responsible of assoc available thumbnails
-          ;; to frames and remove all children shapes from objects if
-          ;; thumbnails is available
-          (assoc-thumbnails [objects page-id thumbnails]
-            (loop [objects objects
-                   frames  (filter cph/frame-shape? (vals objects))]
-
-              (if-let [frame  (-> frames first)]
-                (let [frame-id (:id frame)
-                      object-id (str page-id frame-id)
-                      frame (if-let [thumb (get thumbnails object-id)]
-                              (assoc frame :thumbnail thumb :shapes [])
-                              (dissoc frame :thumbnail))
-
-                      children-ids
-                      (cph/get-children-ids objects frame-id)
-
-                      bounds
-                      (when (:show-content frame)
-                        (gsh/selection-rect (concat [frame] (->> children-ids (map (d/getf objects))))))
-
-                      frame
-                      (cond-> frame
-                        (some? bounds)
-                        (assoc :children-bounds bounds))]
-
-                  (if (:thumbnail frame)
-                    (recur (-> objects
-                               (assoc frame-id frame)
-                               (d/without-keys children-ids))
-                           (rest frames))
-                    (recur (assoc objects frame-id frame)
-                           (rest frames))))
-
-                objects)))]
-
-    (binding [pmap/*load-fn* (partial load-pointer conn id)]
-      (let [frame     (get-thumbnail-frame data)
-            frame-id  (:id frame)
-            page-id   (or (:page-id frame)
-                          (-> data :pages first))
-
-            page      (dm/get-in data [:pages-index page-id])
-            page      (cond-> page (pmap/pointer-map? page) deref)
-            frame-ids (if (some? frame) (list frame-id) (map :id (ctt/get-frames (:objects page))))
-
-            obj-ids   (map #(str page-id %) frame-ids)
-            thumbs    (get-object-thumbnails conn id obj-ids)]
-
-        (cond-> page
-          ;; If we have frame, we need to specify it on the page level
-          ;; and remove the all other unrelated objects.
-          (some? frame-id)
-          (-> (assoc :thumbnail-frame-id frame-id)
-              (update :objects filter-objects frame-id))
-
-          ;; Assoc the available thumbnails and prune not visible shapes
-          ;; for avoid transfer unnecessary data.
-          :always
-          (update :objects assoc-thumbnails page-id thumbs))))))
-
-(s/def ::get-file-data-for-thumbnail
-  (s/keys :req [::rpc/profile-id]
-          :req-un [::file-id]
-          :opt-un [::features]))
-
-(sv/defmethod ::get-file-data-for-thumbnail
-  "Retrieves the data for generate the thumbnail of the file. Used
-  mainly for render thumbnails on dashboard."
-  {::doc/added "1.17"}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id features] :as props}]
-  (dm/with-open [conn (db/open pool)]
-    (check-read-permissions! conn profile-id file-id)
-    ;; NOTE: we force here the "storage/pointer-map" feature, because
-    ;; it used internally only and is independent if user supports it
-    ;; or not.
-    (let [feat (into #{"storage/pointer-map"} features)
-          file (get-file conn file-id feat)]
-      {:file-id file-id
-       :revn (:revn file)
-       :page (get-file-data-for-thumbnail conn file)})))
-
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; MUTATION COMMANDS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1026,66 +833,3 @@
     (check-edition-permissions! conn profile-id file-id)
     (->  (ignore-sync conn params)
          (update :features db/decode-pgarray #{}))))
-
-
-;; --- MUTATION COMMAND: upsert-file-object-thumbnail
-
-(def sql:upsert-object-thumbnail
-  "insert into file_object_thumbnail(file_id, object_id, data)
-   values (?, ?, ?)
-       on conflict(file_id, object_id) do
-          update set data = ?;")
-
-(defn upsert-file-object-thumbnail!
-  [conn {:keys [file-id object-id data]}]
-  (if data
-    (db/exec-one! conn [sql:upsert-object-thumbnail file-id object-id data data])
-    (db/delete! conn :file-object-thumbnail {:file-id file-id :object-id object-id})))
-
-(s/def ::data (s/nilable ::us/string))
-(s/def ::thumbs/object-id ::us/string)
-(s/def ::upsert-file-object-thumbnail
-  (s/keys :req [::rpc/profile-id]
-          :req-un [::file-id ::thumbs/object-id]
-          :opt-un [::data]))
-
-(sv/defmethod ::upsert-file-object-thumbnail
-  {::doc/added "1.17"
-   ::audit/skip true}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
-  (db/with-atomic [conn pool]
-    (check-edition-permissions! conn profile-id file-id)
-    (upsert-file-object-thumbnail! conn params)
-    nil))
-
-;; --- MUTATION COMMAND: upsert-file-thumbnail
-
-(def ^:private sql:upsert-file-thumbnail
-  "insert into file_thumbnail (file_id, revn, data, props)
-   values (?, ?, ?, ?::jsonb)
-       on conflict(file_id, revn) do
-          update set data = ?, props=?, updated_at=now();")
-
-(defn- upsert-file-thumbnail!
-  [conn {:keys [file-id revn data props]}]
-  (let [props (db/tjson (or props {}))]
-    (db/exec-one! conn [sql:upsert-file-thumbnail
-                        file-id revn data props data props])))
-
-(s/def ::revn ::us/integer)
-(s/def ::props map?)
-(s/def ::upsert-file-thumbnail
-  (s/keys :req [::rpc/profile-id]
-          :req-un [::file-id ::revn ::data ::props]))
-
-(sv/defmethod ::upsert-file-thumbnail
-  "Creates or updates the file thumbnail. Mainly used for paint the
-  grid thumbnails."
-  {::doc/added "1.17"
-   ::audit/skip true}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
-  (db/with-atomic [conn pool]
-    (check-edition-permissions! conn profile-id file-id)
-    (when-not (db/read-only? conn)
-      (upsert-file-thumbnail! conn params))
-    nil))

backend/src/app/rpc/commands/files_thumbnails.clj

@@ -0,0 +1,368 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.rpc.commands.files-thumbnails
+  (:require
+   [app.common.data :as d]
+   [app.common.data.macros :as dm]
+   [app.common.exceptions :as ex]
+   [app.common.geom.shapes :as gsh]
+   [app.common.pages.helpers :as cph]
+   [app.common.spec :as us]
+   [app.common.types.shape-tree :as ctt]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.db.sql :as sql]
+   [app.loggers.audit :as-alias audit]
+   [app.loggers.webhooks :as-alias webhooks]
+   [app.media :as media]
+   [app.rpc :as-alias rpc]
+   [app.rpc.commands.files :as files]
+   [app.rpc.cond :as-alias cond]
+   [app.rpc.doc :as-alias doc]
+   [app.rpc.helpers :as rph]
+   [app.storage :as sto]
+   [app.util.pointer-map :as pmap]
+   [app.util.services :as sv]
+   [app.util.time :as dt]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]))
+
+;; --- FEATURES
+
+(def long-cache-duration
+  (dt/duration {:days 7}))
+
+;; --- COMMAND QUERY: get-file-object-thumbnails
+
+(defn- get-public-uri
+  [media-id]
+  (str (cf/get :public-uri) "/assets/by-id/" media-id))
+
+(defn- get-object-thumbnails
+  ([conn file-id]
+   (let [sql (str/concat
+              "select object_id, data, media_id "
+              "  from file_object_thumbnail"
+              " where file_id=?")]
+     (->> (db/exec! conn [sql file-id])
+          (d/index-by :object-id (fn [row]
+                                   (or (some-> row :media-id get-public-uri)
+                                       (:data row))))
+          (d/without-nils))))
+
+  ([conn file-id object-ids]
+   (let [sql (str/concat
+              "select object_id, data "
+              "  from file_object_thumbnail"
+              " where file_id=? and object_id = ANY(?)")
+         ids (db/create-array conn "text" (seq object-ids))]
+     (->> (db/exec! conn [sql file-id ids])
+          (d/index-by :object-id (fn [row]
+                                   (or (some-> row :media-id get-public-uri)
+                                       (:data row))))))))
+
+(s/def ::file-id ::us/uuid)
+(s/def ::get-file-object-thumbnails
+  (s/keys :req [::rpc/profile-id] :req-un [::file-id]))
+
+(sv/defmethod ::get-file-object-thumbnails
+  "Retrieve a file object thumbnails."
+  {::doc/added "1.17"
+   ::cond/get-object #(files/get-minimal-file %1 (:file-id %2))
+   ::cond/reuse-key? true
+   ::cond/key-fn files/get-file-etag}
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
+  (dm/with-open [conn (db/open pool)]
+    (files/check-read-permissions! conn profile-id file-id)
+    (get-object-thumbnails conn file-id)))
+
+;; --- COMMAND QUERY: get-file-thumbnail
+
+;; FIXME: refactor to support uploading data to storage
+
+(defn get-file-thumbnail
+  [conn file-id revn]
+  (let [sql (sql/select :file-thumbnail
+                        (cond-> {:file-id file-id}
+                          revn (assoc :revn revn))
+                        {:limit 1
+                         :order-by [[:revn :desc]]})
+        row (db/exec-one! conn sql)]
+    (when-not row
+      (ex/raise :type :not-found
+                :code :file-thumbnail-not-found))
+
+    {:data (:data row)
+     :props (some-> (:props row) db/decode-transit-pgobject)
+     :revn (:revn row)
+     :file-id (:file-id row)}))
+
+(s/def ::revn ::us/integer)
+
+(s/def ::get-file-thumbnail
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::file-id]
+          :opt-un [::revn]))
+
+(sv/defmethod ::get-file-thumbnail
+  "Method used in frontend for obtain the file thumbnail (used in the
+  dashboard)."
+  {::doc/added "1.17"}
+  [{:keys [::db/pool]} {:keys [::rpc/profile-id file-id revn]}]
+  (dm/with-open [conn (db/open pool)]
+    (files/check-read-permissions! conn profile-id file-id)
+    (-> (get-file-thumbnail conn file-id revn)
+        (rph/with-http-cache long-cache-duration))))
+
+
+;; --- COMMAND QUERY: get-file-data-for-thumbnail
+
+;; FIXME: performance issue, handle new media_id
+;;
+;; We need to improve how we set frame for thumbnail in order to avoid
+;; loading all pages into memory for find the frame set for thumbnail.
+
+(defn get-file-data-for-thumbnail
+  [conn {:keys [data id] :as file}]
+  (letfn [;; function responsible on finding the frame marked to be
+          ;; used as thumbnail; the returned frame always have
+          ;; the :page-id set to the page that it belongs.
+
+          (get-thumbnail-frame [data]
+            ;; NOTE: this is a hack for avoid perform blocking
+            ;; operation inside the for loop, clojure lazy-seq uses
+            ;; synchronized blocks that does not plays well with
+            ;; virtual threads, so we need to perform the load
+            ;; operation first. This operation forces all pointer maps
+            ;; load into the memory.
+            (->> (-> data :pages-index vals)
+                 (filter pmap/pointer-map?)
+                 (run! pmap/load!))
+
+            ;; Then proceed to find the frame set for thumbnail
+
+            (d/seek :use-for-thumbnail?
+                    (for [page  (-> data :pages-index vals)
+                          frame (-> page :objects ctt/get-frames)]
+                      (assoc frame :page-id (:id page)))))
+
+          ;; function responsible to filter objects data structure of
+          ;; all unneeded shapes if a concrete frame is provided. If no
+          ;; frame, the objects is returned untouched.
+          (filter-objects [objects frame-id]
+            (d/index-by :id (cph/get-children-with-self objects frame-id)))
+
+          ;; function responsible of assoc available thumbnails
+          ;; to frames and remove all children shapes from objects if
+          ;; thumbnails is available
+          (assoc-thumbnails [objects page-id thumbnails]
+            (loop [objects objects
+                   frames  (filter cph/frame-shape? (vals objects))]
+
+              (if-let [frame  (-> frames first)]
+                (let [frame-id (:id frame)
+                      object-id (str page-id frame-id)
+                      frame (if-let [thumb (get thumbnails object-id)]
+                              (assoc frame :thumbnail thumb :shapes [])
+                              (dissoc frame :thumbnail))
+
+                      children-ids
+                      (cph/get-children-ids objects frame-id)
+
+                      bounds
+                      (when (:show-content frame)
+                        (gsh/selection-rect (concat [frame] (->> children-ids (map (d/getf objects))))))
+
+                      frame
+                      (cond-> frame
+                        (some? bounds)
+                        (assoc :children-bounds bounds))]
+
+                  (if (:thumbnail frame)
+                    (recur (-> objects
+                               (assoc frame-id frame)
+                               (d/without-keys children-ids))
+                           (rest frames))
+                    (recur (assoc objects frame-id frame)
+                           (rest frames))))
+
+                objects)))]
+
+    (binding [pmap/*load-fn* (partial files/load-pointer conn id)]
+      (let [frame     (get-thumbnail-frame data)
+            frame-id  (:id frame)
+            page-id   (or (:page-id frame)
+                          (-> data :pages first))
+
+            page      (dm/get-in data [:pages-index page-id])
+            page      (cond-> page (pmap/pointer-map? page) deref)
+            frame-ids (if (some? frame) (list frame-id) (map :id (ctt/get-frames (:objects page))))
+
+            obj-ids   (map #(str page-id %) frame-ids)
+            thumbs    (get-object-thumbnails conn id obj-ids)]
+
+        (cond-> page
+          ;; If we have frame, we need to specify it on the page level
+          ;; and remove the all other unrelated objects.
+          (some? frame-id)
+          (-> (assoc :thumbnail-frame-id frame-id)
+              (update :objects filter-objects frame-id))
+
+          ;; Assoc the available thumbnails and prune not visible shapes
+          ;; for avoid transfer unnecessary data.
+          :always
+          (update :objects assoc-thumbnails page-id thumbs))))))
+
+(s/def ::get-file-data-for-thumbnail
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::file-id]
+          :opt-un [::features]))
+
+(sv/defmethod ::get-file-data-for-thumbnail
+  "Retrieves the data for generate the thumbnail of the file. Used
+  mainly for render thumbnails on dashboard."
+  {::doc/added "1.17"}
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id features] :as props}]
+  (dm/with-open [conn (db/open pool)]
+    (files/check-read-permissions! conn profile-id file-id)
+    ;; NOTE: we force here the "storage/pointer-map" feature, because
+    ;; it used internally only and is independent if user supports it
+    ;; or not.
+    (let [feat (into #{"storage/pointer-map"} features)
+          file (files/get-file conn file-id feat)]
+      {:file-id file-id
+       :revn (:revn file)
+       :page (get-file-data-for-thumbnail conn file)})))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; MUTATION COMMANDS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; --- MUTATION COMMAND: upsert-file-object-thumbnail
+
+(def sql:upsert-object-thumbnail-1
+  "insert into file_object_thumbnail(file_id, object_id, data)
+   values (?, ?, ?)
+       on conflict(file_id, object_id) do
+          update set data = ?;")
+
+(def sql:upsert-object-thumbnail-2
+  "insert into file_object_thumbnail(file_id, object_id, media_id)
+   values (?, ?, ?)
+       on conflict(file_id, object_id) do
+          update set media_id = ?;")
+
+(defn upsert-file-object-thumbnail!
+  [{:keys [::db/conn ::sto/storage]} {:keys [file-id object-id] :as params}]
+
+  ;; NOTE: params can come with data set but with `nil` value, so we
+  ;; need first check the existence of the key and then the value.
+  (cond
+    (contains? params :data)
+    (if-let [data (:data params)]
+      (db/exec-one! conn [sql:upsert-object-thumbnail-1 file-id object-id data data])
+      (db/delete! conn :file-object-thumbnail {:file-id file-id :object-id object-id}))
+
+    (contains? params :media)
+    (if-let [{:keys [path mtype] :as media} (:media params)]
+      (let [_     (media/validate-media-type! media)
+            _     (media/validate-media-size! media)
+            hash  (sto/calculate-hash path)
+            data  (-> (sto/content path)
+                      (sto/wrap-with-hash hash))
+            media (sto/put-object! storage
+                                   {::sto/content data
+                                    ::sto/deduplicate? false
+                                    :content-type mtype
+                                    :bucket "file-object-thumbnail"})]
+
+        (db/exec-one! conn [sql:upsert-object-thumbnail-2 file-id object-id (:id media) (:id media)]))
+      (db/delete! conn :file-object-thumbnail {:file-id file-id :object-id object-id}))))
+
+;; FIXME: change it on validation refactor
+(s/def ::data (s/nilable ::us/string))
+(s/def ::media (s/nilable ::media/upload))
+(s/def ::object-id ::us/string)
+
+(s/def ::upsert-file-object-thumbnail
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::file-id ::object-id]
+          :opt-un [::data ::media]))
+
+(sv/defmethod ::upsert-file-object-thumbnail
+  {::doc/added "1.17"
+   ::audit/skip true}
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
+
+  (assert (or (contains? params :data)
+              (contains? params :media)))
+
+  (db/with-atomic [conn pool]
+    (files/check-edition-permissions! conn profile-id file-id)
+
+    (when-not (db/read-only? conn)
+      (let [cfg (-> cfg
+                    (update ::sto/storage media/configure-assets-storage)
+                    (assoc ::db/conn conn))]
+        (upsert-file-object-thumbnail! cfg params)
+        nil))))
+
+;; --- MUTATION COMMAND: upsert-file-thumbnail
+
+(def ^:private sql:upsert-file-thumbnail
+  "insert into file_thumbnail (file_id, revn, data, media_id, props)
+   values (?, ?, ?, ?, ?::jsonb)
+       on conflict(file_id, revn) do
+          update set data=?, media_id=?, props=?, updated_at=now();")
+
+(defn- upsert-file-thumbnail!
+  [{:keys [::db/conn ::sto/storage]} {:keys [file-id revn props] :as params}]
+  (let [props (db/tjson (or props {}))]
+    (cond
+      (contains? params :data)
+      (when-let [data (:data params)]
+        (db/exec-one! conn [sql:upsert-file-thumbnail
+                            file-id revn data nil props data nil props]))
+
+      (contains? params :media)
+      (when-let [{:keys [path mtype] :as media} (:media params)]
+        (let [_     (media/validate-media-type! media)
+              _     (media/validate-media-size! media)
+              hash  (sto/calculate-hash path)
+              data  (-> (sto/content path)
+                        (sto/wrap-with-hash hash))
+              media (sto/put-object! storage
+                                     {::sto/content data
+                                      ::sto/deduplicate? false
+                                      :content-type mtype
+                                      :bucket "file-thumbnail"})]
+          (db/exec-one! conn [sql:upsert-file-thumbnail
+                              file-id revn nil (:id media) props nil (:id media) props]))))))
+
+(s/def ::revn ::us/integer)
+(s/def ::props map?)
+(s/def ::media ::media/upload)
+
+(s/def ::upsert-file-thumbnail
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::file-id ::revn ::props]
+          :opt-un [::data ::media]))
+
+(sv/defmethod ::upsert-file-thumbnail
+  "Creates or updates the file thumbnail. Mainly used for paint the
+  grid thumbnails."
+  {::doc/added "1.17"
+   ::audit/skip true}
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
+  (db/with-atomic [conn pool]
+    (files/check-edition-permissions! conn profile-id file-id)
+    (when-not (db/read-only? conn)
+      (let [cfg (-> cfg
+                    (update ::sto/storage media/configure-assets-storage)
+                    (assoc ::db/conn conn))]
+        (upsert-file-thumbnail! cfg params))
+      nil)))

backend/src/app/rpc/commands/media.clj

@@ -41,15 +41,6 @@
 (s/def ::file-id ::us/uuid)
 (s/def ::team-id ::us/uuid)
 
-(defn validate-content-size!
-  [content]
-  (when (> (:size content) (cf/get :media-max-file-size default-max-file-size))
-    (ex/raise :type :restriction
-              :code :media-max-file-size-reached
-              :hint (str/ffmt "the uploaded file size % is greater than the maximum %"
-                              (:size content)
-                              default-max-file-size))))
-
 ;; --- Create File Media object (upload)
 
 (declare create-file-media-object)
@@ -68,7 +59,7 @@
   (let [cfg (update cfg ::sto/storage media/configure-assets-storage)]
     (files/check-edition-permissions! pool profile-id file-id)
     (media/validate-media-type! content)
-    (validate-content-size! content)
+    (media/validate-media-size! content)
     (let [object (create-file-media-object cfg params)
           props  {:name (:name params)
                   :file-id file-id

backend/src/app/tasks/file_gc.clj

@@ -18,7 +18,9 @@
    [app.common.types.shape-tree :as ctt]
    [app.config :as cf]
    [app.db :as db]
+   [app.media :as media]
    [app.rpc.commands.files :as files]
+   [app.storage :as sto]
    [app.util.blob :as blob]
    [app.util.pointer-map :as pmap]
    [app.util.time :as dt]
@@ -34,7 +36,7 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defmethod ig/pre-init-spec ::handler [_]
-  (s/keys :req [::db/pool]))
+  (s/keys :req [::db/pool ::sto/storage]))
 
 (defmethod ig/prep-key ::handler
   [_ cfg]
@@ -47,6 +49,7 @@
     (db/with-atomic [conn pool]
       (let [min-age (dt/duration (or (:min-age params) (::min-age cfg)))
             cfg     (-> cfg
+                        (update ::sto/storage media/configure-assets-storage conn)
                         (assoc ::db/conn conn)
                         (assoc ::file-id file-id)
                         (assoc ::min-age min-age))
@@ -141,36 +144,53 @@
       (db/delete! conn :file-media-object {:id (:id mobj)}))))
 
 (defn- clean-file-object-thumbnails!
-  [conn file-id data]
+  [{:keys [::db/conn ::sto/storage]} file-id data]
   (let [stored (->> (db/query conn :file-object-thumbnail
                               {:file-id file-id}
                               {:columns [:object-id]})
                     (into #{} (map :object-id)))
 
-        get-objects-ids
+        using  (into #{}
-        (fn [{:keys [id objects]}]
+                     (mapcat (fn [{:keys [id objects]}]
-          (->> (ctt/get-frames objects)
+                               (->> (ctt/get-frames objects)
-               (map #(str id (:id %)))))
+                                    (map #(str id (:id %))))))
-
+                     (vals (:pages-index data)))
-        using (into #{}
-                    (mapcat get-objects-ids)
-                    (vals (:pages-index data)))
 
         unused (set/difference stored using)]
 
     (when (seq unused)
       (let [sql (str "delete from file_object_thumbnail "
-                     " where file_id=? and object_id=ANY(?)")
+                     " where file_id=? and object_id=ANY(?)"
-            res (db/exec-one! conn [sql file-id (db/create-array conn "text" unused)])]
+                     " returning media_id")
-        (l/debug :hint "delete file object thumbnails" :file-id file-id :total (:next.jdbc/update-count res))))))
+            res (db/exec! conn [sql file-id (db/create-array conn "text" unused)])]
+
+        (doseq [media-id (into #{} (keep :media-id) res)]
+          ;; Mark as deleted the storage object related with the
+          ;; photo-id field.
+          (l/trace :hint "mark storage object as deleted" :id media-id)
+          (sto/del-object! storage media-id))
+
+        (l/debug :hint "delete file object thumbnails"
+                 :file-id file-id
+                 :total (count res))))))
 
 (defn- clean-file-thumbnails!
-  [conn file-id revn]
+  [{:keys [::db/conn ::sto/storage]} file-id revn]
   (let [sql (str "delete from file_thumbnail "
-                 " where file_id=? and revn < ?")
+                 " where file_id=? and revn < ? "
-        res (db/exec-one! conn [sql file-id revn])]
+                 " returning media_id")
-    (when-not (zero? (:next.jdbc/update-count res))
+        res (db/exec! conn [sql file-id revn])]
-      (l/debug :hint "delete file thumbnails" :file-id file-id :total (:next.jdbc/update-count res)))))
+
+    (when (seq res)
+      (doseq [media-id (into #{} (keep :media-id) res)]
+        ;; Mark as deleted the storage object related with the
+        ;; photo-id field.
+        (l/trace :hint "mark storage object as deleted" :id media-id)
+        (sto/del-object! storage media-id))
+
+      (l/debug :hint "delete file thumbnails"
+               :file-id file-id
+               :total (count res)))))
 
 (def ^:private
   sql:get-files-for-library
@@ -252,7 +272,7 @@
         (db/delete! conn :file-data-fragment {:id fragment-id :file-id file-id})))))
 
 (defn- process-file
-  [{:keys [::db/conn]} {:keys [id data revn modified-at features] :as file}]
+  [{:keys [::db/conn] :as cfg} {:keys [id data revn modified-at features] :as file}]
   (l/debug :hint "processing file" :id id :modified-at modified-at)
 
   (binding [pmap/*load-fn* (partial files/load-pointer conn id)]
@@ -261,8 +281,8 @@
                    (pmg/migrate-data))]
 
       (clean-file-media! conn id data)
-      (clean-file-object-thumbnails! conn id data)
+      (clean-file-object-thumbnails! cfg id data)
-      (clean-file-thumbnails! conn id revn)
+      (clean-file-thumbnails! cfg id revn)
       (clean-deleted-components! conn id data)
 
       (when (contains? features "storage/pointer-map")

backend/test/backend_tests/helpers.clj

@@ -335,6 +335,20 @@
                                   :session-id session-id
                                   :profile-id profile-id})))))
 
+(declare command!)
+
+(defn update-file! [& {:keys [profile-id file-id changes revn] :or {revn 0}}]
+  (let [params {::type :update-file
+                ::rpc/profile-id profile-id
+                :id file-id
+                :session-id (uuid/random)
+                :revn revn
+                :components-v2 true
+                :changes changes}
+        out    (command! params)]
+    (t/is (nil? (:error out)))
+    (:result out)))
+
 (defn create-webhook*
   ([params] (create-webhook* *pool* params))
   ([pool {:keys [team-id id uri mtype is-active]

backend/test/backend_tests/http_middleware_access_token_test.clj

@@ -0,0 +1,69 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns backend-tests.http-middleware-access-token-test
+  (:require
+   [app.db :as db]
+   [app.http.access-token]
+   [app.main :as-alias main]
+   [app.rpc :as-alias rpc]
+   [app.rpc.commands.access-token]
+   [app.tokens :as tokens]
+   [backend-tests.helpers :as th]
+   [clojure.test :as t]
+   [mockery.core :refer [with-mocks]]))
+
+(t/use-fixtures :once th/state-init)
+(t/use-fixtures :each th/database-reset)
+
+(t/deftest soft-auth-middleware
+  (db/with-atomic [conn (::db/pool th/*system*)]
+    (let [profile (th/create-profile* 1)
+          system  (-> th/*system*
+                      (assoc ::db/conn conn)
+                      (assoc ::main/props (:app.setup/props th/*system*)))
+
+          token   (app.rpc.commands.access-token/create-access-token
+                   system (:id profile) "test" nil)
+
+          request (volatile! nil)
+          handler (#'app.http.access-token/wrap-soft-auth
+                   (fn [req & _] (vreset! request req))
+                   system)]
+
+      (with-mocks [m1 {:target 'app.http.access-token/get-token
+                       :return nil}]
+        (handler {} nil nil)
+        (t/is (= {} @request)))
+
+      (with-mocks [m1 {:target 'app.http.access-token/get-token
+                       :return (:token token)}]
+        (handler {} nil nil)
+
+        (let [token-id (get @request :app.http.access-token/id)]
+          (t/is (= token-id (:id token))))))))
+
+(t/deftest authz-middleware
+  (let [profile (th/create-profile* 1)
+        system  (assoc th/*system* ::main/props (:app.setup/props th/*system*))
+
+        token   (db/with-atomic [conn (::db/pool th/*system*)]
+                  (let [system (assoc system ::db/conn conn)]
+                    (app.rpc.commands.access-token/create-access-token
+                     system (:id profile) "test" nil)))
+
+        request (volatile! {})
+        handler (#'app.http.access-token/wrap-authz
+                 (fn [req] (vreset! request req))
+                 system)]
+
+    (handler nil)
+    (t/is (nil? @request))
+
+    (handler {:app.http.access-token/id (:id token)})
+    (t/is (= #{} (:app.http.access-token/perms @request)))
+    (t/is (= (:id profile) (:app.http.access-token/profile-id @request)))))
+

backend/test/backend_tests/rpc_file_thumbnails_test.clj

@@ -0,0 +1,314 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns backend-tests.rpc-file-thumbnails-test
+  (:require
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.rpc :as-alias rpc]
+   [app.rpc.commands.auth :as cauth]
+   [app.storage :as sto]
+   [app.tokens :as tokens]
+   [app.util.time :as dt]
+   [backend-tests.helpers :as th]
+   [clojure.java.io :as io]
+   [clojure.test :as t]
+   [cuerdas.core :as str]
+   [datoteka.core :as fs]
+   [mockery.core :refer [with-mocks]]))
+
+(t/use-fixtures :once th/state-init)
+(t/use-fixtures :each th/database-reset)
+
+(t/deftest upsert-file-object-thumbnail
+  (let [storage (::sto/storage th/*system*)
+        profile (th/create-profile* 1)
+        file    (th/create-file* 1 {:profile-id (:id profile)
+                                    :project-id (:default-project-id profile)
+                                    :is-shared false})
+
+        shid    (uuid/random)
+        page-id (first (get-in file [:data :pages]))
+
+        ;; Update file inserting a new frame object
+        _       (th/update-file!
+                 :file-id (:id file)
+                 :profile-id (:id profile)
+                 :revn 0
+                 :changes
+                 [{:type :add-obj
+                   :page-id page-id
+                   :id shid
+                   :parent-id uuid/zero
+                   :frame-id uuid/zero
+                   :components-v2 true
+                   :obj {:id shid
+                         :name "Artboard"
+                         :frame-id uuid/zero
+                         :parent-id uuid/zero
+                         :type :frame}}])
+
+        data1   {::th/type :upsert-file-object-thumbnail
+                 ::rpc/profile-id (:id profile)
+                 :file-id (:id file)
+                 :object-id "test-key-1"
+                 :media {:filename "sample.jpg"
+                         :size 312043
+                         :path (th/tempfile "backend_tests/test_files/sample.jpg")
+                         :mtype "image/jpeg"}}
+
+        data2   {::th/type :upsert-file-object-thumbnail
+                 ::rpc/profile-id (:id profile)
+                 :file-id (:id file)
+                 :object-id (str page-id shid)
+                 :media {:filename "sample.jpg"
+                         :size 7923
+                         :path (th/tempfile "backend_tests/test_files/sample2.jpg")
+                         :mtype "image/jpeg"}}]
+
+    (let [out (th/command! data1)]
+      (t/is (nil? (:error out)))
+      (t/is (nil? (:result out))))
+
+    (let [out (th/command! data2)]
+      (t/is (nil? (:error out)))
+      (t/is (nil? (:result out))))
+
+    (let [[row1 row2 :as rows] (th/db-query :file-object-thumbnail
+                                            {:file-id (:id file)}
+                                            {:order-by [[:created-at :asc]]})]
+
+      (t/is (= 2 (count rows)))
+      (t/is (= (:file-id data1) (:file-id row1)))
+      (t/is (= (:object-id data1) (:object-id row1)))
+      (t/is (uuid? (:media-id row1)))
+      (t/is (= (:file-id data2) (:file-id row2)))
+      (t/is (= (:object-id data2) (:object-id row2)))
+      (t/is (uuid? (:media-id row2)))
+
+      (let [sobject (sto/get-object storage (:media-id row1))
+            mobject (meta sobject)]
+        (t/is (= "blake2b:4fdb63b8f3ffc81256ea79f13e53f366723b188554b5afed91b20897c14a1a8e" (:hash mobject)))
+        (t/is (= "file-object-thumbnail" (:bucket mobject)))
+        (t/is (= "image/jpeg" (:content-type mobject)))
+        (t/is (= 312043 (:size sobject))))
+
+      (let [sobject (sto/get-object storage (:media-id row2))
+            mobject (meta sobject)]
+        (t/is (= "blake2b:05870e3f8ee885841ee3799924d80805179ab57e6fde84a605d1068fd3138de9" (:hash mobject)))
+        (t/is (= "file-object-thumbnail" (:bucket mobject)))
+        (t/is (= "image/jpeg" (:content-type mobject)))
+        (t/is (= 7923 (:size sobject))))
+
+      ;; Run the File GC task that should remove unused file object
+      ;; thumbnails
+      (let [result (th/run-task! :file-gc {:min-age (dt/duration 0)})]
+        (t/is (= 1 (:processed result))))
+
+      ;; check if row2 related thumbnail row still exists
+      (let [[row :as rows] (th/db-query :file-object-thumbnail
+                                        {:file-id (:id file)}
+                                        {:order-by [[:created-at :asc]]})]
+        (t/is (= 1 (count rows)))
+        (t/is (= (:file-id data2) (:file-id row)))
+        (t/is (= (:object-id data2) (:object-id row)))
+        (t/is (uuid? (:media-id row2))))
+
+      ;; Check if storage objects still exists after file-gc
+      (t/is (nil? (sto/get-object storage (:media-id row1))))
+      (t/is (some? (sto/get-object storage (:media-id row2))))
+
+      ;; check that storage object is still exists but is marked as deleted
+      (let [row (th/db-get :storage-object {:id (:media-id row1)} {::db/remove-deleted? false})]
+        (t/is (some? (:deleted-at row))))
+
+      ;; Run the storage gc deleted task, it should permanently delete
+      ;; all storage objects related to the deleted thumbnails
+      (let [result (th/run-task! :storage-gc-deleted {:min-age (dt/duration 0)})]
+        (t/is (= 1 (:deleted result))))
+
+      ;; check that storage object is still exists but is marked as deleted
+      (let [row (th/db-get :storage-object {:id (:media-id row1)} {::db/remove-deleted? false})]
+        (t/is (nil? row)))
+
+      (t/is (some? (sto/get-object storage (:media-id row2))))
+
+
+      )))
+
+
+(t/deftest upsert-file-thumbnail
+  (let [storage (::sto/storage th/*system*)
+        profile (th/create-profile* 1)
+        file    (th/create-file* 1 {:profile-id (:id profile)
+                                    :project-id (:default-project-id profile)
+                                    :is-shared false
+                                    :revn 3})
+
+        data1   {::th/type :upsert-file-thumbnail
+                 ::rpc/profile-id (:id profile)
+                 :file-id (:id file)
+                 :props {}
+                 :revn 1
+                 :data "data:base64,1234123124"}
+
+        data2   {::th/type :upsert-file-thumbnail
+                 ::rpc/profile-id (:id profile)
+                 :file-id (:id file)
+                 :props {}
+                 :revn 2
+                 :media {:filename "sample.jpg"
+                         :size 7923
+                         :path (th/tempfile "backend_tests/test_files/sample2.jpg")
+                         :mtype "image/jpeg"}}
+
+        data3   {::th/type :upsert-file-thumbnail
+                 ::rpc/profile-id (:id profile)
+                 :file-id (:id file)
+                 :props {}
+                 :revn 3
+                 :media {:filename "sample.jpg"
+                         :size 312043
+                         :path (th/tempfile "backend_tests/test_files/sample.jpg")
+                         :mtype "image/jpeg"}}]
+
+    (let [out (th/command! data1)]
+      ;; (th/print-result! out)
+      (t/is (nil? (:error out)))
+      (t/is (nil? (:result out))))
+
+    (let [out (th/command! data2)]
+      (t/is (nil? (:error out)))
+      (t/is (nil? (:result out))))
+
+    (let [out (th/command! data3)]
+      (t/is (nil? (:error out)))
+      (t/is (nil? (:result out))))
+
+    (let [[row1 row2 row3 :as rows] (th/db-query :file-thumbnail
+                                                 {:file-id (:id file)}
+                                                 {:order-by [[:created-at :asc]]})]
+      (t/is (= 3 (count rows)))
+
+      (t/is (= (:file-id data1) (:file-id row1)))
+      (t/is (= (:revn data1) (:revn row1)))
+      (t/is (nil? (:media-id row1)))
+
+      (t/is (= (:file-id data2) (:file-id row2)))
+      (t/is (= (:revn data2) (:revn row2)))
+      (t/is (uuid? (:media-id row2)))
+
+      (t/is (= (:file-id data3) (:file-id row3)))
+      (t/is (= (:revn data3) (:revn row3)))
+      (t/is (uuid? (:media-id row3)))
+
+      (let [sobject (sto/get-object storage (:media-id row2))
+            mobject (meta sobject)]
+        (t/is (= "blake2b:05870e3f8ee885841ee3799924d80805179ab57e6fde84a605d1068fd3138de9" (:hash mobject)))
+        (t/is (= "file-thumbnail" (:bucket mobject)))
+        (t/is (= "image/jpeg" (:content-type mobject)))
+        (t/is (= 7923 (:size sobject))))
+
+      (let [sobject (sto/get-object storage (:media-id row3))
+            mobject (meta sobject)]
+        (t/is (= "blake2b:4fdb63b8f3ffc81256ea79f13e53f366723b188554b5afed91b20897c14a1a8e" (:hash mobject)))
+        (t/is (= "file-thumbnail" (:bucket mobject)))
+        (t/is (= "image/jpeg" (:content-type mobject)))
+        (t/is (= 312043 (:size sobject))))
+
+      ;; Run the File GC task that should remove unused file object
+      ;; thumbnails
+      (let [result (th/run-task! :file-gc {:min-age (dt/duration 0)})]
+        (t/is (= 1 (:processed result))))
+
+      ;; check if row2 related thumbnail row still exists
+      (let [[row :as rows] (th/db-query :file-thumbnail
+                                        {:file-id (:id file)}
+                                        {:order-by [[:created-at :asc]]})]
+        (t/is (= 1 (count rows)))
+        (t/is (= (:file-id data2) (:file-id row)))
+        (t/is (= (:object-id data2) (:object-id row)))
+        (t/is (uuid? (:media-id row2))))
+
+      ;; Check if storage objects still exists after file-gc
+      (t/is (nil? (sto/get-object storage (:media-id row1))))
+      (t/is (nil? (sto/get-object storage (:media-id row2))))
+      (t/is (some? (sto/get-object storage (:media-id row3))))
+
+      (let [row (th/db-get :storage-object {:id (:media-id row2)} {::db/remove-deleted? false})]
+        (t/is (some? (:deleted-at row))))
+
+      ;; Run the storage gc deleted task, it should permanently delete
+      ;; all storage objects related to the deleted thumbnails
+      (let [result (th/run-task! :storage-gc-deleted {:min-age (dt/duration 0)})]
+        (t/is (= 1 (:deleted result))))
+
+      ;; check that storage object is still exists but is marked as deleted
+      (let [row (th/db-get :storage-object {:id (:media-id row1)} {::db/remove-deleted? false})]
+        (t/is (nil? row)))
+
+      (t/is (some? (sto/get-object storage (:media-id row3))))
+
+
+      )))
+
+(t/deftest get-file-object-thumbnail
+  (let [storage (::sto/storage th/*system*)
+        profile (th/create-profile* 1)
+        file    (th/create-file* 1 {:profile-id (:id profile)
+                                    :project-id (:default-project-id profile)
+                                    :is-shared false})
+
+        data1   {::th/type :upsert-file-object-thumbnail
+                 ::rpc/profile-id (:id profile)
+                 :file-id (:id file)
+                 :object-id "test-key-1"
+                 :data "data:base64,1234123124"}
+
+        data2   {::th/type :upsert-file-object-thumbnail
+                 ::rpc/profile-id (:id profile)
+                 :file-id (:id file)
+                 :object-id "test-key-2"
+                 :media {:filename "sample.jpg"
+                         :size 7923
+                         :path (th/tempfile "backend_tests/test_files/sample2.jpg")
+                         :mtype "image/jpeg"}}]
+
+    (let [out (th/command! data1)]
+      (t/is (nil? (:error out)))
+      (t/is (nil? (:result out))))
+
+    (let [out (th/command! data2)]
+      (t/is (nil? (:error out)))
+      (t/is (nil? (:result out))))
+
+    (let [[row1 row2 :as rows] (th/db-query :file-object-thumbnail
+                                            {:file-id (:id file)}
+                                            {:order-by [[:created-at :asc]]})]
+      (t/is (= 2 (count rows)))
+
+      (t/is (= (:file-id data1) (:file-id row1)))
+      (t/is (= (:object-id data1) (:object-id row1)))
+      (t/is (nil? (:media-id row1)))
+      (t/is (string? (:data row1)))
+
+      (t/is (= (:file-id data2) (:file-id row2)))
+      (t/is (= (:object-id data2) (:object-id row2)))
+      (t/is (uuid? (:media-id row2))))
+
+
+    (let [params {::th/type :get-file-object-thumbnails
+                  ::rpc/profile-id (:id profile)
+                  :file-id (:id file)}
+          out    (th/command! params)]
+
+      (let [result (:result out)]
+        (t/is (contains? result "test-key-1"))
+        (t/is (contains? result "test-key-2"))))))
+
+
+

frontend/src/app/main/data/workspace.cljs

@@ -449,10 +449,12 @@
             page    (get-in state [:workspace-data :pages-index page-id])
             name    (cp/generate-unique-name unames (:name page))
 
-            no_thumbnails_objects (->> (:objects page)
+            page    (-> page
-                                       (d/mapm (fn [_ val] (dissoc val :use-for-thumbnail?))))
+                        (assoc :name name)
-
+                        (assoc :id id)
-            page (-> page (assoc :name name :id id :objects no_thumbnails_objects))
+                        (assoc :objects
+                               (->> (:objects page)
+                                    (d/mapm (fn [_ val] (dissoc val :use-for-thumbnail?))))))
 
             changes (-> (pcb/empty-changes it)
                         (pcb/add-page id page))]

frontend/src/app/main/data/workspace/shapes.cljs

@@ -481,25 +481,33 @@
       (let [selected (wsh/lookup-selected state)]
         (rx/of (dch/update-shapes selected #(update % :blocked not)))))))
 
+
+;; FIXME: this need to be refactored
+
 (defn toggle-file-thumbnail-selected
   []
   (ptk/reify ::toggle-file-thumbnail-selected
     ptk/WatchEvent
     (watch [_ state _]
       (let [selected   (wsh/lookup-selected state)
-            pages      (-> state :workspace-data :pages-index vals)
+            pages      (-> state :workspace-data :pages-index vals)]
-            get-frames (fn [{:keys [objects id] :as page}]
-                         (->> (ctst/get-frames objects)
-                              (sequence
-                               (comp (filter :use-for-thumbnail?)
-                                     (map :id)
-                                     (remove selected)
-                                     (map (partial vector id))))))]
 
         (rx/concat
+         ;; First: clear the `:use-for-thumbnail?` flag from all not
+         ;; selected frames.
          (rx/from
-          (->> (mapcat get-frames pages)
+          (->> pages
+               (mapcat
+                (fn [{:keys [objects id] :as page}]
+                  (->> (ctst/get-frames objects)
+                       (sequence
+                        (comp (filter :use-for-thumbnail?)
+                              (map :id)
+                              (remove selected)
+                              (map (partial vector id)))))))
                (d/group-by first second)
                (map (fn [[page-id frame-ids]]
                       (dch/update-shapes frame-ids #(dissoc % :use-for-thumbnail?) {:page-id page-id})))))
+
+         ;; And finally: toggle the flag value on all the selected shapes
          (rx/of (dch/update-shapes selected #(update % :use-for-thumbnail? not))))))))

frontend/src/app/main/repo.cljs

@@ -150,6 +150,10 @@
   [id params]
   (send-command! id params {:forward-query-params [:file-id :object-id]}))
 
+(defmethod command :get-file-object-thumbnails
+  [id params]
+  (send-command! id params {:forward-query-params [:file-id]}))
+
 (defmethod command :export-binfile
   [id params]
   (send-command! id params {:response-type :blob}))