mirror of
https://github.com/penpot/penpot.git
synced 2025-05-11 15:06:40 +02:00
🎉 Add support for multipart upload of thumbnails
and improve the thumbnails storage to offloading it to the storage subsystem
This commit is contained in:
Andrey Antukh
parent
c2b6b40554
commit
e8ffcbae69
14 changed files with 771 additions and 289 deletions
|
|
@ -9,16 +9,13 @@
|
|||
[app.common.data :as d]
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.geom.shapes :as gsh]
|
||||
[app.common.pages.helpers :as cph]
|
||||
[app.common.pages.migrations :as pmg]
|
||||
[app.common.spec :as us]
|
||||
[app.common.types.components-list :as ctkl]
|
||||
[app.common.types.file :as ctf]
|
||||
[app.common.types.shape-tree :as ctt]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.db.sql :as sql]
|
||||
[app.loggers.audit :as-alias audit]
|
||||
[app.loggers.webhooks :as-alias webhooks]
|
||||
[app.rpc :as-alias rpc]
|
||||
|
|
@ -332,41 +329,6 @@
|
|||
(-> (get-file-fragment conn file-id fragment-id)
|
||||
(rph/with-http-cache long-cache-duration)))))
|
||||
|
||||
;; --- COMMAND QUERY: get-file-object-thumbnails
|
||||
|
||||
(defn get-object-thumbnails
|
||||
([conn file-id]
|
||||
(let [sql (str/concat
|
||||
"select object_id, data "
|
||||
" from file_object_thumbnail"
|
||||
" where file_id=?")]
|
||||
(->> (db/exec! conn [sql file-id])
|
||||
(d/index-by :object-id :data))))
|
||||
|
||||
([conn file-id object-ids]
|
||||
(let [sql (str/concat
|
||||
"select object_id, data "
|
||||
" from file_object_thumbnail"
|
||||
" where file_id=? and object_id = ANY(?)")
|
||||
ids (db/create-array conn "text" (seq object-ids))]
|
||||
(->> (db/exec! conn [sql file-id ids])
|
||||
(d/index-by :object-id :data)))))
|
||||
|
||||
(s/def ::get-file-object-thumbnails
|
||||
(s/keys :req [::rpc/profile-id] :req-un [::file-id]))
|
||||
|
||||
(sv/defmethod ::get-file-object-thumbnails
|
||||
"Retrieve a file object thumbnails."
|
||||
{::doc/added "1.17"
|
||||
::cond/get-object #(get-minimal-file %1 (:file-id %2))
|
||||
::cond/reuse-key? true
|
||||
::cond/key-fn get-file-etag}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(check-read-permissions! conn profile-id file-id)
|
||||
(get-object-thumbnails conn file-id)))
|
||||
|
||||
|
||||
;; --- COMMAND QUERY: get-project-files
|
||||
|
||||
(def ^:private sql:project-files
|
||||
|
|
@ -662,161 +624,6 @@
|
|||
(teams/check-read-permissions! conn profile-id team-id)
|
||||
(get-team-recent-files conn team-id)))
|
||||
|
||||
|
||||
;; --- COMMAND QUERY: get-file-thumbnail
|
||||
|
||||
(defn get-file-thumbnail
|
||||
[conn file-id revn]
|
||||
(let [sql (sql/select :file-thumbnail
|
||||
(cond-> {:file-id file-id}
|
||||
revn (assoc :revn revn))
|
||||
{:limit 1
|
||||
:order-by [[:revn :desc]]})
|
||||
row (db/exec-one! conn sql)]
|
||||
(when-not row
|
||||
(ex/raise :type :not-found
|
||||
:code :file-thumbnail-not-found))
|
||||
|
||||
{:data (:data row)
|
||||
:props (some-> (:props row) db/decode-transit-pgobject)
|
||||
:revn (:revn row)
|
||||
:file-id (:file-id row)}))
|
||||
|
||||
(s/def ::revn ::us/integer)
|
||||
|
||||
(s/def ::get-file-thumbnail
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::file-id]
|
||||
:opt-un [::revn]))
|
||||
|
||||
(sv/defmethod ::get-file-thumbnail
|
||||
{::doc/added "1.17"}
|
||||
[{:keys [::db/pool]} {:keys [::rpc/profile-id file-id revn]}]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(check-read-permissions! conn profile-id file-id)
|
||||
(-> (get-file-thumbnail conn file-id revn)
|
||||
(rph/with-http-cache long-cache-duration))))
|
||||
|
||||
|
||||
;; --- COMMAND QUERY: get-file-data-for-thumbnail
|
||||
|
||||
;; FIXME: performance issue
|
||||
;;
|
||||
;; We need to improve how we set frame for thumbnail in order to avoid
|
||||
;; loading all pages into memory for find the frame set for thumbnail.
|
||||
|
||||
(defn get-file-data-for-thumbnail
|
||||
[conn {:keys [data id] :as file}]
|
||||
(letfn [;; function responsible on finding the frame marked to be
|
||||
;; used as thumbnail; the returned frame always have
|
||||
;; the :page-id set to the page that it belongs.
|
||||
|
||||
(get-thumbnail-frame [data]
|
||||
;; NOTE: this is a hack for avoid perform blocking
|
||||
;; operation inside the for loop, clojure lazy-seq uses
|
||||
;; synchronized blocks that does not plays well with
|
||||
;; virtual threads, so we need to perform the load
|
||||
;; operation first. This operation forces all pointer maps
|
||||
;; load into the memory.
|
||||
(->> (-> data :pages-index vals)
|
||||
(filter pmap/pointer-map?)
|
||||
(run! pmap/load!))
|
||||
|
||||
;; Then proceed to find the frame set for thumbnail
|
||||
|
||||
(d/seek :use-for-thumbnail?
|
||||
(for [page (-> data :pages-index vals)
|
||||
frame (-> page :objects ctt/get-frames)]
|
||||
(assoc frame :page-id (:id page)))))
|
||||
|
||||
;; function responsible to filter objects data structure of
|
||||
;; all unneeded shapes if a concrete frame is provided. If no
|
||||
;; frame, the objects is returned untouched.
|
||||
(filter-objects [objects frame-id]
|
||||
(d/index-by :id (cph/get-children-with-self objects frame-id)))
|
||||
|
||||
;; function responsible of assoc available thumbnails
|
||||
;; to frames and remove all children shapes from objects if
|
||||
;; thumbnails is available
|
||||
(assoc-thumbnails [objects page-id thumbnails]
|
||||
(loop [objects objects
|
||||
frames (filter cph/frame-shape? (vals objects))]
|
||||
|
||||
(if-let [frame (-> frames first)]
|
||||
(let [frame-id (:id frame)
|
||||
object-id (str page-id frame-id)
|
||||
frame (if-let [thumb (get thumbnails object-id)]
|
||||
(assoc frame :thumbnail thumb :shapes [])
|
||||
(dissoc frame :thumbnail))
|
||||
|
||||
children-ids
|
||||
(cph/get-children-ids objects frame-id)
|
||||
|
||||
bounds
|
||||
(when (:show-content frame)
|
||||
(gsh/selection-rect (concat [frame] (->> children-ids (map (d/getf objects))))))
|
||||
|
||||
frame
|
||||
(cond-> frame
|
||||
(some? bounds)
|
||||
(assoc :children-bounds bounds))]
|
||||
|
||||
(if (:thumbnail frame)
|
||||
(recur (-> objects
|
||||
(assoc frame-id frame)
|
||||
(d/without-keys children-ids))
|
||||
(rest frames))
|
||||
(recur (assoc objects frame-id frame)
|
||||
(rest frames))))
|
||||
|
||||
objects)))]
|
||||
|
||||
(binding [pmap/*load-fn* (partial load-pointer conn id)]
|
||||
(let [frame (get-thumbnail-frame data)
|
||||
frame-id (:id frame)
|
||||
page-id (or (:page-id frame)
|
||||
(-> data :pages first))
|
||||
|
||||
page (dm/get-in data [:pages-index page-id])
|
||||
page (cond-> page (pmap/pointer-map? page) deref)
|
||||
frame-ids (if (some? frame) (list frame-id) (map :id (ctt/get-frames (:objects page))))
|
||||
|
||||
obj-ids (map #(str page-id %) frame-ids)
|
||||
thumbs (get-object-thumbnails conn id obj-ids)]
|
||||
|
||||
(cond-> page
|
||||
;; If we have frame, we need to specify it on the page level
|
||||
;; and remove the all other unrelated objects.
|
||||
(some? frame-id)
|
||||
(-> (assoc :thumbnail-frame-id frame-id)
|
||||
(update :objects filter-objects frame-id))
|
||||
|
||||
;; Assoc the available thumbnails and prune not visible shapes
|
||||
;; for avoid transfer unnecessary data.
|
||||
:always
|
||||
(update :objects assoc-thumbnails page-id thumbs))))))
|
||||
|
||||
(s/def ::get-file-data-for-thumbnail
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::file-id]
|
||||
:opt-un [::features]))
|
||||
|
||||
(sv/defmethod ::get-file-data-for-thumbnail
|
||||
"Retrieves the data for generate the thumbnail of the file. Used
|
||||
mainly for render thumbnails on dashboard."
|
||||
{::doc/added "1.17"}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id features] :as props}]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(check-read-permissions! conn profile-id file-id)
|
||||
;; NOTE: we force here the "storage/pointer-map" feature, because
|
||||
;; it used internally only and is independent if user supports it
|
||||
;; or not.
|
||||
(let [feat (into #{"storage/pointer-map"} features)
|
||||
file (get-file conn file-id feat)]
|
||||
{:file-id file-id
|
||||
:revn (:revn file)
|
||||
:page (get-file-data-for-thumbnail conn file)})))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; MUTATION COMMANDS
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
|
@ -1026,66 +833,3 @@
|
|||
(check-edition-permissions! conn profile-id file-id)
|
||||
(-> (ignore-sync conn params)
|
||||
(update :features db/decode-pgarray #{}))))
|
||||
|
||||
|
||||
;; --- MUTATION COMMAND: upsert-file-object-thumbnail
|
||||
|
||||
(def sql:upsert-object-thumbnail
|
||||
"insert into file_object_thumbnail(file_id, object_id, data)
|
||||
values (?, ?, ?)
|
||||
on conflict(file_id, object_id) do
|
||||
update set data = ?;")
|
||||
|
||||
(defn upsert-file-object-thumbnail!
|
||||
[conn {:keys [file-id object-id data]}]
|
||||
(if data
|
||||
(db/exec-one! conn [sql:upsert-object-thumbnail file-id object-id data data])
|
||||
(db/delete! conn :file-object-thumbnail {:file-id file-id :object-id object-id})))
|
||||
|
||||
(s/def ::data (s/nilable ::us/string))
|
||||
(s/def ::thumbs/object-id ::us/string)
|
||||
(s/def ::upsert-file-object-thumbnail
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::file-id ::thumbs/object-id]
|
||||
:opt-un [::data]))
|
||||
|
||||
(sv/defmethod ::upsert-file-object-thumbnail
|
||||
{::doc/added "1.17"
|
||||
::audit/skip true}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(check-edition-permissions! conn profile-id file-id)
|
||||
(upsert-file-object-thumbnail! conn params)
|
||||
nil))
|
||||
|
||||
;; --- MUTATION COMMAND: upsert-file-thumbnail
|
||||
|
||||
(def ^:private sql:upsert-file-thumbnail
|
||||
"insert into file_thumbnail (file_id, revn, data, props)
|
||||
values (?, ?, ?, ?::jsonb)
|
||||
on conflict(file_id, revn) do
|
||||
update set data = ?, props=?, updated_at=now();")
|
||||
|
||||
(defn- upsert-file-thumbnail!
|
||||
[conn {:keys [file-id revn data props]}]
|
||||
(let [props (db/tjson (or props {}))]
|
||||
(db/exec-one! conn [sql:upsert-file-thumbnail
|
||||
file-id revn data props data props])))
|
||||
|
||||
(s/def ::revn ::us/integer)
|
||||
(s/def ::props map?)
|
||||
(s/def ::upsert-file-thumbnail
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::file-id ::revn ::data ::props]))
|
||||
|
||||
(sv/defmethod ::upsert-file-thumbnail
|
||||
"Creates or updates the file thumbnail. Mainly used for paint the
|
||||
grid thumbnails."
|
||||
{::doc/added "1.17"
|
||||
::audit/skip true}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(check-edition-permissions! conn profile-id file-id)
|
||||
(when-not (db/read-only? conn)
|
||||
(upsert-file-thumbnail! conn params))
|
||||
nil))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue