3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-05-10 12:26:37 +02:00
Code Issues Projects Releases Packages Wiki Activity

🎉 Add generic file object thumbnail abstraction

Browse source 
As replacement to the file frame thumbnail mechanism
This commit is contained in:
Andrey Antukh 2022-03-30 00:11:43 +02:00 • committed by Andrés Moya
parent 147f56749e
commit 20d3251a93
15 changed files with 399 additions and 212 deletions
Download patch file Download diff file Expand all files Collapse all files

6
backend/src/app/migrations.clj
View file

@ -217,6 +217,12 @@
{:name "0069-add-file-thumbnail-table"
:fn (mg/resource "app/migrations/sql/0069-add-file-thumbnail-table.sql")}
{:name "0070-del-frame-thumbnail-table"
:fn (mg/resource "app/migrations/sql/0070-del-frame-thumbnail-table.sql")}
{:name "0071-add-file-object-thumbnail-table"
:fn (mg/resource "app/migrations/sql/0071-add-file-object-thumbnail-table.sql")}
])

1
backend/src/app/migrations/sql/0070-del-frame-thumbnail-table.sql Normal file
View file

@ -0,0 +1 @@
DROP TABLE file_frame_thumbnail;

11
backend/src/app/migrations/sql/0071-add-file-object-thumbnail-table.sql Normal file
View file

@ -0,0 +1,11 @@
CREATE TABLE file_object_thumbnail (
file_id uuid NOT NULL REFERENCES file(id) ON DELETE CASCADE,
object_id uuid NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
data text NULL,
PRIMARY KEY(file_id, object_id)
);
ALTER TABLE file_object_thumbnail
ALTER COLUMN data SET STORAGE external;

27
backend/src/app/rpc/mutations/files.clj
View file

@ -476,30 +476,31 @@
:revn revn
:data (blob/encode data)}
{:id id})))
nil)))
;; --- Mutation: upsert object thumbnail
;; --- Mutation: Upsert frame thumbnail
(def sql:upsert-frame-thumbnail
"insert into file_frame_thumbnail(file_id, frame_id, data)
(def sql:upsert-object-thumbnail
"insert into file_object_thumbnail(file_id, object_id, data)
values (?, ?, ?)
on conflict(file_id, frame_id) do
on conflict(file_id, object_id) do
update set data = ?;")
(s/def ::data ::us/string)
(s/def ::upsert-file-frame-thumbnail
(s/keys :req-un [::profile-id ::file-id ::frame-id ::data]))
(s/def ::data (s/nilable ::us/string))
(s/def ::object-id ::us/uuid)
(s/def ::upsert-file-object-thumbnail
(s/keys :req-un [::profile-id ::file-id ::object-id ::data]))
(sv/defmethod ::upsert-file-frame-thumbnail
[{:keys [pool] :as cfg} {:keys [profile-id file-id frame-id data]}]
(sv/defmethod ::upsert-file-object-thumbnail
[{:keys [pool] :as cfg} {:keys [profile-id file-id object-id data]}]
(db/with-atomic [conn pool]
(files/check-edition-permissions! conn profile-id file-id)
(db/exec-one! conn [sql:upsert-frame-thumbnail file-id frame-id data data])
(if data
(db/exec-one! conn [sql:upsert-object-thumbnail file-id object-id data data])
(db/delete! conn :file-object-thumbnail {:file-id file-id :object-id object-id}))
nil))
;; --- Mutation: Upsert file thumbnail
;; --- Mutation: upsert file thumbnail
(def sql:upsert-file-thumbnail
"insert into file_thumbnail (file_id, revn, data, props)

203
backend/src/app/rpc/queries/files.clj
View file

@ -7,11 +7,11 @@
(ns app.rpc.queries.files
(:require
[app.common.data :as d]
[app.common.data.macros :as dm]
[app.common.exceptions :as ex]
[app.common.pages.helpers :as cph]
[app.common.pages.migrations :as pmg]
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.db :as db]
[app.db.sql :as sql]
[app.rpc.helpers :as rpch]
@ -21,7 +21,8 @@
[app.rpc.queries.teams :as teams]
[app.util.blob :as blob]
[app.util.services :as sv]
[clojure.spec.alpha :as s]))
[clojure.spec.alpha :as s]
[cuerdas.core :as str]))
(declare decode-row)
(declare decode-row-xf)
@ -187,12 +188,30 @@
;; --- Query: File (By ID)
(defn retrieve-object-thumbnails
([{:keys [pool]} file-id]
(let [sql (str/concat
"select object_id, data "
" from file_object_thumbnail"
" where file_id=?")]
(->> (db/exec! pool [sql file-id])
(d/index-by :object-id :data))))
([{:keys [pool]} file-id frame-ids]
(with-open [conn (db/open pool)]
(let [sql (str/concat
"select object_id, data "
" from file_object_thumbnail"
" where file_id=? and object_id = ANY(?)")
ids (db/create-array conn "uuid" (seq frame-ids))]
(->> (db/exec! conn [sql file-id ids])
(d/index-by :object-id :data))))))
(defn retrieve-file
[{:keys [pool] :as cfg} id]
(let [item (db/get-by-id pool :file id)]
(->> item
(decode-row)
(pmg/migrate-file))))
(->> (db/get-by-id pool :file id)
(decode-row)
(pmg/migrate-file)))
(s/def ::file
(s/keys :req-un [::profile-id ::id]))
@ -202,12 +221,16 @@
[{:keys [pool] :as cfg} {:keys [profile-id id] :as params}]
(let [perms (get-permissions pool profile-id id)]
(check-read-permissions! perms)
(-> (retrieve-file cfg id)
(assoc :permissions perms))))
(let [file (retrieve-file cfg id)
thumbs (retrieve-object-thumbnails cfg id)]
(-> file
(assoc :thumbnails thumbs)
(assoc :permissions perms)))))
;; --- FILE THUMBNAIL
(defn- trim-objects
;; --- QUERY: page
(defn- prune-objects
"Given the page data and the object-id returns the page data with all
other not needed objects removed from the `:objects` data
structure."
@ -219,64 +242,19 @@
"Given the page data, removes the `:thumbnail` prop from all
shapes."
[page]
(update page :objects (fn [objects]
(d/mapm #(dissoc %2 :thumbnail) objects))))
(defn- prune-frames-with-thumbnails
"Remove unnecesary shapes from frames that have thumbnail from page
data."
[page]
(let [filter-shape?
(fn [objects [id shape]]
(let [frame-id (:frame-id shape)]
(or (= id uuid/zero)
(= frame-id uuid/zero)
(not (some? (get-in objects [frame-id :thumbnail]))))))
;; We need to remove from the attribute :shapes its children because
;; they will not be sent in the data
remove-frame-children
(fn [[id shape]]
[id (cond-> shape
(some? (:thumbnail shape))
(assoc :shapes []))])
update-objects
(fn [objects]
(into {}
(comp (map remove-frame-children)
(filter (partial filter-shape? objects)))
objects))]
(update page :objects update-objects)))
(defn- get-thumbnail-data
[{:keys [data] :as file}]
(if-let [[page frame] (first
(for [page (-> data :pages-index vals)
frame (-> page :objects cph/get-frames)
:when (:file-thumbnail frame)]
[page frame]))]
(let [objects (->> (cph/get-children-with-self (:objects page) (:id frame))
(d/index-by :id))]
(-> (assoc page :objects objects)
(assoc :thumbnail-frame frame)))
(let [page-id (-> data :pages first)]
(-> (get-in data [:pages-index page-id])
(prune-frames-with-thumbnails)))))
(update page :objects d/update-vals #(dissoc % :thumbnail)))
(s/def ::page-id ::us/uuid)
(s/def ::object-id ::us/uuid)
(s/def ::prune-frames-with-thumbnails ::us/boolean)
(s/def ::prune-thumbnails ::us/boolean)
(s/def ::page
(s/keys :req-un [::profile-id ::file-id]
:opt-un [::page-id
::object-id
::prune-frames-with-thumbnails
::prune-thumbnails]))
(s/and
(s/keys :req-un [::profile-id ::file-id]
:opt-un [::page-id ::object-id])
(fn [obj]
(if (contains? obj :object-id)
(contains? obj :page-id)
true))))
(sv/defmethod ::page
"Retrieves the page data from file and returns it. If no page-id is
@ -284,6 +262,9 @@
specified, only that object and its children will be returned in the
page objects data structure.
If you specify the object-id, the page-id parameter becomes
mandatory.
Mainly used for rendering purposes."
[{:keys [pool] :as cfg} {:keys [profile-id file-id page-id object-id] :as props}]
(check-read-permissions! pool profile-id file-id)
@ -291,28 +272,84 @@
page-id (or page-id (-> file :data :pages first))
page (get-in file [:data :pages-index page-id])]
(cond-> page
(:prune-frames-with-thumbnails props)
(prune-frames-with-thumbnails)
(:prune-thumbnails props)
(prune-thumbnails)
(cond-> (prune-thumbnails page)
(uuid? object-id)
(trim-objects object-id))))
(prune-objects object-id))))
;; --- QUERY: file-data-for-thumbnail
(defn- get-file-thumbnail-data
[cfg {:keys [data id] :as file}]
(letfn [;; function responsible on finding the frame marked to be
;; used as thumbnail; the returned frame always have
;; the :page-id set to the page that it belongs.
(get-thumbnail-frame [data]
(d/seek :use-for-thumbnail?
(for [page (-> data :pages-index vals)
frame (-> page :objects cph/get-frames)]
(assoc frame :page-id (:id page)))))
;; function responsible to filter objects data strucuture of
;; all unneded shapes if a concrete frame is provided. If no
;; frame, the objects is returned untouched.
(filter-objects [objects frame-id]
(d/index-by :id (cph/get-children-with-self objects frame-id)))
;; function responsible of assoc available thumbnails
;; to frames and remove all children shapes from objects if
;; thumbnails is available
(assoc-thumbnails [objects thumbnails]
(loop [objects objects
frames (filter cph/frame-shape? (vals objects))]
(if-let [{:keys [id] :as frame} (first frames)]
(let [frame (if-let [thumb (get thumbnails id)]
(assoc frame :thumbnail thumb :shapes [])
(dissoc frame :thumbnail))]
(if (:thumbnail frame)
(recur (-> (assoc objects id frame)
(d/without-keys (cph/get-children-ids objects id)))
(rest frames))
(recur (assoc objects id frame)
(rest frames))))
objects)))]
(let [frame (get-thumbnail-frame data)
frame-id (:id frame)
page-id (or (:page-id frame)
(-> data :pages first))
page (dm/get-in data [:pages-index page-id])
obj-ids (or (some-> frame-id list)
(map :id (cph/get-frames page)))
thumbs (retrieve-object-thumbnails cfg id obj-ids)]
(cond-> page
;; If we have frame, we need to specify it on the page level
;; and remove the all other unrelated objects.
(some? frame-id)
(-> (assoc :thumbnail-frame-id frame-id)
(update :objects filter-objects frame-id))
;; Assoc the available thumbnails and prune not visible shapes
;; for avoid transfer unnecesary data.
:always
(update :objects assoc-thumbnails thumbs)))))
(s/def ::file-data-for-thumbnail
(s/keys :req-un [::profile-id ::file-id]))
(sv/defmethod ::file-data-for-thumbnail
"Retrieves the data for generate the thumbnail of the file. Used mainly for render
thumbnails on dashboard. Returns the page data."
"Retrieves the data for generate the thumbnail of the file. Used
mainly for render thumbnails on dashboard."
[{:keys [pool] :as cfg} {:keys [profile-id file-id] :as props}]
(check-read-permissions! pool profile-id file-id)
(let [file (retrieve-file cfg file-id)]
{:page (get-thumbnail-data file)
:file-id file-id
:revn (:revn file)}))
{:file-id file-id
:revn (:revn file)
:page (get-file-thumbnail-data cfg file)}))
;; --- Query: Shared Library Files
@ -412,20 +449,6 @@
(teams/check-read-permissions! pool profile-id team-id)
(db/exec! pool [sql:team-recent-files team-id]))
;; --- QUERY: get all file frame thumbnails
(s/def ::file-frame-thumbnails
(s/keys :req-un [::profile-id ::file-id]
:opt-un [::frame-id]))
(sv/defmethod ::file-frame-thumbnails
[{:keys [pool]} {:keys [profile-id file-id frame-id]}]
(check-read-permissions! pool profile-id file-id)
(let [params (cond-> {:file-id file-id}
frame-id (assoc :frame-id frame-id))
rows (db/query pool :file-frame-thumbnail params)]
(d/index-by :frame-id :data rows)))
;; --- QUERY: get file thumbnail
(s/def ::revn ::us/integer)

41
backend/src/app/tasks/file_gc.clj
View file

@ -6,18 +6,19 @@
(ns app.tasks.file-gc
"A maintenance task that is responsible of: purge unused file media,
clean unused frame thumbnails and remove old file thumbnails. The
clean unused object thumbnails and remove old file thumbnails. The
file is eligible to be garbage collected after some period of
inactivity (the default threshold is 72h)."
(:require
[app.common.data :as d]
[app.common.logging :as l]
[app.common.pages.helpers :as cph]
[app.common.pages.migrations :as pmg]
[app.db :as db]
[app.util.blob :as blob]
[app.util.time :as dt]
[clojure.set :as set]
[clojure.spec.alpha :as s]
[cuerdas.core :as str]
[integrant.core :as ig]))
(declare ^:private retrieve-candidates)
@ -117,26 +118,26 @@
;; them.
(db/delete! conn :file-media-object {:id (:id mobj)}))))
(defn- collect-frames
[data]
(let [xform (comp
(map :objects)
(mapcat vals)
(filter cph/frame-shape?)
(keep :id))
pages (concat
(vals (:pages-index data))
(vals (:components data)))]
(into #{} xform pages)))
(defn- clean-file-frame-thumbnails!
[conn file-id data]
(let [sql (str "delete from file_frame_thumbnail "
" where file_id=? and not (frame_id=ANY(?))")
ids (->> (collect-frames data)
(db/create-array conn "uuid"))
res (db/exec-one! conn [sql file-id ids])]
(l/debug :hint "delete frame thumbnails" :total (:next.jdbc/update-count res))))
(let [stored (->> (db/query conn :file-object-thumbnail
{:file-id file-id}
{:columns [:object-id]})
(into #{} (map :object-id)))
using (->> (concat (vals (:pages-index data))
(vals (:components data)))
(into #{} (comp (map :objects)
(mapcat keys))))
unused (set/difference stored using)]
(when (seq unused)
(let [sql (str/concat
"delete from file_object_thumbnail "
" where file_id=? and object_id=ANY(?)")
res (db/exec-one! conn [sql file-id (db/create-array conn "uuid" unused)])]
(l/debug :hint "delete object thumbnails" :total (:next.jdbc/update-count res))))))
(defn- clean-file-thumbnails!
[conn file-id revn]