3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-06-14 04:11:38 +02:00
Code Issues Projects Releases Packages Wiki Activity

♻️ Make the components cleaning mechanism on file-gc task more efficient

Browse source
This commit is contained in:
Andrey Antukh 2025-01-20 16:35:14 +01:00
parent 5513daf17d
commit 151aedcf91
2 changed files with 357 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

67
backend/src/app/tasks/file_gc.clj
View file

@ -26,7 +26,6 @@
[app.util.pointer-map :as pmap]
[app.util.time :as dt]
[app.worker :as wrk]
[clojure.set :as set]
[integrant.core :as ig]))
(declare ^:private get-file)
@ -156,51 +155,47 @@
AND f.deleted_at IS null
ORDER BY f.modified_at ASC")
(def ^:private xf:map-id (map :id))
(defn- get-used-components
"Given a file and a set of components marked for deletion, return a
filtered set of component ids that are still un use"
[components library-id {:keys [data]}]
(filter #(ctf/used-in? data library-id % :component) components))
(defn- clean-deleted-components!
"Performs the garbage collection of unreferenced deleted components."
[{:keys [::db/conn] :as cfg} {:keys [data] :as file}]
(let [file-id (:id file)
get-used-components
(fn [data components]
;; Find which of the components are used in the file.
(into #{}
(filter #(ctf/used-in? data file-id % :component))
components))
deleted-components
(ctkl/deleted-components-seq data)
get-unused-components
(fn [components files]
;; Find and return a set of unused components (on all files).
(reduce (fn [components {:keys [data]}]
(if (seq components)
(->> (get-used-components data components)
(set/difference components))
(reduced components)))
xform
(mapcat (partial get-used-components deleted-components file-id))
components
files))
used-remote
(->> (db/plan conn [sql:get-files-for-library file-id] plan-opts)
(transduce (comp (map (partial decode-file cfg)) xform) conj #{}))
process-fdata
(fn [data unused]
(reduce (fn [data id]
(l/trc :hint "delete component"
:component-id (str id)
:file-id (str file-id))
(ctkl/delete-component data id))
data
unused))
used-local
(into #{} xform [file])
deleted (into #{} (ctkl/deleted-components-seq data))
unused (->> (db/cursor conn [sql:get-files-for-library file-id] {:chunk-size 1})
(map (partial decode-file cfg))
(cons file)
(get-unused-components deleted)
(mapv :id)
(set))
file (update file :data process-fdata unused)]
unused
(transduce xf:map-id disj
(into #{} xf:map-id deleted-components)
(concat used-remote used-local))
file
(update file :data
(fn [data]
(reduce (fn [data id]
(l/trc :hint "delete component"
:component-id (str id)
:file-id (str file-id))
(ctkl/delete-component data id))
data
unused)))]
(l/dbg :hint "clean" :rel "components" :file-id (str file-id) :total (count unused))
file))