3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-06-10 10:41:38 +02:00
Code Issues Projects Releases Packages Wiki Activity

♻️ Make the media cleaning mechanism on file-gc task more efficient

Browse source 
Replaces the use of db/cursor with db/plan, that teorethically allows
processing large results without consuming all result set in memory
This commit is contained in:
Andrey Antukh 2025-01-15 12:36:13 +01:00
parent fde0f3c182
commit 5513daf17d
3 changed files with 44 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

23
backend/src/app/tasks/file_gc.clj
View file

@ -53,16 +53,27 @@
RETURNING id")
(def ^:private xf:collect-used-media
(comp (map :data) (mapcat bfc/collect-used-media)))
(comp
(map :data)
(mapcat bfc/collect-used-media)))
(def ^:private plan-opts
{:fetch-size 1
:concurrency :read-only
:cursors :close
:result-type :forward-only})
(defn- clean-file-media!
"Performs the garbage collection of file media objects."
[{:keys [::db/conn] :as cfg} {:keys [id] :as file}]
(let [used (into #{}
xf:collect-used-media
(cons file
(->> (db/cursor conn [sql:get-snapshots id])
(map (partial decode-file cfg)))))
(let [xform (comp
(map (partial decode-file cfg))
xf:collect-used-media)
used (->> (db/plan conn [sql:get-snapshots id] plan-opts)
(transduce xform conj #{}))
used (into used xf:collect-used-media [file])
ids (db/create-array conn "uuid" used)
unused (->> (db/exec! conn [sql:mark-file-media-object-deleted id ids])
(into #{} (map :id)))]