3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-05-16 17:36:16 +02:00
Code Issues Projects Releases Packages Wiki Activity

✨ Improve iteration and add concat-all and fully lazy mapcat helper

Browse source
This commit is contained in:
Andrey Antukh 2022-09-28 23:26:31 +02:00 • committed by Andrés Moya
parent 058727a44b
commit 886ab0e152
6 changed files with 68 additions and 86 deletions
Download patch file Download diff file Expand all files Collapse all files

7
backend/src/app/srepl/helpers.clj
View file

@ -89,8 +89,8 @@
The `on-file` parameter should be a function that receives the file
and the previous state and returns the new state."
[system & {:keys [chunk-size max-chunks start-at on-file on-error on-end]
:or {chunk-size 10 max-chunks Long/MAX_VALUE}}]
[system & {:keys [chunk-size max-items start-at on-file on-error on-end]
:or {chunk-size 10 max-items Long/MAX_VALUE}}]
(letfn [(get-chunk [conn cursor]
(let [rows (db/exec! conn [sql:retrieve-files-chunk cursor chunk-size])]
[(some->> rows peek :created-at) (seq rows)]))
@ -100,8 +100,7 @@
:vf second
:kf first
:initk (or start-at (dt/now)))
(take max-chunks)
(mapcat identity)
(take max-items)
(map #(update % :data blob/decode))))
(on-error* [file cause]

19
backend/src/app/storage.clj
View file

@ -284,11 +284,10 @@
(some->> (seq rows) (d/group-by #(-> % :backend keyword) :id #{}) seq)]))
(retrieve-deleted-objects [conn min-age]
(->> (d/iteration (partial retrieve-deleted-objects-chunk conn min-age)
:initk (dt/now)
:vf second
:kf first)
(sequence cat)))
(d/iteration (partial retrieve-deleted-objects-chunk conn min-age)
:initk (dt/now)
:vf second
:kf first))
(delete-in-bulk [conn backend-name ids]
(let [backend (impl/resolve-backend storage backend-name)
@ -397,12 +396,10 @@
(d/group-by get-bucket :id #{} rows)])))
(retrieve-touched [conn]
(->> (d/iteration (fn [cursor]
(retrieve-touched-chunk conn cursor))
:initk (dt/now)
:vf second
:kf first)
(sequence cat)))
(d/iteration (partial retrieve-touched-chunk conn)
:initk (dt/now)
:vf second
:kf first))
(process-objects! [conn get-fn ids bucket]
(loop [to-freeze #{}

9
backend/src/app/tasks/file_gc.clj
View file

@ -90,11 +90,10 @@
get-chunk (fn [cursor]
(let [rows (db/exec! conn [sql:retrieve-candidates-chunk interval cursor])]
[(some->> rows peek :modified-at) (seq rows)]))]
(sequence cat (d/iteration get-chunk
:vf second
:kf first
:initk (dt/now))))))
(d/iteration get-chunk
:vf second
:kf first
:initk (dt/now)))))
(defn collect-used-media
[data]