3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-05-11 08:46:38 +02:00
Code Issues Projects Releases Packages Wiki Activity

✨ Improve logging on gc-deleted storage task

Browse source
This commit is contained in:
Andrey Antukh 2022-08-11 16:42:28 +02:00
parent 6ee6e5e23e
commit d533e37ae0
3 changed files with 32 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

3
backend/src/app/main.clj
View file

@ -79,8 +79,7 @@
:app.storage/gc-deleted-task :app.storage/gc-deleted-task
{:pool (ig/ref :app.db/pool) {:pool (ig/ref :app.db/pool)
:storage (ig/ref :app.storage/storage) :storage (ig/ref :app.storage/storage)
:executor (ig/ref [::worker :app.worker/executor]) :executor (ig/ref [::worker :app.worker/executor])}
:min-age (dt/duration {:hours 2})}
:app.storage/gc-touched-task :app.storage/gc-touched-task
{:pool (ig/ref :app.db/pool)} {:pool (ig/ref :app.db/pool)}

47
backend/src/app/storage.clj
View file

@ -270,39 +270,48 @@
(defmethod ig/pre-init-spec ::gc-deleted-task [_] (defmethod ig/pre-init-spec ::gc-deleted-task [_]
(s/keys :req-un [::storage ::db/pool ::min-age ::wrk/executor])) (s/keys :req-un [::storage ::db/pool ::min-age ::wrk/executor]))
(defmethod ig/prep-key ::gc-deleted-task
[_ cfg]
(merge {:min-age (dt/duration {:hours 2})}
(d/without-nils cfg)))
(defmethod ig/init-key ::gc-deleted-task (defmethod ig/init-key ::gc-deleted-task
[_ {:keys [pool storage min-age] :as cfg}] [_ {:keys [pool storage] :as cfg}]
(letfn [(retrieve-deleted-objects-chunk [conn cursor] (letfn [(retrieve-deleted-objects-chunk [conn min-age cursor]
(let [min-age (db/interval min-age) (let [min-age (db/interval min-age)
rows (db/exec! conn [sql:retrieve-deleted-objects-chunk min-age cursor])] rows (db/exec! conn [sql:retrieve-deleted-objects-chunk min-age cursor])]
[(some-> rows peek :created-at) [(some-> rows peek :created-at)
(some->> (seq rows) (d/group-by #(-> % :backend keyword) :id #{}) seq)])) (some->> (seq rows) (d/group-by #(-> % :backend keyword) :id #{}) seq)]))
(retrieve-deleted-objects [conn] (retrieve-deleted-objects [conn min-age]
(->> (d/iteration (fn [cursor] (->> (d/iteration (partial retrieve-deleted-objects-chunk conn min-age)
(retrieve-deleted-objects-chunk conn cursor))
:initk (dt/now) :initk (dt/now)
:vf second :vf second
:kf first) :kf first)
(sequence cat))) (sequence cat)))
(delete-in-bulk [conn backend ids] (delete-in-bulk [conn backend-name ids]
(let [backend (impl/resolve-backend storage backend) (let [backend (impl/resolve-backend storage backend-name)
backend (assoc backend :conn conn)] backend (assoc backend :conn conn)]
(doseq [id ids]
(l/debug :hint "permanently delete storage object" :task "gc-deleted" :backend backend-name :id id))
@(impl/del-objects-in-bulk backend ids)))] @(impl/del-objects-in-bulk backend ids)))]
(fn [_] (fn [params]
(db/with-atomic [conn pool] (let [min-age (or (:min-age params) (:min-age cfg))]
(loop [total 0 (db/with-atomic [conn pool]
groups (retrieve-deleted-objects conn)] (loop [total 0
(if-let [[backend ids] (first groups)] groups (retrieve-deleted-objects conn min-age)]
(do (if-let [[backend ids] (first groups)]
(delete-in-bulk conn backend ids) (do
(recur (+ total (count ids)) (delete-in-bulk conn backend ids)
(rest groups))) (recur (+ total (count ids))
(do (rest groups)))
(l/info :hint "gc-deleted task finished" :total total) (do
{:deleted total}))))))) (l/info :hint "task finished" :min-age (dt/format-duration min-age) :task "gc-deleted" :total total)
{:deleted total}))))))))
(def sql:retrieve-deleted-objects-chunk (def sql:retrieve-deleted-objects-chunk
"with items_part as ( "with items_part as (

3
backend/src/app/util/time.clj
View file

@ -116,6 +116,9 @@
Duration Duration
(-edn [o] (pr-str o))) (-edn [o] (pr-str o)))
(defn format-duration
[o]
(str/lower (subs (str o) 2)))
;; --- INSTANT ;; --- INSTANT