3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-05-05 12:15:54 +02:00
Code Issues Projects Releases Packages Wiki Activity

✨ Improve the file-gc task logging and params

Browse source
This commit is contained in:
Andrey Antukh 2022-08-11 16:59:57 +02:00
parent d533e37ae0
commit ec2a3c0de1
1 changed files with 40 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

66
backend/src/app/tasks/file_gc.clj
View file

@ -14,6 +14,7 @@
[app.common.logging :as l] [app.common.logging :as l]
[app.common.pages.helpers :as cph] [app.common.pages.helpers :as cph]
[app.common.pages.migrations :as pmg] [app.common.pages.migrations :as pmg]
[app.config :as cf]
[app.db :as db] [app.db :as db]
[app.util.blob :as blob] [app.util.blob :as blob]
[app.util.time :as dt] [app.util.time :as dt]
@ -29,26 +30,37 @@
;; HANDLER ;; HANDLER
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(s/def ::max-age ::dt/duration) (s/def ::min-age ::dt/duration)
(defmethod ig/pre-init-spec ::handler [_] (defmethod ig/pre-init-spec ::handler [_]
(s/keys :req-un [::db/pool ::max-age])) (s/keys :req-un [::db/pool ::min-age]))
(defmethod ig/prep-key ::handler
[_ cfg]
(merge {:min-age cf/deletion-delay}
(d/without-nils cfg)))
(defmethod ig/init-key ::handler (defmethod ig/init-key ::handler
[_ {:keys [pool] :as cfg}] [_ {:keys [pool] :as cfg}]
(fn [_] (fn [params]
(db/with-atomic [conn pool] (let [cfg (merge cfg params)]
(let [cfg (assoc cfg :conn conn)] (db/with-atomic [conn pool]
(loop [total 0 (let [cfg (assoc cfg :conn conn)]
files (retrieve-candidates cfg)] (loop [total 0
(if-let [file (first files)] files (retrieve-candidates cfg)]
(do (if-let [file (first files)]
(process-file cfg file) (do
(recur (inc total) (process-file cfg file)
(rest files))) (recur (inc total)
(do (rest files)))
(l/info :hint "files processed" :processed total) (do
{:processed total}))))))) (l/info :hint "files processed" :processed total)
;; Allow optional rollback passed by params
(when (:rollback? params)
(db/rollback! conn))
{:processed total}))))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; IMPL ;; IMPL
@ -69,18 +81,20 @@
for update skip locked") for update skip locked")
(defn- retrieve-candidates (defn- retrieve-candidates
[{:keys [conn max-age] :as cfg}] [{:keys [conn min-age id] :as cfg}]
(let [interval (db/interval max-age) (if id
(do
(l/warn :hint "explicit file id passed on params" :id id)
(db/query conn :file {:id id}))
(let [interval (db/interval min-age)
get-chunk (fn [cursor]
(let [rows (db/exec! conn [sql:retrieve-candidates-chunk interval cursor])]
[(some->> rows peek :modified-at) (seq rows)]))]
get-chunk (sequence cat (d/iteration get-chunk
(fn [cursor] :vf second
(let [rows (db/exec! conn [sql:retrieve-candidates-chunk interval cursor])] :kf first
[(some->> rows peek :modified-at) (seq rows)]))] :initk (dt/now))))))
(sequence cat (d/iteration get-chunk
:vf second
:kf first
:initk (dt/now)))))
(defn collect-used-media (defn collect-used-media
[data] [data]