From 6ee6e5e23e48f529edc77ab06098d2f5d01a039f Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 11 Aug 2022 16:24:23 +0200 Subject: [PATCH] :sparkles: Improve logging on gc-touched storage task --- backend/src/app/storage.clj | 51 +++++++++++++++----------- backend/src/app/tasks/file_gc.clj | 6 +-- backend/src/app/tasks/file_xlog_gc.clj | 3 +- 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/backend/src/app/storage.clj b/backend/src/app/storage.clj index 1a54a08b7..55bf4af67 100644 --- a/backend/src/app/storage.clj +++ b/backend/src/app/storage.clj @@ -301,7 +301,7 @@ (recur (+ total (count ids)) (rest groups))) (do - (l/info :task "gc-deleted" :count total) + (l/info :hint "gc-deleted task finished" :total total) {:deleted total}))))))) (def sql:retrieve-deleted-objects-chunk @@ -345,14 +345,14 @@ (defmethod ig/init-key ::gc-touched-task [_ {:keys [pool] :as cfg}] - (letfn [(has-team-font-variant-nrefs? [conn id] - (-> (db/exec-one! conn [sql:retrieve-team-font-variant-nrefs id id id id]) :nrefs pos?)) + (letfn [(get-team-font-variant-nrefs [conn id] + (-> (db/exec-one! conn [sql:retrieve-team-font-variant-nrefs id id id id]) :nrefs)) - (has-file-media-object-nrefs? [conn id] - (-> (db/exec-one! conn [sql:retrieve-file-media-object-nrefs id id]) :nrefs pos?)) + (get-file-media-object-nrefs [conn id] + (-> (db/exec-one! conn [sql:retrieve-file-media-object-nrefs id id]) :nrefs)) - (has-profile-nrefs? [conn id] - (-> (db/exec-one! conn [sql:retrieve-profile-nrefs id id]) :nrefs pos?)) + (get-profile-nrefs [conn id] + (-> (db/exec-one! conn [sql:retrieve-profile-nrefs id id]) :nrefs)) (mark-freeze-in-bulk [conn ids] (db/exec-one! conn ["update storage_object set touched_at=null where id = ANY(?)" @@ -395,15 +395,23 @@ :kf first) (sequence cat))) - (process-objects! [conn pred-fn ids] + (process-objects! [conn get-fn ids bucket] (loop [to-freeze #{} to-delete #{} ids (seq ids)] (if-let [id (first ids)] - (if (pred-fn conn id) - (recur (conj to-freeze id) to-delete (rest ids)) - (recur to-freeze (conj to-delete id) (rest ids))) - + (let [nrefs (get-fn conn id)] + (if (pos? nrefs) + (do + (l/debug :hint "processing storage object" + :task "gc-touched" :id id :status "freeze" + :bucket bucket :refs nrefs) + (recur (conj to-freeze id) to-delete (rest ids))) + (do + (l/debug :hint "processing storage object" + :task "gc-touched" :id id :status "delete" + :bucket bucket :refs nrefs) + (recur to-freeze (conj to-delete id) (rest ids))))) (do (some->> (seq to-freeze) (mark-freeze-in-bulk conn)) (some->> (seq to-delete) (mark-delete-in-bulk conn)) @@ -417,9 +425,9 @@ groups (retrieve-touched conn)] (if-let [[bucket ids] (first groups)] (let [[f d] (case bucket - "file-media-object" (process-objects! conn has-file-media-object-nrefs? ids) - "team-font-variant" (process-objects! conn has-team-font-variant-nrefs? ids) - "profile" (process-objects! conn has-profile-nrefs? ids) + "file-media-object" (process-objects! conn get-file-media-object-nrefs ids bucket) + "team-font-variant" (process-objects! conn get-team-font-variant-nrefs ids bucket) + "profile" (process-objects! conn get-profile-nrefs ids bucket) (ex/raise :type :internal :code :unexpected-unknown-reference :hint (dm/fmt "unknown reference %" bucket)))] @@ -427,15 +435,16 @@ (+ to-delete d) (rest groups))) (do - (l/info :task "gc-touched" :to-freeze to-freeze :to-delete to-delete) + (l/info :hint "task finished" :task "gc-touched" :to-freeze to-freeze :to-delete to-delete) {:freeze to-freeze :delete to-delete}))))))) (def sql:retrieve-touched-objects-chunk - "select so.* from storage_object as so - where so.touched_at is not null - and so.created_at < ? - order by so.created_at desc - limit 500;") + "SELECT so.* + FROM storage_object AS so + WHERE so.touched_at IS NOT NULL + AND so.created_at < ? + ORDER by so.created_at DESC + LIMIT 500;") (def sql:retrieve-file-media-object-nrefs "select ((select count(*) from file_media_object where media_id = ?) + diff --git a/backend/src/app/tasks/file_gc.clj b/backend/src/app/tasks/file_gc.clj index 29ad2eeb8..ea47b0ac2 100644 --- a/backend/src/app/tasks/file_gc.clj +++ b/backend/src/app/tasks/file_gc.clj @@ -47,7 +47,7 @@ (recur (inc total) (rest files))) (do - (l/debug :msg "finished processing files" :processed total) + (l/info :hint "files processed" :processed total) {:processed total}))))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -142,14 +142,14 @@ "delete from file_object_thumbnail " " where file_id=? and object_id=ANY(?)") res (db/exec-one! conn [sql file-id (db/create-array conn "text" unused)])] - (l/debug :hint "delete object thumbnails" :total (:next.jdbc/update-count res)))))) + (l/debug :hint "delete file object thumbnails" :file-id file-id :total (:next.jdbc/update-count res)))))) (defn- clean-file-thumbnails! [conn file-id revn] (let [sql (str "delete from file_thumbnail " " where file_id=? and revn < ?") res (db/exec-one! conn [sql file-id revn])] - (l/debug :hint "delete file thumbnails" :total (:next.jdbc/update-count res)))) + (l/debug :hint "delete file thumbnails" :file-id file-id :total (:next.jdbc/update-count res)))) (defn- process-file [{:keys [conn] :as cfg} {:keys [id data revn modified-at] :as file}] diff --git a/backend/src/app/tasks/file_xlog_gc.clj b/backend/src/app/tasks/file_xlog_gc.clj index 7b4e21ad5..88c2b1225 100644 --- a/backend/src/app/tasks/file_xlog_gc.clj +++ b/backend/src/app/tasks/file_xlog_gc.clj @@ -28,8 +28,7 @@ (let [interval (db/interval max-age) result (db/exec-one! conn [sql:delete-files-xlog interval]) result (:next.jdbc/update-count result)] - (l/info :hint "remove old file changes" - :removed result) + (l/info :hint "old file changes removed" :total result) result)))) (def ^:private