From b3623ed14cf9abeb353b187ed720ee72d5098bcf Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 11 Aug 2022 15:37:13 +0200 Subject: [PATCH 01/13] :tada: Add migration for remove on cascade action on file-media-object table --- backend/src/app/migrations.clj | 5 ++++- .../0078-mod-file-media-object-table-drop-cascade.sql | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 backend/src/app/migrations/sql/0078-mod-file-media-object-table-drop-cascade.sql diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index 086147ad2..3857f8e71 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -238,9 +238,12 @@ {:name "0076-mod-storage-object-table" :fn (mg/resource "app/migrations/sql/0076-mod-storage-object-table.sql")} - + {:name "0077-mod-comment-thread-table" :fn (mg/resource "app/migrations/sql/0077-mod-comment-thread-table.sql")} + + {:name "0078-mod-file-media-object-table-drop-cascade" + :fn (mg/resource "app/migrations/sql/0078-mod-file-media-object-table-drop-cascade.sql")} ]) diff --git a/backend/src/app/migrations/sql/0078-mod-file-media-object-table-drop-cascade.sql b/backend/src/app/migrations/sql/0078-mod-file-media-object-table-drop-cascade.sql new file mode 100644 index 000000000..1fe423046 --- /dev/null +++ b/backend/src/app/migrations/sql/0078-mod-file-media-object-table-drop-cascade.sql @@ -0,0 +1,9 @@ +ALTER TABLE file_media_object + DROP CONSTRAINT file_media_object_media_id_fkey, + ADD CONSTRAINT file_media_object_media_id_fkey + FOREIGN KEY (media_id) REFERENCES storage_object(id) ON DELETE NO ACTION DEFERRABLE; + +ALTER TABLE file_media_object + DROP CONSTRAINT file_media_object_thumbnail_id_fkey, + ADD CONSTRAINT file_media_object_thumbnail_id_fkey + FOREIGN KEY (thumbnail_id) REFERENCES storage_object(id) ON DELETE NO ACTION DEFERRABLE; From 918d2ab4a9c4cd5fc920d735e980f2a5317ac87d Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 11 Aug 2022 15:37:58 +0200 Subject: [PATCH 02/13] :tada: Add more helpers on srepl ns --- backend/src/app/srepl/helpers.clj | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/backend/src/app/srepl/helpers.clj b/backend/src/app/srepl/helpers.clj index 1db3b5434..307ccbc54 100644 --- a/backend/src/app/srepl/helpers.clj +++ b/backend/src/app/srepl/helpers.clj @@ -88,7 +88,7 @@ The `on-file` parameter should be a function that receives the file and the previous state and returns the new state." - [system {:keys [chunk-size on-file] :or {chunk-size 10}}] + [system & {:keys [chunk-size on-file] :or {chunk-size 10}}] (letfn [(get-chunk [conn cursor] (let [rows (db/exec! conn [sql:retrieve-files-chunk cursor chunk-size])] [(some->> rows peek :created-at) (seq rows)])) @@ -109,6 +109,21 @@ (recur state (rest files))) state))))) + +(defn analyze-file-data + [system & {:keys [id on-form on-data]}] + (let [file (get-file system id)] + (cond + (fn? on-data) + (on-data (:data file)) + + (fn? on-form) + (walk/postwalk (fn [form] + (on-form form) + form) + (:data file))) + nil)) + (defn update-pages "Apply a function to all pages of one file. The function receives a page and returns an updated page." [data f] From ada0938e27f045f0bfa0ae133598315ff14b3e02 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 11 Aug 2022 15:38:27 +0200 Subject: [PATCH 03/13] :sparkles: Remove key warning on import dialog --- frontend/src/app/main/ui/dashboard/import.cljs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/app/main/ui/dashboard/import.cljs b/frontend/src/app/main/ui/dashboard/import.cljs index 93b48d987..565dda8fb 100644 --- a/frontend/src/app/main/ui/dashboard/import.cljs +++ b/frontend/src/app/main/ui/dashboard/import.cljs @@ -7,6 +7,7 @@ (ns app.main.ui.dashboard.import (:require [app.common.data :as d] + [app.common.data.macros :as dm] [app.common.logging :as log] [app.main.data.events :as ev] [app.main.data.modal :as modal] @@ -340,6 +341,7 @@ (let [editing? (and (some? (:file-id file)) (= (:file-id file) (:editing @state)))] [:& import-entry {:state state + :key (dm/str (:id file)) :file file :editing? editing? :can-be-deleted? (> (count files) 1)}]))] From 7626d912b9f2d729f3c010d97ba2796fcd514e7f Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 11 Aug 2022 16:23:42 +0200 Subject: [PATCH 04/13] :tada: Add srepl helpers for run and print available tasks --- backend/src/app/srepl/main.clj | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index ef3dd336c..685adfdf4 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -9,7 +9,24 @@ #_:clj-kondo/ignore (:require [app.common.logging :as l] + [app.common.pprint :as p] + [app.srepl.fixes :as f] [app.srepl.helpers :as h] - [app.srepl.fixes :as f])) + [clojure.pprint :refer [pprint]])) ;; Empty namespace as main entry point for Server REPL + +(defn print-available-tasks + [system] + (let [tasks (:app.worker/registry system)] + (p/pprint (keys tasks) :level 200))) + + +(defn run-task! + ([system name] + (run-task! system name {})) + ([system name params] + (let [tasks (:app.worker/registry system)] + (if-let [task-fn (get tasks name)] + (task-fn params) + (l/warn :hint "no task found" :name name))))) From 6ee6e5e23e48f529edc77ab06098d2f5d01a039f Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 11 Aug 2022 16:24:23 +0200 Subject: [PATCH 05/13] :sparkles: Improve logging on gc-touched storage task --- backend/src/app/storage.clj | 51 +++++++++++++++----------- backend/src/app/tasks/file_gc.clj | 6 +-- backend/src/app/tasks/file_xlog_gc.clj | 3 +- 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/backend/src/app/storage.clj b/backend/src/app/storage.clj index 1a54a08b7..55bf4af67 100644 --- a/backend/src/app/storage.clj +++ b/backend/src/app/storage.clj @@ -301,7 +301,7 @@ (recur (+ total (count ids)) (rest groups))) (do - (l/info :task "gc-deleted" :count total) + (l/info :hint "gc-deleted task finished" :total total) {:deleted total}))))))) (def sql:retrieve-deleted-objects-chunk @@ -345,14 +345,14 @@ (defmethod ig/init-key ::gc-touched-task [_ {:keys [pool] :as cfg}] - (letfn [(has-team-font-variant-nrefs? [conn id] - (-> (db/exec-one! conn [sql:retrieve-team-font-variant-nrefs id id id id]) :nrefs pos?)) + (letfn [(get-team-font-variant-nrefs [conn id] + (-> (db/exec-one! conn [sql:retrieve-team-font-variant-nrefs id id id id]) :nrefs)) - (has-file-media-object-nrefs? [conn id] - (-> (db/exec-one! conn [sql:retrieve-file-media-object-nrefs id id]) :nrefs pos?)) + (get-file-media-object-nrefs [conn id] + (-> (db/exec-one! conn [sql:retrieve-file-media-object-nrefs id id]) :nrefs)) - (has-profile-nrefs? [conn id] - (-> (db/exec-one! conn [sql:retrieve-profile-nrefs id id]) :nrefs pos?)) + (get-profile-nrefs [conn id] + (-> (db/exec-one! conn [sql:retrieve-profile-nrefs id id]) :nrefs)) (mark-freeze-in-bulk [conn ids] (db/exec-one! conn ["update storage_object set touched_at=null where id = ANY(?)" @@ -395,15 +395,23 @@ :kf first) (sequence cat))) - (process-objects! [conn pred-fn ids] + (process-objects! [conn get-fn ids bucket] (loop [to-freeze #{} to-delete #{} ids (seq ids)] (if-let [id (first ids)] - (if (pred-fn conn id) - (recur (conj to-freeze id) to-delete (rest ids)) - (recur to-freeze (conj to-delete id) (rest ids))) - + (let [nrefs (get-fn conn id)] + (if (pos? nrefs) + (do + (l/debug :hint "processing storage object" + :task "gc-touched" :id id :status "freeze" + :bucket bucket :refs nrefs) + (recur (conj to-freeze id) to-delete (rest ids))) + (do + (l/debug :hint "processing storage object" + :task "gc-touched" :id id :status "delete" + :bucket bucket :refs nrefs) + (recur to-freeze (conj to-delete id) (rest ids))))) (do (some->> (seq to-freeze) (mark-freeze-in-bulk conn)) (some->> (seq to-delete) (mark-delete-in-bulk conn)) @@ -417,9 +425,9 @@ groups (retrieve-touched conn)] (if-let [[bucket ids] (first groups)] (let [[f d] (case bucket - "file-media-object" (process-objects! conn has-file-media-object-nrefs? ids) - "team-font-variant" (process-objects! conn has-team-font-variant-nrefs? ids) - "profile" (process-objects! conn has-profile-nrefs? ids) + "file-media-object" (process-objects! conn get-file-media-object-nrefs ids bucket) + "team-font-variant" (process-objects! conn get-team-font-variant-nrefs ids bucket) + "profile" (process-objects! conn get-profile-nrefs ids bucket) (ex/raise :type :internal :code :unexpected-unknown-reference :hint (dm/fmt "unknown reference %" bucket)))] @@ -427,15 +435,16 @@ (+ to-delete d) (rest groups))) (do - (l/info :task "gc-touched" :to-freeze to-freeze :to-delete to-delete) + (l/info :hint "task finished" :task "gc-touched" :to-freeze to-freeze :to-delete to-delete) {:freeze to-freeze :delete to-delete}))))))) (def sql:retrieve-touched-objects-chunk - "select so.* from storage_object as so - where so.touched_at is not null - and so.created_at < ? - order by so.created_at desc - limit 500;") + "SELECT so.* + FROM storage_object AS so + WHERE so.touched_at IS NOT NULL + AND so.created_at < ? + ORDER by so.created_at DESC + LIMIT 500;") (def sql:retrieve-file-media-object-nrefs "select ((select count(*) from file_media_object where media_id = ?) + diff --git a/backend/src/app/tasks/file_gc.clj b/backend/src/app/tasks/file_gc.clj index 29ad2eeb8..ea47b0ac2 100644 --- a/backend/src/app/tasks/file_gc.clj +++ b/backend/src/app/tasks/file_gc.clj @@ -47,7 +47,7 @@ (recur (inc total) (rest files))) (do - (l/debug :msg "finished processing files" :processed total) + (l/info :hint "files processed" :processed total) {:processed total}))))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -142,14 +142,14 @@ "delete from file_object_thumbnail " " where file_id=? and object_id=ANY(?)") res (db/exec-one! conn [sql file-id (db/create-array conn "text" unused)])] - (l/debug :hint "delete object thumbnails" :total (:next.jdbc/update-count res)))))) + (l/debug :hint "delete file object thumbnails" :file-id file-id :total (:next.jdbc/update-count res)))))) (defn- clean-file-thumbnails! [conn file-id revn] (let [sql (str "delete from file_thumbnail " " where file_id=? and revn < ?") res (db/exec-one! conn [sql file-id revn])] - (l/debug :hint "delete file thumbnails" :total (:next.jdbc/update-count res)))) + (l/debug :hint "delete file thumbnails" :file-id file-id :total (:next.jdbc/update-count res)))) (defn- process-file [{:keys [conn] :as cfg} {:keys [id data revn modified-at] :as file}] diff --git a/backend/src/app/tasks/file_xlog_gc.clj b/backend/src/app/tasks/file_xlog_gc.clj index 7b4e21ad5..88c2b1225 100644 --- a/backend/src/app/tasks/file_xlog_gc.clj +++ b/backend/src/app/tasks/file_xlog_gc.clj @@ -28,8 +28,7 @@ (let [interval (db/interval max-age) result (db/exec-one! conn [sql:delete-files-xlog interval]) result (:next.jdbc/update-count result)] - (l/info :hint "remove old file changes" - :removed result) + (l/info :hint "old file changes removed" :total result) result)))) (def ^:private From d533e37ae023f2e254d31f34af7d8cc60d0d2549 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 11 Aug 2022 16:42:28 +0200 Subject: [PATCH 06/13] :sparkles: Improve logging on gc-deleted storage task --- backend/src/app/main.clj | 3 +-- backend/src/app/storage.clj | 47 +++++++++++++++++++++-------------- backend/src/app/util/time.clj | 3 +++ 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index ad0fb139f..5628dc0cf 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -79,8 +79,7 @@ :app.storage/gc-deleted-task {:pool (ig/ref :app.db/pool) :storage (ig/ref :app.storage/storage) - :executor (ig/ref [::worker :app.worker/executor]) - :min-age (dt/duration {:hours 2})} + :executor (ig/ref [::worker :app.worker/executor])} :app.storage/gc-touched-task {:pool (ig/ref :app.db/pool)} diff --git a/backend/src/app/storage.clj b/backend/src/app/storage.clj index 55bf4af67..4fbf05a5a 100644 --- a/backend/src/app/storage.clj +++ b/backend/src/app/storage.clj @@ -270,39 +270,48 @@ (defmethod ig/pre-init-spec ::gc-deleted-task [_] (s/keys :req-un [::storage ::db/pool ::min-age ::wrk/executor])) +(defmethod ig/prep-key ::gc-deleted-task + [_ cfg] + (merge {:min-age (dt/duration {:hours 2})} + (d/without-nils cfg))) + (defmethod ig/init-key ::gc-deleted-task - [_ {:keys [pool storage min-age] :as cfg}] - (letfn [(retrieve-deleted-objects-chunk [conn cursor] + [_ {:keys [pool storage] :as cfg}] + (letfn [(retrieve-deleted-objects-chunk [conn min-age cursor] (let [min-age (db/interval min-age) rows (db/exec! conn [sql:retrieve-deleted-objects-chunk min-age cursor])] [(some-> rows peek :created-at) (some->> (seq rows) (d/group-by #(-> % :backend keyword) :id #{}) seq)])) - (retrieve-deleted-objects [conn] - (->> (d/iteration (fn [cursor] - (retrieve-deleted-objects-chunk conn cursor)) + (retrieve-deleted-objects [conn min-age] + (->> (d/iteration (partial retrieve-deleted-objects-chunk conn min-age) :initk (dt/now) :vf second :kf first) (sequence cat))) - (delete-in-bulk [conn backend ids] - (let [backend (impl/resolve-backend storage backend) + (delete-in-bulk [conn backend-name ids] + (let [backend (impl/resolve-backend storage backend-name) backend (assoc backend :conn conn)] + + (doseq [id ids] + (l/debug :hint "permanently delete storage object" :task "gc-deleted" :backend backend-name :id id)) + @(impl/del-objects-in-bulk backend ids)))] - (fn [_] - (db/with-atomic [conn pool] - (loop [total 0 - groups (retrieve-deleted-objects conn)] - (if-let [[backend ids] (first groups)] - (do - (delete-in-bulk conn backend ids) - (recur (+ total (count ids)) - (rest groups))) - (do - (l/info :hint "gc-deleted task finished" :total total) - {:deleted total}))))))) + (fn [params] + (let [min-age (or (:min-age params) (:min-age cfg))] + (db/with-atomic [conn pool] + (loop [total 0 + groups (retrieve-deleted-objects conn min-age)] + (if-let [[backend ids] (first groups)] + (do + (delete-in-bulk conn backend ids) + (recur (+ total (count ids)) + (rest groups))) + (do + (l/info :hint "task finished" :min-age (dt/format-duration min-age) :task "gc-deleted" :total total) + {:deleted total})))))))) (def sql:retrieve-deleted-objects-chunk "with items_part as ( diff --git a/backend/src/app/util/time.clj b/backend/src/app/util/time.clj index 7adeabf54..422c92fb3 100644 --- a/backend/src/app/util/time.clj +++ b/backend/src/app/util/time.clj @@ -116,6 +116,9 @@ Duration (-edn [o] (pr-str o))) +(defn format-duration + [o] + (str/lower (subs (str o) 2))) ;; --- INSTANT From ec2a3c0de1775e4232d616f16203123508cdd660 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 11 Aug 2022 16:59:57 +0200 Subject: [PATCH 07/13] :sparkles: Improve the file-gc task logging and params --- backend/src/app/tasks/file_gc.clj | 66 +++++++++++++++++++------------ 1 file changed, 40 insertions(+), 26 deletions(-) diff --git a/backend/src/app/tasks/file_gc.clj b/backend/src/app/tasks/file_gc.clj index ea47b0ac2..d9fe8a498 100644 --- a/backend/src/app/tasks/file_gc.clj +++ b/backend/src/app/tasks/file_gc.clj @@ -14,6 +14,7 @@ [app.common.logging :as l] [app.common.pages.helpers :as cph] [app.common.pages.migrations :as pmg] + [app.config :as cf] [app.db :as db] [app.util.blob :as blob] [app.util.time :as dt] @@ -29,26 +30,37 @@ ;; HANDLER ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(s/def ::max-age ::dt/duration) +(s/def ::min-age ::dt/duration) (defmethod ig/pre-init-spec ::handler [_] - (s/keys :req-un [::db/pool ::max-age])) + (s/keys :req-un [::db/pool ::min-age])) + +(defmethod ig/prep-key ::handler + [_ cfg] + (merge {:min-age cf/deletion-delay} + (d/without-nils cfg))) (defmethod ig/init-key ::handler [_ {:keys [pool] :as cfg}] - (fn [_] - (db/with-atomic [conn pool] - (let [cfg (assoc cfg :conn conn)] - (loop [total 0 - files (retrieve-candidates cfg)] - (if-let [file (first files)] - (do - (process-file cfg file) - (recur (inc total) - (rest files))) - (do - (l/info :hint "files processed" :processed total) - {:processed total}))))))) + (fn [params] + (let [cfg (merge cfg params)] + (db/with-atomic [conn pool] + (let [cfg (assoc cfg :conn conn)] + (loop [total 0 + files (retrieve-candidates cfg)] + (if-let [file (first files)] + (do + (process-file cfg file) + (recur (inc total) + (rest files))) + (do + (l/info :hint "files processed" :processed total) + + ;; Allow optional rollback passed by params + (when (:rollback? params) + (db/rollback! conn)) + + {:processed total})))))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; IMPL @@ -69,18 +81,20 @@ for update skip locked") (defn- retrieve-candidates - [{:keys [conn max-age] :as cfg}] - (let [interval (db/interval max-age) + [{:keys [conn min-age id] :as cfg}] + (if id + (do + (l/warn :hint "explicit file id passed on params" :id id) + (db/query conn :file {:id id})) + (let [interval (db/interval min-age) + get-chunk (fn [cursor] + (let [rows (db/exec! conn [sql:retrieve-candidates-chunk interval cursor])] + [(some->> rows peek :modified-at) (seq rows)]))] - get-chunk - (fn [cursor] - (let [rows (db/exec! conn [sql:retrieve-candidates-chunk interval cursor])] - [(some->> rows peek :modified-at) (seq rows)]))] - - (sequence cat (d/iteration get-chunk - :vf second - :kf first - :initk (dt/now))))) + (sequence cat (d/iteration get-chunk + :vf second + :kf first + :initk (dt/now)))))) (defn collect-used-media [data] From ac8ef1d622854f4facef96bbf1884ed57122b414 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 11 Aug 2022 17:05:10 +0200 Subject: [PATCH 08/13] :fire: Remove completly unused file-offload task --- backend/src/app/main.clj | 11 ----- backend/src/app/tasks/file_offload.clj | 63 -------------------------- 2 files changed, 74 deletions(-) delete mode 100644 backend/src/app/tasks/file_offload.clj diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index 5628dc0cf..da0ce98bd 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -270,10 +270,6 @@ {:cron #app/cron "0 30 */3,23 * * ?" :task :telemetry} - (when (cf/get :fdata-storage-backed) - {:cron #app/cron "0 0 * * * ?" ;; hourly - :task :file-offload}) - (when (contains? cf/flags :audit-log-archive) {:cron #app/cron "0 */5 * * * ?" ;; every 5m :task :audit-log-archive}) @@ -294,7 +290,6 @@ :tasks-gc (ig/ref :app.tasks.tasks-gc/handler) :telemetry (ig/ref :app.tasks.telemetry/handler) :session-gc (ig/ref :app.http.session/gc-task) - :file-offload (ig/ref :app.tasks.file-offload/handler) :audit-log-archive (ig/ref :app.loggers.audit/archive-task) :audit-log-gc (ig/ref :app.loggers.audit/gc-task)}} @@ -326,12 +321,6 @@ {:pool (ig/ref :app.db/pool) :max-age (dt/duration {:hours 72})} - :app.tasks.file-offload/handler - {:pool (ig/ref :app.db/pool) - :max-age (dt/duration {:seconds 5}) - :storage (ig/ref :app.storage/storage) - :backend (cf/get :fdata-storage-backed :fdata-s3)} - :app.tasks.telemetry/handler {:pool (ig/ref :app.db/pool) :version (:full cf/version) diff --git a/backend/src/app/tasks/file_offload.clj b/backend/src/app/tasks/file_offload.clj deleted file mode 100644 index e429d3872..000000000 --- a/backend/src/app/tasks/file_offload.clj +++ /dev/null @@ -1,63 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) UXBOX Labs SL - -(ns app.tasks.file-offload - "A maintenance task that offloads file data to an external storage (S3)." - (:require - [app.common.logging :as l] - [app.common.spec :as us] - [app.db :as db] - [app.storage :as sto] - [app.storage.impl :as simpl] - [app.util.time :as dt] - [clojure.spec.alpha :as s] - [integrant.core :as ig])) - -(def sql:offload-candidates-chunk - "select f.id, f.data from file as f - where f.data is not null - and f.modified_at < now() - ?::interval - order by f.modified_at - limit 10") - -(defn- retrieve-candidates - [{:keys [conn max-age]}] - (db/exec! conn [sql:offload-candidates-chunk max-age])) - -(defn- offload-candidate - [{:keys [storage conn backend] :as cfg} {:keys [id data] :as file}] - (l/debug :hint "offload file data" :id id) - (let [backend (simpl/resolve-backend storage backend)] - (->> (simpl/content data) - (simpl/put-object backend file)) - (db/update! conn :file - {:data nil - :data-backend (name (:id backend))} - {:id id}))) - -;; ---- STATE INIT - -(s/def ::max-age ::dt/duration) -(s/def ::backend ::us/keyword) - -(defmethod ig/pre-init-spec ::handler [_] - (s/keys :req-un [::db/pool ::max-age ::sto/storage ::backend])) - -(defmethod ig/init-key ::handler - [_ {:keys [pool max-age] :as cfg}] - (fn [_] - (db/with-atomic [conn pool] - (let [max-age (db/interval max-age) - cfg (-> cfg - (assoc :conn conn) - (assoc :max-age max-age))] - (loop [n 0] - (let [candidates (retrieve-candidates cfg)] - (if (seq candidates) - (do - (run! (partial offload-candidate cfg) candidates) - (recur (+ n (count candidates)))) - (l/debug :hint "offload summary" :count n)))))))) From df00760ffa15989af484586023601ee8befee45a Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 11 Aug 2022 17:08:36 +0200 Subject: [PATCH 09/13] :sparkles: Improve file-xlog-gc task --- backend/src/app/main.clj | 6 ++--- backend/src/app/tasks/file_gc.clj | 2 +- backend/src/app/tasks/file_xlog_gc.clj | 32 ++++++++++++++++++-------- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index da0ce98bd..c89c5b376 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -314,12 +314,10 @@ :max-age cf/deletion-delay} :app.tasks.file-gc/handler - {:pool (ig/ref :app.db/pool) - :max-age cf/deletion-delay} + {:pool (ig/ref :app.db/pool)} :app.tasks.file-xlog-gc/handler - {:pool (ig/ref :app.db/pool) - :max-age (dt/duration {:hours 72})} + {:pool (ig/ref :app.db/pool)} :app.tasks.telemetry/handler {:pool (ig/ref :app.db/pool) diff --git a/backend/src/app/tasks/file_gc.clj b/backend/src/app/tasks/file_gc.clj index d9fe8a498..bd7ff71d4 100644 --- a/backend/src/app/tasks/file_gc.clj +++ b/backend/src/app/tasks/file_gc.clj @@ -54,7 +54,7 @@ (recur (inc total) (rest files))) (do - (l/info :hint "files processed" :processed total) + (l/info :hint "task finished" :total total) ;; Allow optional rollback passed by params (when (:rollback? params) diff --git a/backend/src/app/tasks/file_xlog_gc.clj b/backend/src/app/tasks/file_xlog_gc.clj index 88c2b1225..6971f198a 100644 --- a/backend/src/app/tasks/file_xlog_gc.clj +++ b/backend/src/app/tasks/file_xlog_gc.clj @@ -8,6 +8,7 @@ "A maintenance task that performs a garbage collection of the file change (transaction) log." (:require + [app.common.data :as d] [app.common.logging :as l] [app.db :as db] [app.util.time :as dt] @@ -16,20 +17,31 @@ (declare sql:delete-files-xlog) -(s/def ::max-age ::dt/duration) +(s/def ::min-age ::dt/duration) (defmethod ig/pre-init-spec ::handler [_] - (s/keys :req-un [::db/pool ::max-age])) + (s/keys :req-un [::db/pool] + :opt-un [::min-age])) + +(defmethod ig/prep-key ::handler + [_ cfg] + (merge {:min-age (dt/duration {:hours 72})} + (d/without-nils cfg))) (defmethod ig/init-key ::handler - [_ {:keys [pool max-age] :as cfg}] - (fn [_] - (db/with-atomic [conn pool] - (let [interval (db/interval max-age) - result (db/exec-one! conn [sql:delete-files-xlog interval]) - result (:next.jdbc/update-count result)] - (l/info :hint "old file changes removed" :total result) - result)))) + [_ {:keys [pool] :as cfg}] + (fn [params] + (let [min-age (or (:min-age params) (:min-age cfg))] + (db/with-atomic [conn pool] + (let [interval (db/interval min-age) + result (db/exec-one! conn [sql:delete-files-xlog interval]) + result (:next.jdbc/update-count result)] + (l/info :hint "task finished" :min-age (dt/format-duration min-age) :total result) + + (when (:rollback? params) + (db/rollback! conn)) + + result))))) (def ^:private sql:delete-files-xlog From 5867e64d36225df033a1c1942703d9f422c6b6b8 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 11 Aug 2022 17:24:56 +0200 Subject: [PATCH 10/13] :sparkles: Improve objects-gc task --- backend/src/app/main.clj | 3 +- backend/src/app/tasks/objects_gc.clj | 76 ++++++++++++++++++---------- 2 files changed, 49 insertions(+), 30 deletions(-) diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index c89c5b376..f470ff026 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -310,8 +310,7 @@ :app.tasks.objects-gc/handler {:pool (ig/ref :app.db/pool) - :storage (ig/ref :app.storage/storage) - :max-age cf/deletion-delay} + :storage (ig/ref :app.storage/storage)} :app.tasks.file-gc/handler {:pool (ig/ref :app.db/pool)} diff --git a/backend/src/app/tasks/objects_gc.clj b/backend/src/app/tasks/objects_gc.clj index 4574d71a5..7db23c7ef 100644 --- a/backend/src/app/tasks/objects_gc.clj +++ b/backend/src/app/tasks/objects_gc.clj @@ -8,7 +8,9 @@ "A maintenance task that performs a general purpose garbage collection of deleted objects." (:require + [app.common.data :as d] [app.common.logging :as l] + [app.config :as cf] [app.db :as db] [app.media :as media] [app.storage :as sto] @@ -41,38 +43,38 @@ ;; --- IMPL: generic object deletion (defmethod delete-objects :default - [{:keys [conn max-age table] :as cfg}] + [{:keys [conn min-age table] :as cfg}] (let [sql (str/fmt sql:delete-objects {:table table :limit 50}) - result (db/exec! conn [sql max-age])] + result (db/exec! conn [sql min-age])] (doseq [{:keys [id] :as item} result] - (l/trace :hint "delete object" :table table :id id)) + (l/debug :hint "permanently delete object" :table table :id id)) (count result))) ;; --- IMPL: file deletion (defmethod delete-objects "file" - [{:keys [conn max-age table] :as cfg}] + [{:keys [conn min-age table] :as cfg}] (let [sql (str/fmt sql:delete-objects {:table table :limit 50}) - result (db/exec! conn [sql max-age])] + result (db/exec! conn [sql min-age])] (doseq [{:keys [id] :as item} result] - (l/trace :hint "delete object" :table table :id id)) + (l/debug :hint "permanently delete object" :table table :id id)) (count result))) ;; --- IMPL: team-font-variant deletion (defmethod delete-objects "team_font_variant" - [{:keys [conn max-age storage table] :as cfg}] + [{:keys [conn min-age storage table] :as cfg}] (let [sql (str/fmt sql:delete-objects {:table table :limit 50}) - fonts (db/exec! conn [sql max-age]) + fonts (db/exec! conn [sql min-age]) storage (media/configure-assets-storage storage conn)] (doseq [{:keys [id] :as font} fonts] - (l/trace :hint "delete object" :table table :id id) + (l/debug :hint "permanently delete object" :table table :id id) (some->> (:woff1-file-id font) (sto/touch-object! storage) deref) (some->> (:woff2-file-id font) (sto/touch-object! storage) deref) (some->> (:otf-file-id font) (sto/touch-object! storage) deref) @@ -82,14 +84,14 @@ ;; --- IMPL: team deletion (defmethod delete-objects "team" - [{:keys [conn max-age storage table] :as cfg}] + [{:keys [conn min-age storage table] :as cfg}] (let [sql (str/fmt sql:delete-objects {:table table :limit 50}) - teams (db/exec! conn [sql max-age]) + teams (db/exec! conn [sql min-age]) storage (assoc storage :conn conn)] (doseq [{:keys [id] :as team} teams] - (l/trace :hint "delete object" :table table :id id) + (l/debug :hint "permanently delete object" :table table :id id) (some->> (:photo-id team) (sto/touch-object! storage) deref)) (count teams))) @@ -115,17 +117,17 @@ where id in (select id from owned)") (defmethod delete-objects "profile" - [{:keys [conn max-age storage table] :as cfg}] + [{:keys [conn min-age storage table] :as cfg}] (let [sql (str/fmt sql:retrieve-deleted-profiles {:limit 50}) - profiles (db/exec! conn [sql max-age]) + profiles (db/exec! conn [sql min-age]) storage (assoc storage :conn conn)] (doseq [{:keys [id] :as profile} profiles] - (l/trace :hint "delete object" :table table :id id) + (l/debug :hint "permanently delete object" :table table :id id) ;; Mark the owned teams as deleted; this enables them to be processed ;; in the same transaction in the "team" table step. - (db/exec-one! conn [sql:mark-owned-teams-deleted id max-age]) + (db/exec-one! conn [sql:mark-owned-teams-deleted id min-age]) ;; Mark as deleted the storage object related with the photo-id ;; field. @@ -144,22 +146,40 @@ (let [res (delete-objects cfg)] (if (pos? res) (recur (+ n res)) - (l/debug :hint "table gc summary" :table table :deleted n))))) + (do + (l/debug :hint "delete summary" :table table :total n) + n))))) -(s/def ::max-age ::dt/duration) +(s/def ::min-age ::dt/duration) (defmethod ig/pre-init-spec ::handler [_] - (s/keys :req-un [::db/pool ::sto/storage ::max-age])) + (s/keys :req-un [::db/pool ::sto/storage] + :opt-un [::min-age])) + +(defmethod ig/prep-key ::handler + [_ cfg] + (merge {:min-age cf/deletion-delay} + (d/without-nils cfg))) (defmethod ig/init-key ::handler - [_ {:keys [pool max-age] :as cfg}] - (fn [task] + [_ {:keys [pool] :as cfg}] + (fn [params] ;; Checking first on task argument allows properly testing it. - (let [max-age (get task :max-age max-age)] + (let [min-age (or (:min-age params) (:min-age cfg))] (db/with-atomic [conn pool] - (let [max-age (db/interval max-age) - cfg (-> cfg - (assoc :max-age max-age) - (assoc :conn conn))] - (doseq [table target-tables] - (process-table (assoc cfg :table table)))))))) + (let [cfg (-> cfg + (assoc :min-age (db/interval min-age)) + (assoc :conn conn))] + (loop [tables (seq target-tables) + total 0] + (if-let [table (first tables)] + (recur (rest tables) + (+ total (process-table (assoc cfg :table table)))) + (do + (l/info :hint "task finished" :min-age (dt/format-duration min-age) :total total) + + (when (:rollback? params) + (db/rollback! conn)) + + {:processed total})))))))) + From 8d7baa75dee927be807d19e4424f4d7f8562e7bb Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 11 Aug 2022 17:29:25 +0200 Subject: [PATCH 11/13] :sparkles: Improve tasks-gc task --- backend/src/app/tasks/tasks_gc.clj | 33 +++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/backend/src/app/tasks/tasks_gc.clj b/backend/src/app/tasks/tasks_gc.clj index 1350c4abf..784b7db13 100644 --- a/backend/src/app/tasks/tasks_gc.clj +++ b/backend/src/app/tasks/tasks_gc.clj @@ -8,7 +8,9 @@ "A maintenance task that performs a cleanup of already executed tasks from the database table." (:require + [app.common.data :as d] [app.common.logging :as l] + [app.config :as cf] [app.db :as db] [app.util.time :as dt] [clojure.spec.alpha :as s] @@ -16,20 +18,31 @@ (declare sql:delete-completed-tasks) -(s/def ::max-age ::dt/duration) +(s/def ::min-age ::dt/duration) (defmethod ig/pre-init-spec ::handler [_] - (s/keys :req-un [::db/pool ::max-age])) + (s/keys :req-un [::db/pool] + :opt-un [::min-age])) + +(defmethod ig/prep-key ::handler + [_ cfg] + (merge {:min-age cf/deletion-delay} + (d/without-nils cfg))) (defmethod ig/init-key ::handler - [_ {:keys [pool max-age] :as cfg}] - (fn [_] - (db/with-atomic [conn pool] - (let [interval (db/interval max-age) - result (db/exec-one! conn [sql:delete-completed-tasks interval]) - result (:next.jdbc/update-count result)] - (l/debug :hint "trim completed tasks table" :removed result) - result)))) + [_ {:keys [pool] :as cfg}] + (fn [params] + (let [min-age (or (:min-age params) (:min-age cfg))] + (db/with-atomic [conn pool] + (let [interval (db/interval min-age) + result (db/exec-one! conn [sql:delete-completed-tasks interval]) + result (:next.jdbc/update-count result)] + (l/debug :hint "task finished" :total result) + + (when (:rollback? params) + (db/rollback! conn)) + + result))))) (def ^:private sql:delete-completed-tasks From 95bb3f31af8b21570f07917fbc002a7fd6a0c45b Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Fri, 12 Aug 2022 08:34:23 +0200 Subject: [PATCH 12/13] :sparkles: Fix all tasks related tests --- backend/resources/log4j2-devenv.xml | 2 +- backend/src/app/main.clj | 8 +-- backend/src/app/tasks/file_gc.clj | 32 +++++----- backend/test/app/services_files_test.clj | 65 ++++++++++++--------- backend/test/app/services_profile_test.clj | 8 +-- backend/test/app/services_projects_test.clj | 12 ++-- backend/test/app/services_teams_test.clj | 12 ++-- 7 files changed, 74 insertions(+), 65 deletions(-) diff --git a/backend/resources/log4j2-devenv.xml b/backend/resources/log4j2-devenv.xml index ff13a6c43..3b2814253 100644 --- a/backend/resources/log4j2-devenv.xml +++ b/backend/resources/log4j2-devenv.xml @@ -25,7 +25,7 @@ - + diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index f470ff026..d02a91d60 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -253,10 +253,10 @@ :task :file-xlog-gc} {:cron #app/cron "0 0 0 * * ?" ;; daily - :task :storage-deleted-gc} + :task :storage-gc-deleted} {:cron #app/cron "0 0 0 * * ?" ;; daily - :task :storage-touched-gc} + :task :storage-gc-touched} {:cron #app/cron "0 0 0 * * ?" ;; daily :task :session-gc} @@ -285,8 +285,8 @@ :objects-gc (ig/ref :app.tasks.objects-gc/handler) :file-gc (ig/ref :app.tasks.file-gc/handler) :file-xlog-gc (ig/ref :app.tasks.file-xlog-gc/handler) - :storage-deleted-gc (ig/ref :app.storage/gc-deleted-task) - :storage-touched-gc (ig/ref :app.storage/gc-touched-task) + :storage-gc-deleted (ig/ref :app.storage/gc-deleted-task) + :storage-gc-touched (ig/ref :app.storage/gc-touched-task) :tasks-gc (ig/ref :app.tasks.tasks-gc/handler) :telemetry (ig/ref :app.tasks.telemetry/handler) :session-gc (ig/ref :app.http.session/gc-task) diff --git a/backend/src/app/tasks/file_gc.clj b/backend/src/app/tasks/file_gc.clj index bd7ff71d4..5998b25c9 100644 --- a/backend/src/app/tasks/file_gc.clj +++ b/backend/src/app/tasks/file_gc.clj @@ -43,24 +43,24 @@ (defmethod ig/init-key ::handler [_ {:keys [pool] :as cfg}] (fn [params] - (let [cfg (merge cfg params)] - (db/with-atomic [conn pool] - (let [cfg (assoc cfg :conn conn)] - (loop [total 0 - files (retrieve-candidates cfg)] - (if-let [file (first files)] - (do - (process-file cfg file) - (recur (inc total) - (rest files))) - (do - (l/info :hint "task finished" :total total) + (db/with-atomic [conn pool] + (let [min-age (or (:min-age params) (:min-age cfg)) + cfg (assoc cfg :min-age min-age :conn conn)] + (loop [total 0 + files (retrieve-candidates cfg)] + (if-let [file (first files)] + (do + (process-file cfg file) + (recur (inc total) + (rest files))) + (do + (l/info :hint "task finished" :min-age (dt/format-duration min-age) :total total) - ;; Allow optional rollback passed by params - (when (:rollback? params) - (db/rollback! conn)) + ;; Allow optional rollback passed by params + (when (:rollback? params) + (db/rollback! conn)) - {:processed total})))))))) + {:processed total}))))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; IMPL diff --git a/backend/test/app/services_files_test.clj b/backend/test/app/services_files_test.clj index aa188b215..3f3cd5feb 100644 --- a/backend/test/app/services_files_test.clj +++ b/backend/test/app/services_files_test.clj @@ -187,23 +187,18 @@ ;; freeze because of the deduplication (we have uploaded 2 times ;; 2 two same files). (let [task (:app.storage/gc-touched-task th/*system*) - res (task {})] - + res (task {:min-age (dt/duration 0)})] (t/is (= 2 (:freeze res))) (t/is (= 0 (:delete res)))) - ;; run the task immediately + ;; run the file-gc task immediately without forced min-age (let [task (:app.tasks.file-gc/handler th/*system*) res (task {})] (t/is (= 0 (:processed res)))) - ;; make the file eligible for GC waiting 300ms (configured - ;; timeout for testing) - (th/sleep 300) - ;; run the task again (let [task (:app.tasks.file-gc/handler th/*system*) - res (task {})] + res (task {:min-age (dt/duration 0)})] (t/is (= 1 (:processed res)))) ;; retrieve file and check trimmed attribute @@ -220,22 +215,36 @@ (t/is (some? @(sto/get-object storage (:media-id fmo1)))) (t/is (some? @(sto/get-object storage (:thumbnail-id fmo1)))) - ;; now, we have deleted the unused file-media-object, if we - ;; execute the touched-gc task, we should see that two of them - ;; are marked to be deleted. + ;; proceed to remove usage of the file + (update-file {:file-id (:id file) + :profile-id (:id profile) + :revn 0 + :changes [{:type :del-obj + :page-id (first (get-in file [:data :pages])) + :id shid}]}) + + ;; Now, we have deleted the usag of pointers to the + ;; file-media-objects, if we pase file-gc, they should be marked + ;; as deleted. + (let [task (:app.tasks.file-gc/handler th/*system*) + res (task {:min-age (dt/duration 0)})] + (t/is (= 1 (:processed res)))) + + ;; Now that file-gc have deleted the file-media-object usage, + ;; lets execute the touched-gc task, we should see that two of + ;; them are marked to be deleted. (let [task (:app.storage/gc-touched-task th/*system*) - res (task {})] - (t/is (= 2 (:freeze res))) - (t/is (= 0 (:delete res)))) + res (task {:min-age (dt/duration 0)})] + (t/is (= 0 (:freeze res))) + (t/is (= 2 (:delete res)))) ;; Finally, check that some of the objects that are marked as ;; deleted we are unable to retrieve them using standard storage ;; public api. - (t/is (some? @(sto/get-object storage (:media-id fmo2)))) - (t/is (some? @(sto/get-object storage (:thumbnail-id fmo2)))) - (t/is (some? @(sto/get-object storage (:media-id fmo1)))) - (t/is (some? @(sto/get-object storage (:thumbnail-id fmo1)))) - + (t/is (nil? @(sto/get-object storage (:media-id fmo2)))) + (t/is (nil? @(sto/get-object storage (:thumbnail-id fmo2)))) + (t/is (nil? @(sto/get-object storage (:media-id fmo1)))) + (t/is (nil? @(sto/get-object storage (:thumbnail-id fmo1)))) ))) (t/deftest permissions-checks-creating-file @@ -353,8 +362,8 @@ :profile-id (:id profile1)})] ;; file is not deleted because it does not meet all ;; conditions to be deleted. - (let [result (task {:max-age (dt/duration 0)})] - (t/is (nil? result))) + (let [result (task {:min-age (dt/duration 0)})] + (t/is (= 0 (:processed result)))) ;; query the list of files (let [data {::th/type :project-files @@ -384,8 +393,8 @@ (t/is (= 0 (count result))))) ;; run permanent deletion (should be noop) - (let [result (task {:max-age (dt/duration {:minutes 1})})] - (t/is (nil? result))) + (let [result (task {:min-age (dt/duration {:minutes 1})})] + (t/is (= 0 (:processed result)))) ;; query the list of file libraries of a after hard deletion (let [data {::th/type :file-libraries @@ -398,8 +407,8 @@ (t/is (= 0 (count result))))) ;; run permanent deletion - (let [result (task {:max-age (dt/duration 0)})] - (t/is (nil? result))) + (let [result (task {:min-age (dt/duration 0)})] + (t/is (= 1 (:processed result)))) ;; query the list of file libraries of a after hard deletion (let [data {::th/type :file-libraries @@ -590,7 +599,7 @@ ;; run the task again (let [task (:app.tasks.file-gc/handler th/*system*) - res (task {})] + res (task {:min-age (dt/duration 0)})] (t/is (= 1 (:processed res)))) ;; check that object thumbnails are still here @@ -617,7 +626,7 @@ ;; run the task again (let [task (:app.tasks.file-gc/handler th/*system*) - res (task {})] + res (task {:min-age (dt/duration 0)})] (t/is (= 1 (:processed res)))) ;; check that the unknown frame thumbnail is deleted @@ -701,7 +710,7 @@ ;; run the task again (let [task (:app.tasks.file-gc/handler th/*system*) - res (task {})] + res (task {:min-age (dt/duration 0)})] (t/is (= 1 (:processed res)))) ;; Then query the specific revn diff --git a/backend/test/app/services_profile_test.clj b/backend/test/app/services_profile_test.clj index f9c4bd10a..68f14c3b4 100644 --- a/backend/test/app/services_profile_test.clj +++ b/backend/test/app/services_profile_test.clj @@ -126,8 +126,8 @@ ;; profile is not deleted because it does not meet all ;; conditions to be deleted. - (let [result (task {:max-age (dt/duration 0)})] - (t/is (nil? result))) + (let [result (task {:min-age (dt/duration 0)})] + (t/is (= 0 (:processed result)))) ;; Request profile to be deleted (let [params {::th/type :delete-profile @@ -145,8 +145,8 @@ (t/is (= 1 (count (:result out))))) ;; execute permanent deletion task - (let [result (task {:max-age (dt/duration "-1m")})] - (t/is (nil? result))) + (let [result (task {:min-age (dt/duration "-1m")})] + (t/is (= 1 (:processed result)))) ;; query profile after delete (let [params {::th/type :profile diff --git a/backend/test/app/services_projects_test.clj b/backend/test/app/services_projects_test.clj index a59991e38..3f7cbd63a 100644 --- a/backend/test/app/services_projects_test.clj +++ b/backend/test/app/services_projects_test.clj @@ -179,8 +179,8 @@ ;; project is not deleted because it does not meet all ;; conditions to be deleted. - (let [result (task {:max-age (dt/duration 0)})] - (t/is (nil? result))) + (let [result (task {:min-age (dt/duration 0)})] + (t/is (= 0 (:processed result)))) ;; query the list of projects (let [data {::th/type :projects @@ -210,8 +210,8 @@ (t/is (= 1 (count result))))) ;; run permanent deletion (should be noop) - (let [result (task {:max-age (dt/duration {:minutes 1})})] - (t/is (nil? result))) + (let [result (task {:min-age (dt/duration {:minutes 1})})] + (t/is (= 0 (:processed result)))) ;; query the list of files of a after soft deletion (let [data {::th/type :project-files @@ -224,8 +224,8 @@ (t/is (= 0 (count result))))) ;; run permanent deletion - (let [result (task {:max-age (dt/duration 0)})] - (t/is (nil? result))) + (let [result (task {:min-age (dt/duration 0)})] + (t/is (= 1 (:processed result)))) ;; query the list of files of a after hard deletion (let [data {::th/type :project-files diff --git a/backend/test/app/services_teams_test.clj b/backend/test/app/services_teams_test.clj index 61199fbf3..275cf58d3 100644 --- a/backend/test/app/services_teams_test.clj +++ b/backend/test/app/services_teams_test.clj @@ -99,8 +99,8 @@ ;; team is not deleted because it does not meet all ;; conditions to be deleted. - (let [result (task {:max-age (dt/duration 0)})] - (t/is (nil? result))) + (let [result (task {:min-age (dt/duration 0)})] + (t/is (= 0 (:processed result)))) ;; query the list of teams (let [data {::th/type :teams @@ -132,8 +132,8 @@ (t/is (= (:default-team-id profile1) (get-in result [0 :id]))))) ;; run permanent deletion (should be noop) - (let [result (task {:max-age (dt/duration {:minutes 1})})] - (t/is (nil? result))) + (let [result (task {:min-age (dt/duration {:minutes 1})})] + (t/is (= 0 (:processed result)))) ;; query the list of projects after hard deletion (let [data {::th/type :projects @@ -147,8 +147,8 @@ (t/is (= (:type error-data) :not-found)))) ;; run permanent deletion - (let [result (task {:max-age (dt/duration 0)})] - (t/is (nil? result))) + (let [result (task {:min-age (dt/duration 0)})] + (t/is (= 1 (:processed result)))) ;; query the list of projects of a after hard deletion (let [data {::th/type :projects From 7fa609d5f4f5a9ac5cbe7181cc9f6f043f38c93f Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Fri, 12 Aug 2022 08:52:36 +0200 Subject: [PATCH 13/13] :sparkles: Allow disable worker --- backend/resources/log4j2.xml | 9 +-- backend/src/app/config.clj | 1 + backend/src/app/main.clj | 94 ++++++++++++++++--------------- backend/src/app/rpc.clj | 2 +- backend/src/app/worker.clj | 4 +- backend/test/app/test_helpers.clj | 5 +- 6 files changed, 56 insertions(+), 59 deletions(-) diff --git a/backend/resources/log4j2.xml b/backend/resources/log4j2.xml index 5033f4726..3a0d04e3f 100644 --- a/backend/resources/log4j2.xml +++ b/backend/resources/log4j2.xml @@ -11,14 +11,7 @@ - - - - - - - - + diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index a09efb92e..5bb0119b2 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -308,6 +308,7 @@ (def default-flags [:enable-backend-api-doc + :enable-backend-worker :enable-secure-session-cookies]) (defn- parse-flags diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index d02a91d60..7700c8985 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -234,50 +234,6 @@ :app.rpc/routes {:methods (ig/ref :app.rpc/methods)} - :app.worker/worker - {:executor (ig/ref [::worker :app.worker/executor]) - :tasks (ig/ref :app.worker/registry) - :metrics (ig/ref :app.metrics/metrics) - :pool (ig/ref :app.db/pool)} - - :app.worker/cron - {:executor (ig/ref [::worker :app.worker/executor]) - :scheduler (ig/ref :app.worker/scheduler) - :tasks (ig/ref :app.worker/registry) - :pool (ig/ref :app.db/pool) - :entries - [{:cron #app/cron "0 0 0 * * ?" ;; daily - :task :file-gc} - - {:cron #app/cron "0 0 * * * ?" ;; hourly - :task :file-xlog-gc} - - {:cron #app/cron "0 0 0 * * ?" ;; daily - :task :storage-gc-deleted} - - {:cron #app/cron "0 0 0 * * ?" ;; daily - :task :storage-gc-touched} - - {:cron #app/cron "0 0 0 * * ?" ;; daily - :task :session-gc} - - {:cron #app/cron "0 0 0 * * ?" ;; daily - :task :objects-gc} - - {:cron #app/cron "0 0 0 * * ?" ;; daily - :task :tasks-gc} - - {:cron #app/cron "0 30 */3,23 * * ?" - :task :telemetry} - - (when (contains? cf/flags :audit-log-archive) - {:cron #app/cron "0 */5 * * * ?" ;; every 5m - :task :audit-log-archive}) - - (when (contains? cf/flags :audit-log-gc) - {:cron #app/cron "0 0 0 * * ?" ;; daily - :task :audit-log-gc})]} - :app.worker/registry {:metrics (ig/ref :app.metrics/metrics) :tasks @@ -394,14 +350,62 @@ {:directory (cf/get :storage-assets-fs-directory)} }) + +(def worker-config + { :app.worker/cron + {:executor (ig/ref [::worker :app.worker/executor]) + :scheduler (ig/ref :app.worker/scheduler) + :tasks (ig/ref :app.worker/registry) + :pool (ig/ref :app.db/pool) + :entries + [{:cron #app/cron "0 0 0 * * ?" ;; daily + :task :file-gc} + + {:cron #app/cron "0 0 * * * ?" ;; hourly + :task :file-xlog-gc} + + {:cron #app/cron "0 0 0 * * ?" ;; daily + :task :storage-gc-deleted} + + {:cron #app/cron "0 0 0 * * ?" ;; daily + :task :storage-gc-touched} + + {:cron #app/cron "0 0 0 * * ?" ;; daily + :task :session-gc} + + {:cron #app/cron "0 0 0 * * ?" ;; daily + :task :objects-gc} + + {:cron #app/cron "0 0 0 * * ?" ;; daily + :task :tasks-gc} + + {:cron #app/cron "0 30 */3,23 * * ?" + :task :telemetry} + + (when (contains? cf/flags :audit-log-archive) + {:cron #app/cron "0 */5 * * * ?" ;; every 5m + :task :audit-log-archive}) + + (when (contains? cf/flags :audit-log-gc) + {:cron #app/cron "0 0 0 * * ?" ;; daily + :task :audit-log-gc})]} + + :app.worker/worker + {:executor (ig/ref [::worker :app.worker/executor]) + :tasks (ig/ref :app.worker/registry) + :metrics (ig/ref :app.metrics/metrics) + :pool (ig/ref :app.db/pool)}}) + (def system nil) (defn start [] - (ig/load-namespaces system-config) + (ig/load-namespaces (merge system-config worker-config)) (alter-var-root #'system (fn [sys] (when sys (ig/halt! sys)) (-> system-config + (cond-> (contains? cf/flags :backend-worker) + (merge worker-config)) (ig/prep) (ig/init)))) (l/info :msg "welcome to penpot" diff --git a/backend/src/app/rpc.clj b/backend/src/app/rpc.clj index 477b0a633..804a62412 100644 --- a/backend/src/app/rpc.clj +++ b/backend/src/app/rpc.clj @@ -186,7 +186,7 @@ spec (or (::sv/spec mdata) (s/spec any?)) auth? (:auth mdata true)] - (l/trace :action "register" :name (::sv/name mdata)) + (l/debug :hint "register method" :name (::sv/name mdata)) (with-meta (fn [{:keys [::request] :as params}] ;; Raise authentication error when rpc method requires auth but diff --git a/backend/src/app/worker.clj b/backend/src/app/worker.clj index 52ce5ddd4..c7259b24a 100644 --- a/backend/src/app/worker.clj +++ b/backend/src/app/worker.clj @@ -224,7 +224,7 @@ :name (d/name name) :queue (d/name queue)) (do - (l/info :hint "worker started" + (l/info :hint "worker initialized" :name (d/name name) :queue (d/name queue)) (event-loop cfg))) @@ -565,7 +565,7 @@ (l/info :hint "registry initialized" :tasks (count tasks)) (reduce-kv (fn [res k v] (let [tname (name k)] - (l/trace :hint "register task" :name tname) + (l/debug :hint "register task" :name tname) (assoc res k (wrap-task-handler metrics tname v)))) {} tasks)) diff --git a/backend/test/app/test_helpers.clj b/backend/test/app/test_helpers.clj index 8849ac3cc..92642d171 100644 --- a/backend/test/app/test_helpers.clj +++ b/backend/test/app/test_helpers.clj @@ -51,6 +51,7 @@ (defn state-init [next] (let [config (-> main/system-config + (merge main/worker-config) (assoc-in [:app.msgbus/msgbus :redis-uri] (:redis-uri config)) (assoc-in [:app.db/pool :uri] (:database-uri config)) (assoc-in [:app.db/pool :username] (:database-username config)) @@ -75,9 +76,7 @@ :app.loggers.database/reporter :app.loggers.zmq/receiver :app.worker/cron - :app.worker/worker) - (d/deep-merge - {:app.tasks.file-gc/handler {:max-age (dt/duration 300)}})) + :app.worker/worker)) _ (ig/load-namespaces config) system (-> (ig/prep config) (ig/init))]