♻️ Integrate new storage subsystem.

backend/src/app/tasks/delete_object.clj

@@ -57,8 +57,3 @@
   [conn {:keys [id] :as props}]
   (let [sql "delete from project where id=? and deleted_at is not null"]
     (db/exec-one! conn [sql id])))
-
-(defmethod handle-deletion :media-object
-  [conn {:keys [id] :as props}]
-  (let [sql "delete from media_object where id=? and deleted_at is not null"]
-    (db/exec-one! conn [sql id])))

backend/src/app/tasks/file_media_gc.clj

@@ -14,35 +14,43 @@
   (:require
    [app.common.pages.migrations :as pmg]
    [app.common.spec :as us]
-   [app.config :as cfg]
    [app.db :as db]
    [app.metrics :as mtx]
+   [app.storage :as sto]
    [app.tasks :as tasks]
    [app.util.blob :as blob]
    [app.util.time :as dt]
-   [integrant.core :as ig]
    [clojure.spec.alpha :as s]
-   [clojure.tools.logging :as log]))
+   [clojure.tools.logging :as log]
+   [integrant.core :as ig]))
 
 (declare handler)
 (declare retrieve-candidates)
 (declare process-file)
 
+(s/def ::storage some?)
+
 (defmethod ig/pre-init-spec ::handler [_]
-  (s/keys :req-un [::db/pool]))
+  (s/keys :req-un [::db/pool ::storage]))
 
 (defmethod ig/init-key ::handler
-  [_ cfg]
-  (partial handler cfg))
+  [_ {:keys [metrics] :as cfg}]
+  (let [handler #(handler cfg %)]
+    (->> {:registry (:registry metrics)
+          :type :summary
+          :name "task_file_media_gc_timing"
+          :help "file media garbage collection task timing"}
+         (mtx/instrument handler))))
 
 (defn- handler
-  [{:keys [pool]} _]
+  [{:keys [pool] :as cfg} _]
   (db/with-atomic [conn pool]
-    (loop []
-      (let [files (retrieve-candidates conn)]
-        (when (seq files)
-          (run! (partial process-file conn) files)
-          (recur))))))
+    (let [cfg (assoc cfg :conn conn)]
+      (loop []
+        (let [files (retrieve-candidates cfg)]
+          (when files
+            (run! (partial process-file cfg) files)
+            (recur)))))))
 
 (defn- decode-row
   [{:keys [data] :as row}]
@@ -62,12 +70,12 @@
     for update skip locked")
 
 (defn- retrieve-candidates
-  [conn]
-  (let [threshold (:file-trimming-threshold cfg/config)
-        interval  (db/interval threshold)]
+  [{:keys [conn max-age] :as cfg}]
+  (let [interval (db/interval max-age)]
     (->> (db/exec! conn [sql:retrieve-candidates-chunk interval])
          (map (fn [{:keys [age] :as row}]
-                (assoc row :age (dt/duration {:seconds age})))))))
+                (assoc row :age (dt/duration {:seconds age}))))
+         (seq))))
 
 (def ^:private
   collect-media-xf
@@ -86,7 +94,7 @@
       (into (keys (:media data)))))
 
 (defn- process-file
-  [conn {:keys [id data age] :as file}]
+  [{:keys [conn storage] :as cfg} {:keys [id data age] :as file}]
   (let [data    (-> (blob/decode data)
                     (assoc :id id)
                     (pmg/migrate-data))
@@ -103,15 +111,11 @@
                 {:id id})
 
     (doseq [mobj unused]
-      (log/debugf "schduling object deletion: id='%s' path='%s' delay='%s'"
-                  (:id mobj) (:path mobj) cfg/default-deletion-delay)
-      (tasks/submit! conn {:name "delete-object"
-                           :delay cfg/default-deletion-delay
-                           :props {:id id :type :media-object}})
-
+      (log/debugf "deleting media object: id='%s' media-id='%s' thumb-id='%s'"
+                  (:id mobj) (:media-id mobj) (:thumbnail-id mobj))
+      (sto/del-object storage (:media-id mobj))
+      (sto/del-object storage (:thumbnail-id mobj))
       ;; Mark object as deleted
-      (db/update! conn :media-object
-                  {:deleted-at (dt/now)}
-                  {:id id}))
+      (db/delete! conn :media-object {:id (:id mobj)}))
 
     nil))

backend/src/app/tasks/remove_media.clj

@@ -1,94 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
-
-(ns app.tasks.remove-media
-  "TODO: pending to be refactored together with the storage
-  subsystem."
-  (:require
-   [app.common.spec :as us]
-   [app.db :as db]
-   ;; [app.media-storage :as mst]
-   ;; [app.metrics :as mtx]
-   ;; [app.util.storage :as ust]
-   [clojure.spec.alpha :as s]
-   [clojure.tools.logging :as log]))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Task: Remove Media
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; Task responsible of explicit action of removing a media from file
-;; system. Mainly used for profile photo change; when we really know
-;; that the previous photo becomes unused.
-
-;; (s/def ::path ::us/not-empty-string)
-;; (s/def ::props
-;;   (s/keys :req-un [::path]))
-
-;; (defn handler
-;;   [{:keys [props] :as task}]
-;;   (us/verify ::props props)
-;;   (when (ust/exists? mst/media-storage (:path props))
-;;     (ust/delete! mst/media-storage (:path props))
-;;     (log/debug "Media " (:path props) " removed.")))
-
-;; (mtx/instrument-with-summary!
-;;  {:var #'handler
-;;   :id "tasks__remove_media"
-;;   :help "Timing of remove-media task."})
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Task: Trim Media Storage
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; The main purpose of this task is analize the `pending_to_delete`
-;; table. This table stores the references to the physical files on
-;; the file system thanks to `handle_delete()` trigger.
-
-;; Example:
-;; (1) You delete an media-object. (2) This media object is marked as
-;; deleted. (3) A task (`delete-object`) is scheduled for permanent
-;; delete the object.  - If that object stores media, the database
-;; will execute the `handle_delete()` trigger which will place
-;; filesystem paths into the `pendint_to_delete` table. (4) This
-;; task (`remove-deleted-media`) permanently delete the file from the
-;; filesystem when is executed (by scheduler).
-
-;; (def ^:private
-;;   sql:retrieve-peding-to-delete
-;;   "with items_part as (
-;;      select i.id
-;;        from pending_to_delete as i
-;;       order by i.created_at
-;;       limit ?
-;;       for update skip locked
-;;    )
-;;    delete from pending_to_delete
-;;     where id in (select id from items_part)
-;;    returning *")
-
-;; (defn trim-media-storage
-;;   [_task]
-;;   (letfn [(decode-row [{:keys [data] :as row}]
-;;             (cond-> row
-;;               (db/pgobject? data) (assoc :data (db/decode-json-pgobject data))))
-;;           (retrieve-items [conn]
-;;             (->> (db/exec! conn [sql:retrieve-peding-to-delete 10])
-;;                  (map decode-row)
-;;                  (map :data)))
-;;           (remove-media [rows]
-;;             (run! (fn [item]
-;;                     (let [path (get item "path")]
-;;                       (ust/delete! mst/media-storage path)))
-;;                   rows))]
-;;     (loop []
-;;       (let [rows (retrieve-items db/pool)]
-;;         (when-not (empty? rows)
-;;           (remove-media rows)
-;;           (recur))))))