✨ Initial work on remove-media and remove-demo-profile tasks.

backend/src/uxbox/tasks/impl.clj

@@ -21,17 +21,16 @@
    [uxbox.util.blob :as blob]
    [uxbox.util.time :as tm]
    [vertx.core :as vc]
+   [vertx.util :as vu]
    [vertx.timers :as vt])
   (:import
    java.time.Duration
    java.time.Instant
    java.util.Date))
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Implementation
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; --- Task Execution
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Tasks
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defn- string-strack-trace
   [^Throwable err]
@@ -102,7 +101,7 @@
     (cond-> row
       props (assoc :props (blob/decode props)))))
 
-(defn- log-error
+(defn- log-task-error
   [item err]
   (log/error "Unhandled exception on task '" (:name item)
              "' (retry:" (:retry-num item) ") \n"
@@ -118,11 +117,12 @@
           (p/then decode-task-row)
           (p/then (fn [item]
                     (when item
+                      (log/debug "Execute task " (:name item))
                       (-> (p/do! (handle-task tasks item))
                           (p/handle (fn [v e]
                                       (if e
                                         (do
-                                          (log-error item e)
+                                          (log-task-error item e)
                                           (if (>= (:retry-num item) max-retries)
                                             (mark-as-failed conn item e)
                                             (reschedule conn item e)))
@@ -156,8 +156,9 @@
                            ::vt/delay 3000
                            ::vt/repeat true)))
 
-
-;; --- Task Scheduling
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Scheduled Tasks
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (def ^:privatr sql:upsert-scheduled-task
   "insert into scheduled_tasks (id, cron_expr)
@@ -180,24 +181,29 @@
 
 (declare schedule-task)
 
-(defn thr-name
-  []
-  (.getName (Thread/currentThread)))
+(defn- log-scheduled-task-error
+  [item err]
+  (log/error "Unhandled exception on scheduled task '" (:id item) "' \n"
+             (with-out-str
+               (.printStackTrace ^Throwable err (java.io.PrintWriter. *out*)))))
 
 (defn- execute-scheduled-task
   [{:keys [id cron] :as stask}]
   (db/with-atomic [conn db/pool]
+    ;; First we try to lock the task in the database, if locking us
+    ;; successful, then we execute the scheduled task; if locking is
+    ;; not possible (because other instance is already locked id) we
+    ;; just skip it and schedule to be executed in the next slot.
     (-> (db/query-one conn [sql:lock-scheduled-task id])
         (p/then (fn [result]
                   (when result
                     (-> (p/do! ((:fn stask) stask))
                         (p/catch (fn [e]
-                                   (log/warn "Excepton happens on executing scheduled task" e)
+                                   (log-scheduled-task-error stask e)
                                    nil))))))
         (p/finally (fn [v e]
-                     (-> (vc/current-context)
+                     (-> (vu/current-context)
                          (schedule-task stask)))))))
-
 (defn ms-until-valid
   [cron]
   (s/assert tm/cron? cron)
@@ -221,9 +227,9 @@
       (p/then' (fn [_]
                  (run! #(schedule-task ctx %) schedule)))))
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Public API
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; --- Worker Verticle
 
@@ -270,14 +276,15 @@
 (s/def ::delay ::us/integer)
 (s/def ::queue ::us/string)
 (s/def ::task-options
-  (s/keys :req-un [::name ::delay]
-          :opt-un [::props ::queue]))
+  (s/keys :req-un [::name]
+          :opt-un [::delay ::props ::queue]))
 
 (defn schedule!
-  [conn {:keys [name delay props queue key] :as options}]
+  [conn {:keys [name delay props queue key]
+         :or {delay 0 props {} queue "default"}
+         :as options}]
   (us/verify ::task-options options)
-  (let [queue (if (string? queue) queue "default")
-        duration (-> (tm/duration delay)
+  (let [duration (-> (tm/duration delay)
                      (duration->pginterval))
         props (blob/encode props)]
     (-> (db/query-one conn  [sql:insert-new-task name props queue duration])

backend/src/uxbox/tasks/remove_demo_profile.clj

@@ -0,0 +1,93 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; This Source Code Form is "Incompatible With Secondary Licenses", as
+;; defined by the Mozilla Public License, v. 2.0.
+;;
+;; Copyright (c) 2020 Andrey Antukh <niwi@niwi.nz>
+
+(ns uxbox.tasks.remove-demo-profile
+  "Demo accounts garbage collector."
+  (:require
+   [clojure.spec.alpha :as s]
+   [clojure.tools.logging :as log]
+   [promesa.core :as p]
+   [uxbox.common.exceptions :as ex]
+   [uxbox.common.spec :as us]
+   [uxbox.db :as db]
+   [uxbox.media :as media]
+   [uxbox.util.storage :as ust]
+   [vertx.util :as vu]))
+
+(declare remove-file-images)
+(declare remove-images)
+(declare remove-profile)
+
+(s/def ::id ::us/uuid)
+(s/def ::props
+  (s/keys :req-un [::id]))
+
+(defn handler
+  [{:keys [props] :as task}]
+  (us/verify ::props props)
+  (prn "handler" props (.getName (Thread/currentThread)))
+  (db/with-atomic [conn db/pool]
+    (remove-file-images conn (:id props))
+    (remove-images conn (:id props))
+    (remove-profile conn (:id props))
+    (prn "finished" (.getName (Thread/currentThread)))))
+
+(def ^:private sql:file-images-to-delete
+  "select pfi.id, pfi.path, pfi.thumb_path
+     from project_file_images as pfi
+    inner join project_files as pf on (pf.id = pfi.file_id)
+    inner join projects as p on (p.id = pf.project_id)
+    where p.user_id = $1
+    limit 2")
+
+(defn remove-file-images
+  [conn id]
+  (p/loop []
+    (p/let [files (db/query conn [sql:file-images-to-delete id])]
+      (prn "remove-file-images" files)
+      (when-not (empty? files)
+        (-> (vu/blocking
+             (doseq [item files]
+               (ust/delete! media/media-storage (:path item))
+               (ust/delete! media/media-storage (:thumb-path item))))
+            (p/then' #(p/recur)))))))
+
+(def ^:private sql:images
+  "select img.id, img.path, img.thumb_path
+     from images as img
+    where img.user_id = $1
+    limit 5")
+
+(defn remove-files
+  [files]
+  (prn "remove-files" (.getName (Thread/currentThread)))
+  (doseq [item files]
+    (ust/delete! media/media-storage (:path item))
+    (ust/delete! media/media-storage (:thumb-path item)))
+  files)
+
+(defn remove-images
+  [conn id]
+  (prn "remove-images" (.getName (Thread/currentThread)))
+  (vu/loop [i 0]
+    (prn "remove-images loop" i (.getName (Thread/currentThread)))
+    (-> (db/query conn [sql:images id])
+        (p/then (vu/wrap-blocking remove-files))
+        (p/then (fn [images]
+                  (prn "ending" (.getName (Thread/currentThread)))
+                  (when (and (not (empty? images))
+                             (< i 1000))
+                    (p/recur (inc i))))))))
+
+(defn remove-profile
+  [conn id]
+  (let [sql "delete from users where id=$1"]
+    (db/query conn [sql id])))
+
+

backend/src/uxbox/tasks/remove_media.clj

@@ -7,17 +7,26 @@
 ;;
 ;; Copyright (c) 2020 Andrey Antukh <niwi@niwi.nz>
 
-(ns uxbox.tasks.demo-gc
+(ns uxbox.tasks.remove-media
   "Demo accounts garbage collector."
   (:require
+   [clojure.spec.alpha :as s]
    [clojure.tools.logging :as log]
-   [uxbox.common.exceptions :as ex]))
+   [uxbox.common.exceptions :as ex]
+   [uxbox.common.spec :as us]
+   [uxbox.media :as media]
+   [uxbox.util.storage :as ust]
+   [vertx.util :as vu]))
+
+(s/def ::path ::us/string)
+(s/def ::props
+  (s/keys :req-un [::path]))
 
 (defn handler
-  {:uxbox.tasks/name "demo-gc"}
   [{:keys [props] :as task}]
-  (try
-    (Thread/sleep 100)
-    (prn (.getName (Thread/currentThread)) "demo-gc" (:id task) (:props task))
-    (catch Throwable e
-      nil)))
+  (us/verify ::props props)
+  (vu/blocking
+   (when (ust/exists? media/media-storage (:path props))
+     (ust/delete! media/media-storage (:path props))
+     (log/debug "Media " (:path props) " removed."))))
+