♻️ Big refactor of the default data model.

Introduce teams.

backend/src/uxbox/tasks/delete_object.clj

@@ -0,0 +1,81 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; This Source Code Form is "Incompatible With Secondary Licenses", as
+;; defined by the Mozilla Public License, v. 2.0.
+;;
+;; Copyright (c) 2020 Andrey Antukh <niwi@niwi.nz>
+
+(ns uxbox.tasks.delete-object
+  "Generic task for permanent deletion of objects."
+  (:require
+   [clojure.spec.alpha :as s]
+   [clojure.tools.logging :as log]
+   [promesa.core :as p]
+   [uxbox.common.exceptions :as ex]
+   [uxbox.common.spec :as us]
+   [uxbox.db :as db]
+   [uxbox.media :as media]
+   [uxbox.util.storage :as ust]
+   [vertx.util :as vu]))
+
+(s/def ::type keyword?)
+(s/def ::id ::us/uuid)
+
+(s/def ::props
+  (s/keys :req-un [::id ::type]))
+
+(defmulti handle-deletion (fn [conn props] (:type props)))
+
+(defmethod handle-deletion :default
+  [conn {:keys [type id] :as props}]
+  (log/warn "no handler found for" type))
+
+(defn handler
+  [{:keys [props] :as task}]
+  (us/verify ::props props)
+  (db/with-atomic [conn db/pool]
+    (handle-deletion conn props)))
+
+(defmethod handle-deletion :image
+  [conn {:keys [id] :as props}]
+  (let [sql "delete from image where id=$1 and deleted_at is not null"]
+    (db/query-one conn [sql id])))
+
+(defmethod handle-deletion :image-collection
+  [conn {:keys [id] :as props}]
+  (let [sql "delete from image_collection
+              where id=$1 and deleted_at is not null"]
+    (db/query-one conn [sql id])))
+
+(defmethod handle-deletion :icon
+  [conn {:keys [id] :as props}]
+  (let [sql "delete from icon where id=$1 and deleted_at is not null"]
+    (db/query-one conn [sql id])))
+
+(defmethod handle-deletion :icon-collection
+  [conn {:keys [id] :as props}]
+  (let [sql "delete from icon_collection
+              where id=$1 and deleted_at is not null"]
+    (db/query-one conn [sql id])))
+
+(defmethod handle-deletion :file
+  [conn {:keys [id] :as props}]
+  (let [sql "delete from file where id=$1 and deleted_at is not null"]
+    (db/query-one conn [sql id])))
+
+(defmethod handle-deletion :file-image
+  [conn {:keys [id] :as props}]
+  (let [sql "delete from file_image where id=$1 and deleted_at is not null"]
+    (db/query-one conn [sql id])))
+
+(defmethod handle-deletion :page
+  [conn {:keys [id] :as props}]
+  (let [sql "delete from page where id=$1 and deleted_at is not null"]
+    (db/query-one conn [sql id])))
+
+(defmethod handle-deletion :page-version
+  [conn {:keys [id] :as props}]
+  (let [sql "delete from page_version where id=$1 and deleted_at is not null"]
+    (db/query-one conn [sql id])))

backend/src/uxbox/tasks/delete_profile.clj

@@ -0,0 +1,110 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; This Source Code Form is "Incompatible With Secondary Licenses", as
+;; defined by the Mozilla Public License, v. 2.0.
+;;
+;; Copyright (c) 2020 Andrey Antukh <niwi@niwi.nz>
+
+(ns uxbox.tasks.delete-profile
+  "Task for permanent deletion of profiles."
+  (:require
+   [clojure.spec.alpha :as s]
+   [clojure.tools.logging :as log]
+   [promesa.core :as p]
+   [uxbox.common.exceptions :as ex]
+   [uxbox.common.spec :as us]
+   [uxbox.db :as db]
+   [uxbox.media :as media]
+   [uxbox.util.storage :as ust]
+   [vertx.util :as vu]))
+
+(declare select-profile)
+(declare delete-profile-data)
+(declare delete-teams)
+(declare delete-files)
+(declare delete-profile)
+
+(s/def ::profile-id ::us/uuid)
+(s/def ::props
+  (s/keys :req-un [::profile-id]))
+
+(defn handler
+  [{:keys [props] :as task}]
+  (us/verify ::props props)
+  (db/with-atomic [conn db/pool]
+    (-> (select-profile conn (:profile-id props))
+        (p/then (fn [profile]
+                  (if (or (:is-demo profile)
+                          (not (nil? (:deleted-at profile))))
+                    (delete-profile-data conn (:id profile))
+                    (log/warn "Profile " (:id profile)
+                              "does not match constraints for deletion")))))))
+
+(defn- delete-profile-data
+  [conn profile-id]
+  (log/info "Proceding to delete all data related to profile" profile-id)
+  (p/do!
+   (delete-teams conn profile-id)
+   (delete-files conn profile-id)
+   (delete-profile conn profile-id)))
+
+(def ^:private sql:select-profile
+  "select id, is_demo, deleted_at
+     from profile
+    where id=$1 for update")
+
+(defn- select-profile
+  [conn profile-id]
+  (db/query-one conn [sql:select-profile profile-id]))
+
+
+(def ^:private sql:remove-owned-teams
+  "with teams as (
+     select distinct
+            tpr.team_id as id
+       from team_profile_rel as tpr
+      where tpr.profile_id = $1
+        and tpr.is_owner is true
+   ), to_delete_teams as (
+     select tpr.team_id as id
+       from team_profile_rel as tpr
+      where tpr.team_id in (select id from teams)
+      group by tpr.team_id
+     having count(tpr.profile_id) = 1
+   )
+   delete from team
+    where id in (select id from to_delete_teams)
+   returning id")
+
+(defn- delete-teams
+  [conn profile-id]
+  (-> (db/query-one conn [sql:remove-owned-teams profile-id])
+      (p/then' (constantly nil))))
+
+(def ^:private sql:remove-owned-files
+  "with files_to_delete as (
+     select distinct
+            fpr.file_id as id
+       from file_profile_rel as fpr
+      inner join file as f on (fpr.file_id = f.id)
+      where fpr.profile_id = $1
+        and fpr.is_owner is true
+        and f.project_id is null
+   )
+   delete from file
+    where id in (select id from files_to_delete)
+   returning id")
+
+(defn- delete-files
+  [conn profile-id]
+  (-> (db/query-one conn [sql:remove-owned-files profile-id])
+      (p/then' (constantly nil))))
+
+(defn delete-profile
+  [conn profile-id]
+  (let [sql "delete from profile where id=$1"]
+    (-> (db/query conn [sql profile-id])
+        (p/then' (constantly profile-id)))))
+

backend/src/uxbox/tasks/gc.clj

@@ -20,31 +20,34 @@
    [uxbox.db :as db]
    [uxbox.util.blob :as blob]))
 
-;; TODO: add images-gc with proper resource removal
-;; TODO: add icons-gc
-;; TODO: add pages-gc
-;; TODO: test this
+;; TODO: delete media referenced in pendint_to_delete table
 
-;; --- Delete Projects
+;; (def ^:private sql:delete-item
+;;   "with items_part as (
+;;      select i.id
+;;        from pending_to_delete as i
+;;       order by i.created_at
+;;       limit 1
+;;       for update skip locked
+;;    )
+;;    delete from pending_to_delete
+;;     where id in (select id from items_part)
+;;    returning *")
 
-(def ^:private sql:delete-project
-  "delete from projects
-    where id = $1
-      and deleted_at is not null;")
+;; (defn- remove-items
+;;   []
+;;   (vu/loop []
+;;     (db/with-atomic [conn db/pool]
+;;       (-> (db/query-one conn sql:delete-item)
+;;           (p/then decode-row)
+;;           (p/then (vu/wrap-blocking remove-media))
+;;           (p/then (fn [item]
+;;                     (when (not (empty? items))
+;;                       (p/recur))))))))
 
-(s/def ::id ::us/uuid)
-(s/def ::delete-project
-  (s/keys :req-un [::id]))
-
-(defn- delete-project
-  "Clean deleted projects."
-  [{:keys [id] :as props}]
-  (us/verify ::delete-project props)
-  (db/with-atomic [conn db/pool]
-    (-> (db/query-one conn [sql:delete-project id])
-        (p/then (constantly nil)))))
-
-(defn handler
-  {:uxbox.tasks/name "delete-project"}
-  [{:keys [props] :as task}]
-  (delete-project props))
+;; (defn- remove-media
+;;   [{:keys
+;;   (doseq [item files]
+;;     (ust/delete! media/media-storage (:path item))
+;;     (ust/delete! media/media-storage (:thumb-path item)))
+;;   files)

backend/src/uxbox/tasks/impl.clj

@@ -38,7 +38,7 @@
     (.printStackTrace err (java.io.PrintWriter. *out*))))
 
 (def ^:private sql:mark-as-retry
-  "update tasks
+  "update task
       set scheduled_at = clock_timestamp() + '5 seconds'::interval,
           error = $1,
           status = 'retry',
@@ -53,7 +53,7 @@
         (p/then' (constantly nil)))))
 
 (def ^:private sql:mark-as-failed
-  "update tasks
+  "update task
       set scheduled_at = clock_timestamp() + '5 seconds'::interval,
           error = $1,
           status = 'failed'
@@ -67,7 +67,7 @@
         (p/then' (constantly nil)))))
 
 (def ^:private sql:mark-as-completed
-  "update tasks
+  "update task
       set completed_at = clock_timestamp(),
           status = 'completed'
     where id = $1")
@@ -87,7 +87,7 @@
         nil))))
 
 (def ^:private sql:select-next-task
-  "select * from tasks as t
+  "select * from task as t
     where t.scheduled_at <= now()
       and t.queue = $1
       and (t.status = 'new' or (t.status = 'retry' and t.retry_num <= $2))
@@ -141,7 +141,7 @@
                     (event-loop-handler (assoc options ::counter (inc counter)))))))))
 
 (def ^:private sql:insert-new-task
-  "insert into tasks (name, props, queue, scheduled_at)
+  "insert into task (name, props, queue, scheduled_at)
    values ($1, $2, $3, clock_timestamp()+cast($4::text as interval))
    returning id")
 
@@ -162,7 +162,7 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (def ^:privatr sql:upsert-scheduled-task
-  "insert into scheduled_tasks (id, cron_expr)
+  "insert into scheduled_task (id, cron_expr)
    values ($1, $2)
        on conflict (id)
        do update set cron_expr=$2")
@@ -178,7 +178,7 @@
     (p/run! (partial synchronize-schedule-item conn) schedule)))
 
 (def ^:private sql:lock-scheduled-task
-  "select id from scheduled_tasks where id=$1 for update skip locked")
+  "select id from scheduled_task where id=$1 for update skip locked")
 
 (declare schedule-task)
 

backend/src/uxbox/tasks/remove_demo_profile.clj

@@ -1,93 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 Andrey Antukh <niwi@niwi.nz>
-
-(ns uxbox.tasks.remove-demo-profile
-  "Demo accounts garbage collector."
-  (:require
-   [clojure.spec.alpha :as s]
-   [clojure.tools.logging :as log]
-   [promesa.core :as p]
-   [uxbox.common.exceptions :as ex]
-   [uxbox.common.spec :as us]
-   [uxbox.db :as db]
-   [uxbox.media :as media]
-   [uxbox.util.storage :as ust]
-   [vertx.util :as vu]))
-
-(declare remove-file-images)
-(declare remove-images)
-(declare remove-profile)
-
-(s/def ::id ::us/uuid)
-(s/def ::props
-  (s/keys :req-un [::id]))
-
-(defn handler
-  [{:keys [props] :as task}]
-  (us/verify ::props props)
-  (db/with-atomic [conn db/pool]
-    (remove-file-images conn (:id props))
-    (remove-images conn (:id props))
-    (remove-profile conn (:id props))))
-
-(defn- remove-files
-  [files]
-  (doseq [item files]
-    (ust/delete! media/media-storage (:path item))
-    (ust/delete! media/media-storage (:thumb-path item)))
-  files)
-
-(def ^:private sql:delete-file-images
-  "with images_part as (
-     select pfi.id
-       from project_file_images as pfi
-      inner join project_files as pf on (pf.id = pfi.file_id)
-      inner join projects as p on (p.id = pf.project_id)
-      where p.user_id = $1
-      limit 10
-   )
-   delete from project_file_images
-    where id in (select id from images_part)
-   returning id, path, thumb_path")
-
-(defn remove-file-images
-  [conn id]
-  (vu/loop []
-    (-> (db/query conn [sql:delete-file-images id])
-        (p/then (vu/wrap-blocking remove-files))
-        (p/then (fn [images]
-                  (when (not (empty? images))
-                    (p/recur)))))))
-
-(def ^:private sql:delete-images
-  "with images_part as (
-     select img.id
-       from images as img
-      where img.user_id = $1
-      limit 10
-   )
-   delete from images
-    where id in (select id from images_part)
-   returning id, path, thumb_path")
-
-(defn- remove-images
-  [conn id]
-  (vu/loop []
-    (-> (db/query conn [sql:delete-images id])
-        (p/then (vu/wrap-blocking remove-files))
-        (p/then (fn [images]
-                  (when (not (empty? images))
-                    (p/recur)))))))
-
-(defn remove-profile
-  [conn id]
-  (let [sql "delete from users where id=$1"]
-    (db/query conn [sql id])))
-
-