♻️ Refactor file persistence layer.

This commit is contained in:
Andrey Antukh 2020-09-07 10:56:42 +02:00 committed by Alonso Torres
parent 182afedc54
commit 4e694ff194
86 changed files with 3205 additions and 3313 deletions

View file

@ -36,9 +36,7 @@
:num-profiles-per-team 5
:num-projects-per-team 5
:num-files-per-project 5
:num-pages-per-file 3
:num-draft-files-per-profile 10
:num-draft-pages-per-file 3})
:num-draft-files-per-profile 10})
(defn- rng-ids
[rng n max]
@ -75,179 +73,145 @@
(defn impl-run
[opts]
(let [rng (java.util.Random. 1)
(let [rng (java.util.Random. 1)]
(letfn [(create-profile [conn index]
(let [id (mk-uuid "profile" index)
_ (log/info "create profile" id)
create-profile
(fn [conn index]
(let [id (mk-uuid "profile" index)]
(log/info "create profile" id)
(register-profile conn
{:id id
:fullname (str "Profile " index)
:password "123123"
:demo? true
:email (str "profile" index ".test@uxbox.io")})))
prof (register-profile conn
{:id id
:fullname (str "Profile " index)
:password "123123"
:demo? true
:email (str "profile" index ".test@uxbox.io")})
team-id (:default-team-id prof)
owner-id id]
(let [project-ids (collect (partial create-project conn team-id owner-id)
(range (:num-projects-per-team opts)))]
(run! (partial create-files conn owner-id) project-ids))
prof))
create-profiles
(fn [conn]
(log/info "create profiles")
(collect (partial create-profile conn)
(range (:num-profiles opts))))
(create-profiles [conn]
(log/info "create profiles")
(collect (partial create-profile conn)
(range (:num-profiles opts))))
create-team
(fn [conn index]
(let [id (mk-uuid "team" index)
name (str "Team" index)]
(log/info "create team" id)
(db/insert! conn :team {:id id
:name name
:photo ""})
id))
(create-team [conn index]
(let [id (mk-uuid "team" index)
name (str "Team" index)]
(log/info "create team" id)
(db/insert! conn :team {:id id
:name name
:photo ""})
id))
create-teams
(fn [conn]
(log/info "create teams")
(collect (partial create-team conn)
(range (:num-teams opts))))
(create-teams [conn]
(log/info "create teams")
(collect (partial create-team conn)
(range (:num-teams opts))))
create-page
(fn [conn owner-id project-id file-id index]
(let [id (mk-uuid "page" project-id file-id index)
data cp/default-page-data
name (str "page " index)
version 0
ordering index
data (blob/encode data)]
(log/info "create page" id)
(db/insert! conn :page
{:id id
:file-id file-id
:name name
:ordering ordering
:data data})))
(create-file [conn owner-id project-id index]
(let [id (mk-uuid "file" project-id index)
name (str "file" index)
data (cp/make-file-data)]
(log/info "create file" id)
(db/insert! conn :file
{:id id
:data (blob/encode data)
:project-id project-id
:name name})
(db/insert! conn :file-profile-rel
{:file-id id
:profile-id owner-id
:is-owner true
:is-admin true
:can-edit true})
id))
create-pages
(fn [conn owner-id project-id file-id]
(log/info "create pages")
(run! (partial create-page conn owner-id project-id file-id)
(range (:num-pages-per-file opts))))
(create-files [conn owner-id project-id]
(log/info "create files")
(run! (partial create-file conn owner-id project-id)
(range (:num-files-per-project opts))))
create-file
(fn [conn owner-id project-id index]
(let [id (mk-uuid "file" project-id index)
name (str "file" index)]
(log/info "create file" id)
(db/insert! conn :file
{:id id
:project-id project-id
:name name})
(db/insert! conn :file-profile-rel
{:file-id id
:profile-id owner-id
:is-owner true
:is-admin true
:can-edit true})
id))
(create-project [conn team-id owner-id index]
(let [id (mk-uuid "project" team-id index)
name (str "project " index)]
(log/info "create project" id)
(db/insert! conn :project
{:id id
:team-id team-id
:name name})
(db/insert! conn :project-profile-rel
{:project-id id
:profile-id owner-id
:is-owner true
:is-admin true
:can-edit true})
id))
create-files
(fn [conn owner-id project-id]
(log/info "create files")
(let [file-ids (collect (partial create-file conn owner-id project-id)
(range (:num-files-per-project opts)))]
(run! (partial create-pages conn owner-id project-id) file-ids)))
(create-projects [conn team-id profile-ids]
(log/info "create projects")
(let [owner-id (rng-nth rng profile-ids)
project-ids (collect (partial create-project conn team-id owner-id)
(range (:num-projects-per-team opts)))]
(run! (partial create-files conn owner-id) project-ids)))
create-project
(fn [conn team-id owner-id index]
(let [id (mk-uuid "project" team-id index)
name (str "project " index)]
(log/info "create project" id)
(db/insert! conn :project
{:id id
:team-id team-id
:name name})
(db/insert! conn :project-profile-rel
{:project-id id
:profile-id owner-id
:is-owner true
:is-admin true
:can-edit true})
id))
(assign-profile-to-team [conn team-id owner? profile-id]
(db/insert! conn :team-profile-rel
{:team-id team-id
:profile-id profile-id
:is-owner owner?
:is-admin true
:can-edit true}))
create-projects
(fn [conn team-id profile-ids]
(log/info "create projects")
(let [owner-id (rng-nth rng profile-ids)
project-ids (collect (partial create-project conn team-id owner-id)
(range (:num-projects-per-team opts)))]
(run! (partial create-files conn owner-id) project-ids)))
(setup-team [conn team-id profile-ids]
(log/info "setup team" team-id profile-ids)
(assign-profile-to-team conn team-id true (first profile-ids))
(run! (partial assign-profile-to-team conn team-id false)
(rest profile-ids))
(create-projects conn team-id profile-ids))
assign-profile-to-team
(fn [conn team-id owner? profile-id]
(db/insert! conn :team-profile-rel
{:team-id team-id
:profile-id profile-id
:is-owner owner?
:is-admin true
:can-edit true}))
(assign-teams-and-profiles [conn teams profiles]
(log/info "assign teams and profiles")
(loop [team-id (first teams)
teams (rest teams)]
(when-not (nil? team-id)
(let [n-profiles-team (:num-profiles-per-team opts)
selected-profiles (rng-vec rng profiles n-profiles-team)]
(setup-team conn team-id selected-profiles)
(recur (first teams)
(rest teams))))))
setup-team
(fn [conn team-id profile-ids]
(log/info "setup team" team-id profile-ids)
(assign-profile-to-team conn team-id true (first profile-ids))
(run! (partial assign-profile-to-team conn team-id false)
(rest profile-ids))
(create-projects conn team-id profile-ids))
(create-draft-file [conn owner index]
(let [owner-id (:id owner)
id (mk-uuid "file" "draft" owner-id index)
name (str "file" index)
project-id (:default-project-id owner)
data (cp/make-file-data)]
assign-teams-and-profiles
(fn [conn teams profiles]
(log/info "assign teams and profiles")
(loop [team-id (first teams)
teams (rest teams)]
(when-not (nil? team-id)
(let [n-profiles-team (:num-profiles-per-team opts)
selected-profiles (rng-vec rng profiles n-profiles-team)]
(setup-team conn team-id selected-profiles)
(recur (first teams)
(rest teams))))))
(log/info "create draft file" id)
(db/insert! conn :file
{:id id
:data (blob/encode data)
:project-id project-id
:name name})
(db/insert! conn :file-profile-rel
{:file-id id
:profile-id owner-id
:is-owner true
:is-admin true
:can-edit true})
id))
create-draft-pages
(fn [conn owner-id file-id]
(log/info "create draft pages")
(run! (partial create-page conn owner-id nil file-id)
(range (:num-draft-pages-per-file opts))))
create-draft-file
(fn [conn owner index]
(let [owner-id (:id owner)
id (mk-uuid "file" "draft" owner-id index)
name (str "file" index)
project-id (:default-project-id owner)]
(log/info "create draft file" id)
(db/insert! conn :file
{:id id
:project-id project-id
:name name})
(db/insert! conn :file-profile-rel
{:file-id id
:profile-id owner-id
:is-owner true
:is-admin true
:can-edit true})
id))
create-draft-files
(fn [conn profile]
(let [file-ids (collect (partial create-draft-file conn profile)
(range (:num-draft-files-per-profile opts)))]
(run! (partial create-draft-pages conn (:id profile)) file-ids)))
]
(db/with-atomic [conn db/pool]
(let [profiles (create-profiles conn)
teams (create-teams conn)]
(assign-teams-and-profiles conn teams (map :id profiles))
(run! (partial create-draft-files conn) profiles)))))
(create-draft-files [conn profile]
(run! (partial create-draft-file conn profile)
(range (:num-draft-files-per-profile opts))))
]
(db/with-atomic [conn db/pool]
(let [profiles (create-profiles conn)
teams (create-teams conn)]
(assign-teams-and-profiles conn teams (map :id profiles))
(run! (partial create-draft-files conn) profiles))))))
(defn run*
[preset]

View file

@ -31,203 +31,203 @@
;; --- Constants & Helpers
(def ^:const +graphics-uuid-ns+ #uuid "3642a582-565f-4070-beba-af797ab27a6a")
(def ^:const +colors-uuid-ns+ #uuid "3642a582-565f-4070-beba-af797ab27a6c")
;; (def ^:const +graphics-uuid-ns+ #uuid "3642a582-565f-4070-beba-af797ab27a6a")
;; (def ^:const +colors-uuid-ns+ #uuid "3642a582-565f-4070-beba-af797ab27a6c")
(s/def ::id ::us/uuid)
(s/def ::name ::us/string)
(s/def ::path ::us/string)
(s/def ::regex #(instance? java.util.regex.Pattern %))
;; (s/def ::id ::us/uuid)
;; (s/def ::name ::us/string)
;; (s/def ::path ::us/string)
;; (s/def ::regex #(instance? java.util.regex.Pattern %))
(s/def ::import-graphics
(s/keys :req-un [::path ::regex]))
;; (s/def ::import-graphics
;; (s/keys :req-un [::path ::regex]))
(s/def ::import-color
(s/* (s/cat :name ::us/string :color ::us/color)))
;; (s/def ::import-color
;; (s/* (s/cat :name ::us/string :color ::us/color)))
(s/def ::import-colors (s/coll-of ::import-color))
;; (s/def ::import-colors (s/coll-of ::import-color))
(s/def ::import-library
(s/keys :req-un [::name]
:opt-un [::import-graphics ::import-colors]))
;; (s/def ::import-library
;; (s/keys :req-un [::name]
;; :opt-un [::import-graphics ::import-colors]))
(defn exit!
([] (exit! 0))
([code]
(System/exit code)))
;; (defn exit!
;; ([] (exit! 0))
;; ([code]
;; (System/exit code)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Graphics Importer
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ;; Graphics Importer
;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn- create-media-object
[conn file-id media-object-id localpath]
(s/assert fs/path? localpath)
(s/assert ::us/uuid file-id)
(s/assert ::us/uuid media-object-id)
(let [filename (fs/name localpath)
extension (second (fs/split-ext filename))
file (io/as-file localpath)
mtype (case extension
".jpg" "image/jpeg"
".png" "image/png"
".webp" "image/webp"
".svg" "image/svg+xml")]
(log/info "Creating image" filename media-object-id)
(media/create-media-object conn {:content {:tempfile localpath
:filename filename
:content-type mtype
:size (.length file)}
:id media-object-id
:file-id file-id
:name filename
:is-local false})))
;; (defn- create-media-object
;; [conn file-id media-object-id localpath]
;; (s/assert fs/path? localpath)
;; (s/assert ::us/uuid file-id)
;; (s/assert ::us/uuid media-object-id)
;; (let [filename (fs/name localpath)
;; extension (second (fs/split-ext filename))
;; file (io/as-file localpath)
;; mtype (case extension
;; ".jpg" "image/jpeg"
;; ".png" "image/png"
;; ".webp" "image/webp"
;; ".svg" "image/svg+xml")]
;; (log/info "Creating image" filename media-object-id)
;; (media/create-media-object conn {:content {:tempfile localpath
;; :filename filename
;; :content-type mtype
;; :size (.length file)}
;; :id media-object-id
;; :file-id file-id
;; :name filename
;; :is-local false})))
(defn- media-object-exists?
[conn id]
(s/assert ::us/uuid id)
(let [row (db/get-by-id conn :media-object id)]
(if row true false)))
;; (defn- media-object-exists?
;; [conn id]
;; (s/assert ::us/uuid id)
;; (let [row (db/get-by-id conn :media-object id)]
;; (if row true false)))
(defn- import-media-object-if-not-exists
[conn file-id fpath]
(s/assert ::us/uuid file-id)
(s/assert fs/path? fpath)
(let [media-object-id (uuid/namespaced +graphics-uuid-ns+ (str file-id (fs/name fpath)))]
(when-not (media-object-exists? conn media-object-id)
(create-media-object conn file-id media-object-id fpath))
media-object-id))
;; (defn- import-media-object-if-not-exists
;; [conn file-id fpath]
;; (s/assert ::us/uuid file-id)
;; (s/assert fs/path? fpath)
;; (let [media-object-id (uuid/namespaced +graphics-uuid-ns+ (str file-id (fs/name fpath)))]
;; (when-not (media-object-exists? conn media-object-id)
;; (create-media-object conn file-id media-object-id fpath))
;; media-object-id))
(defn- import-graphics
[conn file-id {:keys [path regex]}]
(run! (fn [fpath]
(when (re-matches regex (str fpath))
(import-media-object-if-not-exists conn file-id fpath)))
(->> (fs/list-dir path)
(filter fs/regular-file?))))
;; (defn- import-graphics
;; [conn file-id {:keys [path regex]}]
;; (run! (fn [fpath]
;; (when (re-matches regex (str fpath))
;; (import-media-object-if-not-exists conn file-id fpath)))
;; (->> (fs/list-dir path)
;; (filter fs/regular-file?))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Colors Importer
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ;; Colors Importer
;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn- create-color
[conn file-id name content]
(s/assert ::us/uuid file-id)
(s/assert ::us/color content)
(let [color-id (uuid/namespaced +colors-uuid-ns+ (str file-id content))]
(log/info "Creating color" color-id "-" name content)
(colors/create-color conn {:id color-id
:file-id file-id
:name name
:content content})
color-id))
;; (defn- create-color
;; [conn file-id name content]
;; (s/assert ::us/uuid file-id)
;; (s/assert ::us/color content)
;; (let [color-id (uuid/namespaced +colors-uuid-ns+ (str file-id content))]
;; (log/info "Creating color" color-id "-" name content)
;; (colors/create-color conn {:id color-id
;; :file-id file-id
;; :name name
;; :content content})
;; color-id))
(defn- import-colors
[conn file-id colors]
(db/delete! conn :color {:file-id file-id})
(run! (fn [[name content]]
(create-color conn file-id name content))
(partition-all 2 colors)))
;; (defn- import-colors
;; [conn file-id colors]
;; (db/delete! conn :color {:file-id file-id})
;; (run! (fn [[name content]]
;; (create-color conn file-id name content))
;; (partition-all 2 colors)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Library files Importer
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ;; Library files Importer
;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn- library-file-exists?
[conn id]
(s/assert ::us/uuid id)
(let [row (db/get-by-id conn :file id)]
(if row true false)))
;; (defn- library-file-exists?
;; [conn id]
;; (s/assert ::us/uuid id)
;; (let [row (db/get-by-id conn :file id)]
;; (if row true false)))
(defn- create-library-file-if-not-exists
[conn project-id {:keys [name]}]
(let [id (uuid/namespaced +colors-uuid-ns+ name)]
(when-not (library-file-exists? conn id)
(log/info "Creating library-file:" name)
(files/create-file conn {:id id
:profile-id uuid/zero
:project-id project-id
:name name
:is-shared true})
(files/create-page conn {:file-id id}))
id))
;; (defn- create-library-file-if-not-exists
;; [conn project-id {:keys [name]}]
;; (let [id (uuid/namespaced +colors-uuid-ns+ name)]
;; (when-not (library-file-exists? conn id)
;; (log/info "Creating library-file:" name)
;; (files/create-file conn {:id id
;; :profile-id uuid/zero
;; :project-id project-id
;; :name name
;; :is-shared true})
;; (files/create-page conn {:file-id id}))
;; id))
(defn- process-library
[conn basedir project-id {:keys [graphics colors] :as library}]
(us/verify ::import-library library)
(let [library-file-id (create-library-file-if-not-exists conn project-id library)]
(when graphics
(->> (assoc graphics :path (fs/join basedir (:path graphics)))
(import-graphics conn library-file-id)))
(when colors
(import-colors conn library-file-id colors))))
;; (defn- process-library
;; [conn basedir project-id {:keys [graphics colors] :as library}]
;; (us/verify ::import-library library)
;; (let [library-file-id (create-library-file-if-not-exists conn project-id library)]
;; (when graphics
;; (->> (assoc graphics :path (fs/join basedir (:path graphics)))
;; (import-graphics conn library-file-id)))
;; (when colors
;; (import-colors conn library-file-id colors))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Entry Point
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ;; Entry Point
;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn- project-exists?
[conn id]
(s/assert ::us/uuid id)
(let [row (db/get-by-id conn :project id)]
(if row true false)))
;; (defn- project-exists?
;; [conn id]
;; (s/assert ::us/uuid id)
;; (let [row (db/get-by-id conn :project id)]
;; (if row true false)))
(defn- create-project-if-not-exists
[conn {:keys [name] :as project}]
(let [id (uuid/namespaced +colors-uuid-ns+ name)]
(when-not (project-exists? conn id)
(log/info "Creating project" name)
(projects/create-project conn {:id id
:team-id uuid/zero
:name name
:default? false}))
id))
;; (defn- create-project-if-not-exists
;; [conn {:keys [name] :as project}]
;; (let [id (uuid/namespaced +colors-uuid-ns+ name)]
;; (when-not (project-exists? conn id)
;; (log/info "Creating project" name)
;; (projects/create-project conn {:id id
;; :team-id uuid/zero
;; :name name
;; :default? false}))
;; id))
(defn- validate-path
[path]
(let [path (if (symbol? path) (str path) path)]
(log/infof "Trying to load config from '%s'." path)
(when-not path
(log/error "No path is provided")
(exit! -1))
(when-not (fs/exists? path)
(log/error "Path does not exists.")
(exit! -1))
(when (fs/directory? path)
(log/error "The provided path is a directory.")
(exit! -1))
(fs/path path)))
;; (defn- validate-path
;; [path]
;; (let [path (if (symbol? path) (str path) path)]
;; (log/infof "Trying to load config from '%s'." path)
;; (when-not path
;; (log/error "No path is provided")
;; (exit! -1))
;; (when-not (fs/exists? path)
;; (log/error "Path does not exists.")
;; (exit! -1))
;; (when (fs/directory? path)
;; (log/error "The provided path is a directory.")
;; (exit! -1))
;; (fs/path path)))
(defn- read-file
[path]
(let [reader (PushbackReader. (io/reader path))]
[(fs/parent path)
(read reader)]))
;; (defn- read-file
;; [path]
;; (let [reader (PushbackReader. (io/reader path))]
;; [(fs/parent path)
;; (read reader)]))
(defn run*
[path]
(let [[basedir libraries] (read-file path)]
(db/with-atomic [conn db/pool]
(let [project-id (create-project-if-not-exists conn {:name "System libraries"})]
(run! #(process-library conn basedir project-id %) libraries)))))
;; (defn run*
;; [path]
;; (let [[basedir libraries] (read-file path)]
;; (db/with-atomic [conn db/pool]
;; (let [project-id (create-project-if-not-exists conn {:name "System libraries"})]
;; (run! #(process-library conn basedir project-id %) libraries)))))
(defn run
[{:keys [path] :as params}]
(log/infof "Starting media loader.")
(let [path (validate-path path)]
;; (defn run
;; [{:keys [path] :as params}]
;; (log/infof "Starting media loader.")
;; (let [path (validate-path path)]
(try
(-> (mount/only #{#'app.config/config
#'app.db/pool
#'app.migrations/migrations
#'app.media/semaphore
#'app.media-storage/media-storage})
(mount/start))
(run* path)
(catch Exception e
(log/errorf e "Unhandled exception."))
(finally
(mount/stop)))))
;; (try
;; (-> (mount/only #{#'app.config/config
;; #'app.db/pool
;; #'app.migrations/migrations
;; #'app.media/semaphore
;; #'app.media-storage/media-storage})
;; (mount/start))
;; (run* path)
;; (catch Exception e
;; (log/errorf e "Unhandled exception."))
;; (finally
;; (mount/stop)))))