3pp-mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-05-10 06:16:37 +02:00
Code Issues Projects Releases Packages Wiki Activity

♻️ Refactor file persistence layer.

Browse source
This commit is contained in:
Andrey Antukh 2020-09-07 10:56:42 +02:00 • committed by Alonso Torres
parent 182afedc54
commit 4e694ff194
86 changed files with 3205 additions and 3313 deletions
Download patch file Download diff file Expand all files Collapse all files

288
backend/src/app/cli/fixtures.clj
View file

@ -36,9 +36,7 @@
:num-profiles-per-team 5
:num-projects-per-team 5
:num-files-per-project 5
:num-pages-per-file 3
:num-draft-files-per-profile 10
:num-draft-pages-per-file 3})
:num-draft-files-per-profile 10})
(defn- rng-ids
[rng n max]
@ -75,179 +73,145 @@
(defn impl-run
[opts]
(let [rng (java.util.Random. 1)
(let [rng (java.util.Random. 1)]
(letfn [(create-profile [conn index]
(let [id (mk-uuid "profile" index)
_ (log/info "create profile" id)
create-profile
(fn [conn index]
(let [id (mk-uuid "profile" index)]
(log/info "create profile" id)
(register-profile conn
{:id id
:fullname (str "Profile " index)
:password "123123"
:demo? true
:email (str "profile" index ".test@uxbox.io")})))
prof (register-profile conn
{:id id
:fullname (str "Profile " index)
:password "123123"
:demo? true
:email (str "profile" index ".test@uxbox.io")})
team-id (:default-team-id prof)
owner-id id]
(let [project-ids (collect (partial create-project conn team-id owner-id)
(range (:num-projects-per-team opts)))]
(run! (partial create-files conn owner-id) project-ids))
prof))
create-profiles
(fn [conn]
(log/info "create profiles")
(collect (partial create-profile conn)
(range (:num-profiles opts))))
(create-profiles [conn]
(log/info "create profiles")
(collect (partial create-profile conn)
(range (:num-profiles opts))))
create-team
(fn [conn index]
(let [id (mk-uuid "team" index)
name (str "Team" index)]
(log/info "create team" id)
(db/insert! conn :team {:id id
:name name
:photo ""})
id))
(create-team [conn index]
(let [id (mk-uuid "team" index)
name (str "Team" index)]
(log/info "create team" id)
(db/insert! conn :team {:id id
:name name
:photo ""})
id))
create-teams
(fn [conn]
(log/info "create teams")
(collect (partial create-team conn)
(range (:num-teams opts))))
(create-teams [conn]
(log/info "create teams")
(collect (partial create-team conn)
(range (:num-teams opts))))
create-page
(fn [conn owner-id project-id file-id index]
(let [id (mk-uuid "page" project-id file-id index)
data cp/default-page-data
name (str "page " index)
version 0
ordering index
data (blob/encode data)]
(log/info "create page" id)
(db/insert! conn :page
{:id id
:file-id file-id
:name name
:ordering ordering
:data data})))
(create-file [conn owner-id project-id index]
(let [id (mk-uuid "file" project-id index)
name (str "file" index)
data (cp/make-file-data)]
(log/info "create file" id)
(db/insert! conn :file
{:id id
:data (blob/encode data)
:project-id project-id
:name name})
(db/insert! conn :file-profile-rel
{:file-id id
:profile-id owner-id
:is-owner true
:is-admin true
:can-edit true})
id))
create-pages
(fn [conn owner-id project-id file-id]
(log/info "create pages")
(run! (partial create-page conn owner-id project-id file-id)
(range (:num-pages-per-file opts))))
(create-files [conn owner-id project-id]
(log/info "create files")
(run! (partial create-file conn owner-id project-id)
(range (:num-files-per-project opts))))
create-file
(fn [conn owner-id project-id index]
(let [id (mk-uuid "file" project-id index)
name (str "file" index)]
(log/info "create file" id)
(db/insert! conn :file
{:id id
:project-id project-id
:name name})
(db/insert! conn :file-profile-rel
{:file-id id
:profile-id owner-id
:is-owner true
:is-admin true
:can-edit true})
id))
(create-project [conn team-id owner-id index]
(let [id (mk-uuid "project" team-id index)
name (str "project " index)]
(log/info "create project" id)
(db/insert! conn :project
{:id id
:team-id team-id
:name name})
(db/insert! conn :project-profile-rel
{:project-id id
:profile-id owner-id
:is-owner true
:is-admin true
:can-edit true})
id))
create-files
(fn [conn owner-id project-id]
(log/info "create files")
(let [file-ids (collect (partial create-file conn owner-id project-id)
(range (:num-files-per-project opts)))]
(run! (partial create-pages conn owner-id project-id) file-ids)))
(create-projects [conn team-id profile-ids]
(log/info "create projects")
(let [owner-id (rng-nth rng profile-ids)
project-ids (collect (partial create-project conn team-id owner-id)
(range (:num-projects-per-team opts)))]
(run! (partial create-files conn owner-id) project-ids)))
create-project
(fn [conn team-id owner-id index]
(let [id (mk-uuid "project" team-id index)
name (str "project " index)]
(log/info "create project" id)
(db/insert! conn :project
{:id id
:team-id team-id
:name name})
(db/insert! conn :project-profile-rel
{:project-id id
:profile-id owner-id
:is-owner true
:is-admin true
:can-edit true})
id))
(assign-profile-to-team [conn team-id owner? profile-id]
(db/insert! conn :team-profile-rel
{:team-id team-id
:profile-id profile-id
:is-owner owner?
:is-admin true
:can-edit true}))
create-projects
(fn [conn team-id profile-ids]
(log/info "create projects")
(let [owner-id (rng-nth rng profile-ids)
project-ids (collect (partial create-project conn team-id owner-id)
(range (:num-projects-per-team opts)))]
(run! (partial create-files conn owner-id) project-ids)))
(setup-team [conn team-id profile-ids]
(log/info "setup team" team-id profile-ids)
(assign-profile-to-team conn team-id true (first profile-ids))
(run! (partial assign-profile-to-team conn team-id false)
(rest profile-ids))
(create-projects conn team-id profile-ids))
assign-profile-to-team
(fn [conn team-id owner? profile-id]
(db/insert! conn :team-profile-rel
{:team-id team-id
:profile-id profile-id
:is-owner owner?
:is-admin true
:can-edit true}))
(assign-teams-and-profiles [conn teams profiles]
(log/info "assign teams and profiles")
(loop [team-id (first teams)
teams (rest teams)]
(when-not (nil? team-id)
(let [n-profiles-team (:num-profiles-per-team opts)
selected-profiles (rng-vec rng profiles n-profiles-team)]
(setup-team conn team-id selected-profiles)
(recur (first teams)
(rest teams))))))
setup-team
(fn [conn team-id profile-ids]
(log/info "setup team" team-id profile-ids)
(assign-profile-to-team conn team-id true (first profile-ids))
(run! (partial assign-profile-to-team conn team-id false)
(rest profile-ids))
(create-projects conn team-id profile-ids))
(create-draft-file [conn owner index]
(let [owner-id (:id owner)
id (mk-uuid "file" "draft" owner-id index)
name (str "file" index)
project-id (:default-project-id owner)
data (cp/make-file-data)]
assign-teams-and-profiles
(fn [conn teams profiles]
(log/info "assign teams and profiles")
(loop [team-id (first teams)
teams (rest teams)]
(when-not (nil? team-id)
(let [n-profiles-team (:num-profiles-per-team opts)
selected-profiles (rng-vec rng profiles n-profiles-team)]
(setup-team conn team-id selected-profiles)
(recur (first teams)
(rest teams))))))
(log/info "create draft file" id)
(db/insert! conn :file
{:id id
:data (blob/encode data)
:project-id project-id
:name name})
(db/insert! conn :file-profile-rel
{:file-id id
:profile-id owner-id
:is-owner true
:is-admin true
:can-edit true})
id))
create-draft-pages
(fn [conn owner-id file-id]
(log/info "create draft pages")
(run! (partial create-page conn owner-id nil file-id)
(range (:num-draft-pages-per-file opts))))
create-draft-file
(fn [conn owner index]
(let [owner-id (:id owner)
id (mk-uuid "file" "draft" owner-id index)
name (str "file" index)
project-id (:default-project-id owner)]
(log/info "create draft file" id)
(db/insert! conn :file
{:id id
:project-id project-id
:name name})
(db/insert! conn :file-profile-rel
{:file-id id
:profile-id owner-id
:is-owner true
:is-admin true
:can-edit true})
id))
create-draft-files
(fn [conn profile]
(let [file-ids (collect (partial create-draft-file conn profile)
(range (:num-draft-files-per-profile opts)))]
(run! (partial create-draft-pages conn (:id profile)) file-ids)))
]
(db/with-atomic [conn db/pool]
(let [profiles (create-profiles conn)
teams (create-teams conn)]
(assign-teams-and-profiles conn teams (map :id profiles))
(run! (partial create-draft-files conn) profiles)))))
(create-draft-files [conn profile]
(run! (partial create-draft-file conn profile)
(range (:num-draft-files-per-profile opts))))
]
(db/with-atomic [conn db/pool]
(let [profiles (create-profiles conn)
teams (create-teams conn)]
(assign-teams-and-profiles conn teams (map :id profiles))
(run! (partial create-draft-files conn) profiles))))))
(defn run*
[preset]

342
backend/src/app/cli/media_loader.clj
View file

@ -31,203 +31,203 @@
;; --- Constants & Helpers
(def ^:const +graphics-uuid-ns+ #uuid "3642a582-565f-4070-beba-af797ab27a6a")
(def ^:const +colors-uuid-ns+ #uuid "3642a582-565f-4070-beba-af797ab27a6c")
;; (def ^:const +graphics-uuid-ns+ #uuid "3642a582-565f-4070-beba-af797ab27a6a")
;; (def ^:const +colors-uuid-ns+ #uuid "3642a582-565f-4070-beba-af797ab27a6c")
(s/def ::id ::us/uuid)
(s/def ::name ::us/string)
(s/def ::path ::us/string)
(s/def ::regex #(instance? java.util.regex.Pattern %))
;; (s/def ::id ::us/uuid)
;; (s/def ::name ::us/string)
;; (s/def ::path ::us/string)
;; (s/def ::regex #(instance? java.util.regex.Pattern %))
(s/def ::import-graphics
(s/keys :req-un [::path ::regex]))
;; (s/def ::import-graphics
;; (s/keys :req-un [::path ::regex]))
(s/def ::import-color
(s/* (s/cat :name ::us/string :color ::us/color)))
;; (s/def ::import-color
;; (s/* (s/cat :name ::us/string :color ::us/color)))
(s/def ::import-colors (s/coll-of ::import-color))
;; (s/def ::import-colors (s/coll-of ::import-color))
(s/def ::import-library
(s/keys :req-un [::name]
:opt-un [::import-graphics ::import-colors]))
;; (s/def ::import-library
;; (s/keys :req-un [::name]
;; :opt-un [::import-graphics ::import-colors]))
(defn exit!
([] (exit! 0))
([code]
(System/exit code)))
;; (defn exit!
;; ([] (exit! 0))
;; ([code]
;; (System/exit code)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Graphics Importer
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ;; Graphics Importer
;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn- create-media-object
[conn file-id media-object-id localpath]
(s/assert fs/path? localpath)
(s/assert ::us/uuid file-id)
(s/assert ::us/uuid media-object-id)
(let [filename (fs/name localpath)
extension (second (fs/split-ext filename))
file (io/as-file localpath)
mtype (case extension
".jpg" "image/jpeg"
".png" "image/png"
".webp" "image/webp"
".svg" "image/svg+xml")]
(log/info "Creating image" filename media-object-id)
(media/create-media-object conn {:content {:tempfile localpath
:filename filename
:content-type mtype
:size (.length file)}
:id media-object-id
:file-id file-id
:name filename
:is-local false})))
;; (defn- create-media-object
;; [conn file-id media-object-id localpath]
;; (s/assert fs/path? localpath)
;; (s/assert ::us/uuid file-id)
;; (s/assert ::us/uuid media-object-id)
;; (let [filename (fs/name localpath)
;; extension (second (fs/split-ext filename))
;; file (io/as-file localpath)
;; mtype (case extension
;; ".jpg" "image/jpeg"
;; ".png" "image/png"
;; ".webp" "image/webp"
;; ".svg" "image/svg+xml")]
;; (log/info "Creating image" filename media-object-id)
;; (media/create-media-object conn {:content {:tempfile localpath
;; :filename filename
;; :content-type mtype
;; :size (.length file)}
;; :id media-object-id
;; :file-id file-id
;; :name filename
;; :is-local false})))
(defn- media-object-exists?
[conn id]
(s/assert ::us/uuid id)
(let [row (db/get-by-id conn :media-object id)]
(if row true false)))
;; (defn- media-object-exists?
;; [conn id]
;; (s/assert ::us/uuid id)
;; (let [row (db/get-by-id conn :media-object id)]
;; (if row true false)))
(defn- import-media-object-if-not-exists
[conn file-id fpath]
(s/assert ::us/uuid file-id)
(s/assert fs/path? fpath)
(let [media-object-id (uuid/namespaced +graphics-uuid-ns+ (str file-id (fs/name fpath)))]
(when-not (media-object-exists? conn media-object-id)
(create-media-object conn file-id media-object-id fpath))
media-object-id))
;; (defn- import-media-object-if-not-exists
;; [conn file-id fpath]
;; (s/assert ::us/uuid file-id)
;; (s/assert fs/path? fpath)
;; (let [media-object-id (uuid/namespaced +graphics-uuid-ns+ (str file-id (fs/name fpath)))]
;; (when-not (media-object-exists? conn media-object-id)
;; (create-media-object conn file-id media-object-id fpath))
;; media-object-id))
(defn- import-graphics
[conn file-id {:keys [path regex]}]
(run! (fn [fpath]
(when (re-matches regex (str fpath))
(import-media-object-if-not-exists conn file-id fpath)))
(->> (fs/list-dir path)
(filter fs/regular-file?))))
;; (defn- import-graphics
;; [conn file-id {:keys [path regex]}]
;; (run! (fn [fpath]
;; (when (re-matches regex (str fpath))
;; (import-media-object-if-not-exists conn file-id fpath)))
;; (->> (fs/list-dir path)
;; (filter fs/regular-file?))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Colors Importer
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ;; Colors Importer
;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn- create-color
[conn file-id name content]
(s/assert ::us/uuid file-id)
(s/assert ::us/color content)
(let [color-id (uuid/namespaced +colors-uuid-ns+ (str file-id content))]
(log/info "Creating color" color-id "-" name content)
(colors/create-color conn {:id color-id
:file-id file-id
:name name
:content content})
color-id))
;; (defn- create-color
;; [conn file-id name content]
;; (s/assert ::us/uuid file-id)
;; (s/assert ::us/color content)
;; (let [color-id (uuid/namespaced +colors-uuid-ns+ (str file-id content))]
;; (log/info "Creating color" color-id "-" name content)
;; (colors/create-color conn {:id color-id
;; :file-id file-id
;; :name name
;; :content content})
;; color-id))
(defn- import-colors
[conn file-id colors]
(db/delete! conn :color {:file-id file-id})
(run! (fn [[name content]]
(create-color conn file-id name content))
(partition-all 2 colors)))
;; (defn- import-colors
;; [conn file-id colors]
;; (db/delete! conn :color {:file-id file-id})
;; (run! (fn [[name content]]
;; (create-color conn file-id name content))
;; (partition-all 2 colors)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Library files Importer
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ;; Library files Importer
;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn- library-file-exists?
[conn id]
(s/assert ::us/uuid id)
(let [row (db/get-by-id conn :file id)]
(if row true false)))
;; (defn- library-file-exists?
;; [conn id]
;; (s/assert ::us/uuid id)
;; (let [row (db/get-by-id conn :file id)]
;; (if row true false)))
(defn- create-library-file-if-not-exists
[conn project-id {:keys [name]}]
(let [id (uuid/namespaced +colors-uuid-ns+ name)]
(when-not (library-file-exists? conn id)
(log/info "Creating library-file:" name)
(files/create-file conn {:id id
:profile-id uuid/zero
:project-id project-id
:name name
:is-shared true})
(files/create-page conn {:file-id id}))
id))
;; (defn- create-library-file-if-not-exists
;; [conn project-id {:keys [name]}]
;; (let [id (uuid/namespaced +colors-uuid-ns+ name)]
;; (when-not (library-file-exists? conn id)
;; (log/info "Creating library-file:" name)
;; (files/create-file conn {:id id
;; :profile-id uuid/zero
;; :project-id project-id
;; :name name
;; :is-shared true})
;; (files/create-page conn {:file-id id}))
;; id))
(defn- process-library
[conn basedir project-id {:keys [graphics colors] :as library}]
(us/verify ::import-library library)
(let [library-file-id (create-library-file-if-not-exists conn project-id library)]
(when graphics
(->> (assoc graphics :path (fs/join basedir (:path graphics)))
(import-graphics conn library-file-id)))
(when colors
(import-colors conn library-file-id colors))))
;; (defn- process-library
;; [conn basedir project-id {:keys [graphics colors] :as library}]
;; (us/verify ::import-library library)
;; (let [library-file-id (create-library-file-if-not-exists conn project-id library)]
;; (when graphics
;; (->> (assoc graphics :path (fs/join basedir (:path graphics)))
;; (import-graphics conn library-file-id)))
;; (when colors
;; (import-colors conn library-file-id colors))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Entry Point
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ;; Entry Point
;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn- project-exists?
[conn id]
(s/assert ::us/uuid id)
(let [row (db/get-by-id conn :project id)]
(if row true false)))
;; (defn- project-exists?
;; [conn id]
;; (s/assert ::us/uuid id)
;; (let [row (db/get-by-id conn :project id)]
;; (if row true false)))
(defn- create-project-if-not-exists
[conn {:keys [name] :as project}]
(let [id (uuid/namespaced +colors-uuid-ns+ name)]
(when-not (project-exists? conn id)
(log/info "Creating project" name)
(projects/create-project conn {:id id
:team-id uuid/zero
:name name
:default? false}))
id))
;; (defn- create-project-if-not-exists
;; [conn {:keys [name] :as project}]
;; (let [id (uuid/namespaced +colors-uuid-ns+ name)]
;; (when-not (project-exists? conn id)
;; (log/info "Creating project" name)
;; (projects/create-project conn {:id id
;; :team-id uuid/zero
;; :name name
;; :default? false}))
;; id))
(defn- validate-path
[path]
(let [path (if (symbol? path) (str path) path)]
(log/infof "Trying to load config from '%s'." path)
(when-not path
(log/error "No path is provided")
(exit! -1))
(when-not (fs/exists? path)
(log/error "Path does not exists.")
(exit! -1))
(when (fs/directory? path)
(log/error "The provided path is a directory.")
(exit! -1))
(fs/path path)))
;; (defn- validate-path
;; [path]
;; (let [path (if (symbol? path) (str path) path)]
;; (log/infof "Trying to load config from '%s'." path)
;; (when-not path
;; (log/error "No path is provided")
;; (exit! -1))
;; (when-not (fs/exists? path)
;; (log/error "Path does not exists.")
;; (exit! -1))
;; (when (fs/directory? path)
;; (log/error "The provided path is a directory.")
;; (exit! -1))
;; (fs/path path)))
(defn- read-file
[path]
(let [reader (PushbackReader. (io/reader path))]
[(fs/parent path)
(read reader)]))
;; (defn- read-file
;; [path]
;; (let [reader (PushbackReader. (io/reader path))]
;; [(fs/parent path)
;; (read reader)]))
(defn run*
[path]
(let [[basedir libraries] (read-file path)]
(db/with-atomic [conn db/pool]
(let [project-id (create-project-if-not-exists conn {:name "System libraries"})]
(run! #(process-library conn basedir project-id %) libraries)))))
;; (defn run*
;; [path]
;; (let [[basedir libraries] (read-file path)]
;; (db/with-atomic [conn db/pool]
;; (let [project-id (create-project-if-not-exists conn {:name "System libraries"})]
;; (run! #(process-library conn basedir project-id %) libraries)))))
(defn run
[{:keys [path] :as params}]
(log/infof "Starting media loader.")
(let [path (validate-path path)]
;; (defn run
;; [{:keys [path] :as params}]
;; (log/infof "Starting media loader.")
;; (let [path (validate-path path)]
(try
(-> (mount/only #{#'app.config/config
#'app.db/pool
#'app.migrations/migrations
#'app.media/semaphore
#'app.media-storage/media-storage})
(mount/start))
(run* path)
(catch Exception e
(log/errorf e "Unhandled exception."))
(finally
(mount/stop)))))
;; (try
;; (-> (mount/only #{#'app.config/config
;; #'app.db/pool
;; #'app.migrations/migrations
;; #'app.media/semaphore
;; #'app.media-storage/media-storage})
;; (mount/start))
;; (run* path)
;; (catch Exception e
;; (log/errorf e "Unhandled exception."))
;; (finally
;; (mount/stop)))))

10
backend/src/app/migrations.clj
View file

@ -12,6 +12,7 @@
[mount.core :as mount :refer [defstate]]
[app.db :as db]
[app.config :as cfg]
[app.migrations.migration-0023 :as mg0023]
[app.util.migrations :as mg]))
(def +migrations+
@ -100,6 +101,15 @@
{:desc "Improve http session tables"
:name "0021-http-session-improvements"
:fn (mg/resource "migrations/0021-http-session-improvements.sql")}
{:desc "Refactor pages and files"
:name "0022-page-file-refactor"
:fn (mg/resource "migrations/0022-page-file-refactor.sql")}
{:desc "Adapt old pages and files to new format"
:name "0023-adapt-old-pages-and-files"
:fn mg0023/migrate}
]})
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

64
backend/src/app/migrations/migration_0023.clj Normal file
View file

@ -0,0 +1,64 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
(ns app.migrations.migration-0023
(:require
[app.db :as db]
[app.util.blob :as blob]))
(defn decode-row
[{:keys [data] :as row}]
(when row
(cond-> row
data (assoc :data (blob/decode data)))))
(defn retrieve-files
[conn]
(->> (db/exec! conn ["select * from file;"])
(map decode-row)))
(defn retrieve-pages
[conn file-id]
(->> (db/query conn :page {:file-id file-id})
(map decode-row)
(sort-by :ordering)))
(def empty-file-data
{:version 1
:pages []
:pages-index {}})
(defn pages->data
[pages]
(reduce (fn [acc {:keys [id data name] :as page}]
(let [data (-> data
(dissoc :version)
(assoc :id id :name name))]
(-> acc
(update :pages (fnil conj []) id)
(update :pages-index assoc id data))))
empty-file-data
pages))
(defn migrate-file
[conn {:keys [id] :as file}]
(let [pages (retrieve-pages conn (:id file))
data (pages->data pages)]
(db/update! conn :file
{:data (blob/encode data)}
{:id id})))
(defn migrate
[conn]
(let [files (retrieve-files conn)]
(doseq [file files]
(when (nil? (:data file))
(migrate-file conn file)))
(db/exec-one! conn ["drop table page cascade;"])))

4
backend/src/app/services/init.clj
View file

@ -5,7 +5,7 @@
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2019-2020 Andrey Antukh <niwi@niwi.nz>
;; Copyright (c) 2020 UXBOX Labs SL
(ns app.services.init
"A initialization of services."
@ -18,7 +18,6 @@
(require 'app.services.queries.colors)
(require 'app.services.queries.projects)
(require 'app.services.queries.files)
(require 'app.services.queries.pages)
(require 'app.services.queries.profile)
(require 'app.services.queries.recent-files)
(require 'app.services.queries.viewer))
@ -30,7 +29,6 @@
(require 'app.services.mutations.colors)
(require 'app.services.mutations.projects)
(require 'app.services.mutations.files)
(require 'app.services.mutations.pages)
(require 'app.services.mutations.profile))
(defstate query-services

2
backend/src/app/services/mutations/colors.clj
View file

@ -5,7 +5,7 @@
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 Andrey Antukh <niwi@niwi.nz>
;; Copyright (c) 2020 UXBOX Labs SL
(ns app.services.mutations.colors
(:require

2
backend/src/app/services/mutations/demo.clj
View file

@ -5,7 +5,7 @@
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2016-2020 Andrey Antukh <niwi@niwi.nz>
;; Copyright (c) 2020 UXBOX Labs SL
(ns app.services.mutations.demo
"A demo specific mutations."

118
backend/src/app/services/mutations/files.clj
View file

@ -5,7 +5,7 @@
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2019-2020 Andrey Antukh <niwi@niwi.nz>
;; Copyright (c) 2020 UXBOX Labs SL
(ns app.services.mutations.files
(:require
@ -14,16 +14,19 @@
[promesa.core :as p]
[app.common.exceptions :as ex]
[app.common.pages :as cp]
[app.common.pages-migrations :as pmg]
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.config :as cfg]
[app.db :as db]
[app.redis :as redis]
[app.services.mutations :as sm]
[app.services.mutations.projects :as proj]
[app.services.queries.files :as files]
[app.tasks :as tasks]
[app.util.blob :as blob]
[app.util.storage :as ust]
[app.util.transit :as t]
[app.util.time :as dt]))
;; --- Helpers & Specs
@ -37,7 +40,6 @@
;; --- Mutation: Create File
(declare create-file)
(declare create-page)
(s/def ::is-shared ::us/boolean)
(s/def ::create-file
@ -47,9 +49,7 @@
(sm/defmutation ::create-file
[{:keys [profile-id project-id] :as params}]
(db/with-atomic [conn db/pool]
(let [file (create-file conn params)
page (create-page conn (assoc params :file-id (:id file)))]
(assoc file :pages [(:id page)]))))
(create-file conn params)))
(defn- create-file-profile
[conn {:keys [profile-id file-id] :as params}]
@ -64,25 +64,17 @@
[conn {:keys [id profile-id name project-id is-shared]
:or {is-shared false}
:as params}]
(let [id (or id (uuid/next))
(let [id (or id (uuid/next))
data (cp/make-file-data)
file (db/insert! conn :file
{:id id
:project-id project-id
:name name
:is-shared is-shared})]
:is-shared is-shared
:data (blob/encode data)})]
(->> (assoc params :file-id id)
(create-file-profile conn))
file))
(defn create-page
[conn {:keys [file-id] :as params}]
(let [id (uuid/next)]
(db/insert! conn :page
{:id id
:file-id file-id
:name "Page 1"
:ordering 1
:data (blob/encode cp/default-page-data)})))
(assoc file :data data)))
;; --- Mutation: Rename File
@ -195,3 +187,93 @@
{:file-id file-id
:library-file-id library-id}))
;; A generic, Changes based (granular) file update method.
(s/def ::changes
(s/coll-of map? :kind vector?))
(s/def ::session-id ::us/uuid)
(s/def ::revn ::us/integer)
(s/def ::update-file
(s/keys :req-un [::id ::session-id ::profile-id ::revn ::changes]))
(declare update-file)
(declare retrieve-lagged-changes)
(declare insert-change)
(sm/defmutation ::update-file
[{:keys [id profile-id] :as params}]
(db/with-atomic [conn db/pool]
(let [{:keys [id] :as file} (db/get-by-id conn :file id {:for-update true})]
(files/check-edition-permissions! conn profile-id id)
(update-file conn file params))))
(defn- update-file
[conn file params]
(when (> (:revn params)
(:revn file))
(ex/raise :type :validation
:code :revn-conflict
:hint "The incoming revision number is greater that stored version."
:context {:incoming-revn (:revn params)
:stored-revn (:revn file)}))
(let [sid (:session-id params)
changes (:changes params)
file (-> file
(update :data blob/decode)
(update :data pmg/migrate-data)
(update :data cp/process-changes changes)
(update :data blob/encode)
(update :revn inc)
(assoc :changes (blob/encode changes)
:session-id sid))
chng (insert-change conn file)
msg {:type :file-change
:profile-id (:profile-id params)
:file-id (:id file)
:session-id sid
:revn (:revn file)
:changes changes}]
@(redis/run! :publish {:channel (str (:id file))
:message (t/encode-str msg)})
(db/update! conn :file
{:revn (:revn file)
:data (:data file)}
{:id (:id file)})
(retrieve-lagged-changes conn chng params)))
(defn- insert-change
[conn {:keys [revn data changes session-id] :as file}]
(let [id (uuid/next)
file-id (:id file)]
(db/insert! conn :file-change
{:id id
:session-id session-id
:file-id file-id
:revn revn
:data data
:changes changes})))
(def ^:private
sql:lagged-changes
"select s.id, s.revn, s.file_id,
s.session_id, s.changes
from file_change as s
where s.file_id = ?
and s.revn > ?
order by s.created_at asc")
(defn- retrieve-lagged-changes
[conn snapshot params]
(->> (db/exec! conn [sql:lagged-changes (:id params) (:revn params)])
(mapv files/decode-row)))

255
backend/src/app/services/mutations/pages.clj
View file

@ -1,255 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2019-2020 Andrey Antukh <niwi@niwi.nz>
(ns app.services.mutations.pages
(:require
[clojure.spec.alpha :as s]
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.pages :as cp]
[app.common.pages-migrations :as pmg]
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.config :as cfg]
[app.db :as db]
[app.services.mutations :as sm]
[app.services.queries.files :as files]
[app.services.queries.pages :refer [decode-row]]
[app.tasks :as tasks]
[app.redis :as redis]
[app.util.blob :as blob]
[app.util.time :as dt]
[app.util.transit :as t]))
;; --- Helpers & Specs
(s/def ::id ::us/uuid)
(s/def ::name ::us/string)
(s/def ::data ::cp/data)
(s/def ::profile-id ::us/uuid)
(s/def ::project-id ::us/uuid)
(s/def ::ordering ::us/number)
(s/def ::file-id ::us/uuid)
;; --- Mutation: Create Page
(declare create-page)
(s/def ::create-page
(s/keys :req-un [::profile-id ::file-id ::name ::ordering ::data]
:opt-un [::id]))
(sm/defmutation ::create-page
[{:keys [profile-id file-id] :as params}]
(db/with-atomic [conn db/pool]
(files/check-edition-permissions! conn profile-id file-id)
(create-page conn params)))
(defn- create-page
[conn {:keys [id file-id name ordering data] :as params}]
(let [id (or id (uuid/next))
data (blob/encode data)]
(-> (db/insert! conn :page
{:id id
:file-id file-id
:name name
:ordering ordering
:data data})
(decode-row))))
;; --- Mutation: Rename Page
(declare rename-page)
(declare select-page-for-update)
(s/def ::rename-page
(s/keys :req-un [::id ::name ::profile-id]))
(sm/defmutation ::rename-page
[{:keys [id name profile-id]}]
(db/with-atomic [conn db/pool]
(let [page (select-page-for-update conn id)]
(files/check-edition-permissions! conn profile-id (:file-id page))
(rename-page conn (assoc page :name name)))))
(defn- select-page-for-update
[conn id]
(db/get-by-id conn :page id {:for-update true}))
(defn- rename-page
[conn {:keys [id name] :as params}]
(db/update! conn :page
{:name name}
{:id id}))
;; --- Mutation: Sort Pages
(s/def ::page-ids (s/every ::us/uuid :kind vector?))
(s/def ::reorder-pages
(s/keys :req-un [::profile-id ::file-id ::page-ids]))
(declare update-page-ordering)
(sm/defmutation ::reorder-pages
[{:keys [profile-id file-id page-ids]}]
(db/with-atomic [conn db/pool]
(run! #(update-page-ordering conn file-id %)
(d/enumerate page-ids))
nil))
(defn- update-page-ordering
[conn file-id [ordering page-id]]
(db/update! conn :page
{:ordering ordering}
{:file-id file-id
:id page-id}))
;; --- Mutation: Generate Share Token
(declare assign-page-share-token)
(s/def ::generate-page-share-token
(s/keys :req-un [::id]))
(sm/defmutation ::generate-page-share-token
[{:keys [id] :as params}]
(let [token (-> (sodi.prng/random-bytes 16)
(sodi.util/bytes->b64s))]
(db/with-atomic [conn db/pool]
(db/update! conn :page
{:share-token token}
{:id id}))))
;; --- Mutation: Clear Share Token
(s/def ::clear-page-share-token
(s/keys :req-un [::id]))
(sm/defmutation ::clear-page-share-token
[{:keys [id] :as params}]
(db/with-atomic [conn db/pool]
(db/update! conn :page
{:share-token nil}
{:id id})))
;; --- Mutation: Update Page
;; A generic, Changes based (granular) page update method.
(s/def ::changes
(s/coll-of map? :kind vector?))
(s/def ::session-id ::us/uuid)
(s/def ::revn ::us/integer)
(s/def ::update-page
(s/keys :req-un [::id ::session-id ::profile-id ::revn ::changes]))
(declare update-page)
(declare retrieve-lagged-changes)
(declare insert-page-change!)
(sm/defmutation ::update-page
[{:keys [id profile-id] :as params}]
(db/with-atomic [conn db/pool]
(let [{:keys [file-id] :as page} (select-page-for-update conn id)]
(files/check-edition-permissions! conn profile-id file-id)
(update-page conn page params))))
(defn- update-page
[conn page params]
(when (> (:revn params)
(:revn page))
(ex/raise :type :validation
:code :revn-conflict
:hint "The incoming revision number is greater that stored version."
:context {:incoming-revn (:revn params)
:stored-revn (:revn page)}))
(let [sid (:session-id params)
changes (:changes params)
page (-> page
(update :data blob/decode)
(update :data pmg/migrate-data)
(update :data cp/process-changes changes)
(update :data blob/encode)
(update :revn inc)
(assoc :changes (blob/encode changes)
:session-id sid))
chng (insert-page-change! conn page)
msg {:type :page-change
:profile-id (:profile-id params)
:page-id (:id page)
:session-id sid
:revn (:revn page)
:changes changes}]
@(redis/run! :publish {:channel (str (:file-id page))
:message (t/encode-str msg)})
(db/update! conn :page
{:revn (:revn page)
:data (:data page)}
{:id (:id page)})
(retrieve-lagged-changes conn chng params)))
(defn- insert-page-change!
[conn {:keys [revn data changes session-id] :as page}]
(let [id (uuid/next)
page-id (:id page)]
(db/insert! conn :page-change
{:id id
:session-id session-id
:page-id page-id
:revn revn
:data data
:changes changes})))
(def ^:private
sql:lagged-changes
"select s.id, s.revn, s.page_id,
s.session_id, s.changes
from page_change as s
where s.page_id = ?
and s.revn > ?
order by s.created_at asc")
(defn- retrieve-lagged-changes
[conn snapshot params]
(->> (db/exec! conn [sql:lagged-changes (:id params) (:revn params)])
(mapv decode-row)))
;; --- Mutation: Delete Page
(declare mark-page-deleted)
(s/def ::delete-page
(s/keys :req-un [::profile-id ::id]))
(sm/defmutation ::delete-page
[{:keys [id profile-id]}]
(db/with-atomic [conn db/pool]
(let [page (select-page-for-update conn id)]
(files/check-edition-permissions! conn profile-id (:file-id page))
;; Schedule object deletion
(tasks/submit! conn {:name "delete-object"
:delay cfg/default-deletion-delay
:props {:id id :type :page}})
(db/update! conn :page
{:deleted-at (dt/now)}
{:id id})
nil)))

22
backend/src/app/services/mutations/teams.clj
View file

@ -5,7 +5,7 @@
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 Andrey Antukh <niwi@niwi.nz>
;; Copyright (c) 2020 UXBOX Labs SL
(ns app.services.mutations.teams
(:require
@ -57,23 +57,3 @@
:is-owner true
:is-admin true
:can-edit true}))
;; --- Mutation: Team Edition Permissions
(def ^:private sql:team-permissions
"select tpr.is_owner,
tpr.is_admin,
tpr.can_edit
from team_profile_rel as tpr
where tpr.profile_id = ?
and tpr.team_id = ?")
(defn check-edition-permissions!
[conn profile-id team-id]
(let [row (db/exec-one! conn [sql:team-permissions profile-id team-id])]
(when-not (or (= team-id uuid/zero)
(:can-edit row)
(:is-admin row)
(:is-owner row))
(ex/raise :type :validation
:code :not-authorized))))

65
backend/src/app/services/mutations/viewer.clj Normal file
View file

@ -0,0 +1,65 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2019-2020 Andrey Antukh <niwi@niwi.nz>
(ns app.services.mutations.viewer
(:require
[app.common.exceptions :as ex]
[app.common.pages :as cp]
[app.common.pages-migrations :as pmg]
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.config :as cfg]
[app.db :as db]
[app.redis :as redis]
[app.services.mutations :as sm]
[app.services.mutations.projects :as proj]
[app.services.queries.files :as files]
[app.tasks :as tasks]
[app.util.blob :as blob]
[app.util.storage :as ust]
[app.util.time :as dt]
[app.util.transit :as t]
[clojure.spec.alpha :as s]
[datoteka.core :as fs]
[promesa.core :as p]
[sodi.prng]
[sodi.util]))
(s/def ::profile-id ::us/uuid)
(s/def ::file-id ::us/uuid)
(s/def ::page-id ::us/uuid)
(s/def ::create-file-share-token
(s/keys :req-un [::profile-id ::file-id ::page-id]))
(sm/defmutation ::create-file-share-token
[{:keys [profile-id file-id page-id] :as params}]
(db/with-atomic [conn db/pool]
(files/check-edition-permissions! conn profile-id file-id)
(let [token (-> (sodi.prng/random-bytes 16)
(sodi.util/bytes->b64s))]
(db/insert! conn :file-share-token
{:file-id file-id
:page-id page-id
:token token})
{:token token})))
(s/def ::token ::us/not-empty-string)
(s/def ::delete-file-share-token
(s/keys :req-un [::profile-id ::file-id ::token]))
(sm/defmutation ::delete-file-share-token
[{:keys [profile-id file-id token]}]
(db/with-atomic [conn db/pool]
(files/check-edition-permissions! conn profile-id file-id)
(db/delete! conn :file-share-token
{:file-id file-id
:token token})
nil))

76
backend/src/app/services/notifications.clj
View file

@ -10,17 +10,17 @@
(ns app.services.notifications
"A websocket based notifications mechanism."
(:require
[clojure.core.async :as a :refer [>! <!]]
[clojure.tools.logging :as log]
[promesa.core :as p]
[ring.adapter.jetty9 :as jetty]
[app.common.exceptions :as ex]
[app.common.uuid :as uuid]
[app.db :as db]
[app.redis :as redis]
[app.metrics :as mtx]
[app.redis :as redis]
[app.util.time :as dt]
[app.util.transit :as t]))
[app.util.transit :as t]
[clojure.core.async :as a :refer [>! <!]]
[clojure.tools.logging :as log]
[promesa.core :as p]
[ring.adapter.jetty9 :as jetty]))
(defmacro go-try
[& body]
@ -44,8 +44,6 @@
(catch Throwable e#
e#))))
;; --- Redis Interactions
(defn- publish
[channel message]
(go-try
@ -187,14 +185,6 @@
(defrecord WebSocket [conn in out sub])
(defn- start-rcv-loop!
[{:keys [conn out] :as ws}]
(a/go-loop []
(let [val (a/<! out)]
(when-not (nil? val)
(jetty/send! conn (t/encode-str val))
(recur)))))
(defonce metrics-active-connections
(mtx/gauge {:id "notificatons__active_connections"
:help "Active connections to the notifications service."}))
@ -207,30 +197,42 @@
[{:keys [file-id profile-id] :as params}]
(let [in (a/chan 32)
out (a/chan 32)]
{:on-connect (fn [conn]
(metrics-active-connections :inc)
(let [xf (map t/decode-str)
sub (redis/subscribe (str file-id) xf)
ws (WebSocket. conn in out sub nil params)]
(start-rcv-loop! ws)
(a/go
(a/<! (on-subscribed ws))
(a/close! sub))))
{:on-connect
(fn [conn]
(metrics-active-connections :inc)
(let [xf (map t/decode-str)
sub (redis/subscribe (str file-id) xf)
ws (WebSocket. conn in out sub nil params)]
:on-error (fn [conn e]
(a/close! out)
(a/close! in))
;; RCV LOOP
(a/go-loop []
(let [val (a/<! out)]
(when-not (nil? val)
(jetty/send! conn (t/encode-str val))
(recur))))
:on-close (fn [conn status-code reason]
(metrics-active-connections :dec)
(a/close! out)
(a/close! in))
(a/go
(a/<! (on-subscribed ws))
(a/close! sub))))
:on-text (fn [ws message]
(metrics-message-counter :inc)
(let [message (t/decode-str message)]
(a/>!! in message)))
:on-error
(fn [conn e]
(a/close! out)
(a/close! in))
:on-bytes (constantly nil)}))
:on-close
(fn [conn status-code reason]
(metrics-active-connections :dec)
(a/close! out)
(a/close! in))
:on-text
(fn [ws message]
(metrics-message-counter :inc)
(let [message (t/decode-str message)]
(a/>!! in message)))
:on-bytes
(constantly nil)}))

139
backend/src/app/services/queries/files.clj
View file

@ -11,14 +11,17 @@
(:require
[clojure.spec.alpha :as s]
[promesa.core :as p]
[app.common.pages-migrations :as pmg]
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.db :as db]
[app.media :as media]
[app.services.queries :as sq]
[app.services.queries.projects :as projects]
[app.util.blob :as blob]))
(declare decode-row)
(declare decode-row-xf)
;; --- Helpers & Specs
@ -32,6 +35,8 @@
;; --- Query: Files search
;; TODO: this query need to a good refactor
(def ^:private sql:search-files
"with projects as (
select p.*
@ -82,58 +87,16 @@
profile-id team-id
profile-id team-id
search-term])]
(mapv decode-row rows)))
(into [] decode-row-xf rows)))
;; --- Query: Project Files
(def ^:private sql:files
"with projects as (
select p.*
from project as p
inner join team_profile_rel as tpr on (tpr.team_id = p.team_id)
where tpr.profile_id = ?
and p.deleted_at is null
and (tpr.is_admin = true or
tpr.is_owner = true or
tpr.can_edit = true)
union
select p.*
from project as p
inner join project_profile_rel as ppr on (ppr.project_id = p.id)
where ppr.profile_id = ?
and p.deleted_at is null
and (ppr.is_admin = true or
ppr.is_owner = true or
ppr.can_edit = true)
union
select p.*
from project as p
where p.team_id = uuid_nil()
and p.deleted_at is null
)
select distinct
f.*,
array_agg(pg.id) over pages_w as pages,
first_value(pg.data) over pages_w as data
"select f.*
from file as f
left join page as pg on (f.id = pg.file_id)
where f.project_id = ?
and (exists (select *
from file_profile_rel as fp_r
where fp_r.profile_id = ?
and fp_r.file_id = f.id
and (fp_r.is_admin = true or
fp_r.is_owner = true or
fp_r.can_edit = true))
or exists (select *
from projects as p
where p.id = f.project_id))
and f.deleted_at is null
and pg.deleted_at is null
window pages_w as (partition by f.id order by pg.ordering
range between unbounded preceding
and unbounded following)
order by f.modified_at desc")
(s/def ::project-id ::us/uuid)
@ -142,10 +105,10 @@
(sq/defquery ::files
[{:keys [profile-id project-id] :as params}]
(->> (db/exec! db/pool [sql:files
profile-id profile-id
project-id profile-id])
(mapv decode-row)))
(with-open [conn (db/open)]
(let [project (db/get-by-id conn :project project-id)]
(projects/check-edition-permissions! conn profile-id project)
(into [] decode-row-xf (db/exec! conn [sql:files project-id])))))
;; --- Query: File Permissions
@ -173,12 +136,7 @@
from project_profile_rel as ppr
inner join file as f on (f.project_id = ppr.project_id)
where f.id = ?
and ppr.profile_id = ?
union all
select true, true, true
from file as f
inner join project as p on (f.project_id = p.id)
and p.team_id = uuid_nil();")
and ppr.profile_id = ?")
(defn check-edition-permissions!
[conn profile-id file-id]
@ -198,24 +156,11 @@
;; --- Query: File (By ID)
(def ^:private sql:file
"select f.*,
array_agg(pg.id) over pages_w as pages
from file as f
left join page as pg on (f.id = pg.file_id)
where f.id = ?
and f.deleted_at is null
and pg.deleted_at is null
window pages_w as (partition by f.id order by pg.ordering
range between unbounded preceding
and unbounded following)")
(defn retrieve-file
[conn id]
(let [row (db/exec-one! conn [sql:file id])]
(when-not row
(ex/raise :type :not-found))
(decode-row row)))
(let [file (db/get-by-id conn :file id)]
(-> (decode-row file)
(pmg/migrate-file))))
(s/def ::file
(s/keys :req-un [::profile-id ::id]))
@ -226,6 +171,15 @@
(check-edition-permissions! conn profile-id id)
(retrieve-file conn id)))
(s/def ::page
(s/keys :req-un [::profile-id ::id ::file-id]))
(sq/defquery ::page
[{:keys [profile-id file-id id]}]
(db/with-atomic [conn db/pool]
(check-edition-permissions! conn profile-id file-id)
(let [file (retrieve-file conn file-id)]
(get-in file [:data :pages-index id]))))
;; --- Query: File users
@ -256,14 +210,12 @@
(check-edition-permissions! conn profile-id id)
(retrieve-file-users conn id)))
;; --- Query: Shared Library Files
;; TODO: remove the counts, because they are no longer needed.
(def ^:private sql:shared-files
"select distinct
f.*,
array_agg(pg.id) over pages_w as pages,
first_value(pg.data) over pages_w as data,
"select f.*,
(select count(*) from color as c
where c.file_id = f.id
and c.deleted_at is null) as colors_count,
@ -272,16 +224,11 @@
and m.is_local = false
and m.deleted_at is null) as graphics_count
from file as f
left join page as pg on (f.id = pg.file_id)
inner join project as p on (p.id = f.project_id)
where f.is_shared = true
and f.deleted_at is null
and pg.deleted_at is null
and p.deleted_at is null
and p.team_id = ?
window pages_w as (partition by f.id order by pg.ordering
range between unbounded preceding
and unbounded following)
order by f.modified_at desc")
(s/def ::shared-files
@ -289,30 +236,21 @@
(sq/defquery ::shared-files
[{:keys [profile-id team-id] :as params}]
(->> (db/exec! db/pool [sql:shared-files team-id])
(mapv decode-row)))
(into [] decode-row-xf (db/exec! db/pool [sql:shared-files team-id])))
;; --- Query: File Libraries used by a File
(def ^:private sql:file-libraries
"select fl.*,
array_agg(pg.id) over pages_w as pages,
first_value(pg.data) over pages_w as data
"select fl.*
from file as fl
left join page as pg on (fl.id = pg.file_id)
inner join file_library_rel as flr on (flr.library_file_id = fl.id)
where flr.file_id = ?
and fl.deleted_at is null
and pg.deleted_at is null
window pages_w as (partition by fl.id order by pg.ordering
range between unbounded preceding
and unbounded following)")
and fl.deleted_at is null")
(defn retrieve-file-libraries
[conn file-id]
(->> (db/exec! conn [sql:file-libraries file-id])
(mapv decode-row)))
(into [] decode-row-xf (db/exec! conn [sql:file-libraries file-id])))
(s/def ::file-libraries
(s/keys :req-un [::profile-id ::file-id]))
@ -326,6 +264,8 @@
;; --- Query: Single File Library
;; TODO: this looks like is duplicate of `::file`
(def ^:private sql:file-library
"select fl.*
from file as fl
@ -333,10 +273,10 @@
(defn retrieve-file-library
[conn file-id]
(let [row (db/exec-one! conn [sql:file-library file-id])]
(when-not row
(let [rows (db/exec! conn [sql:file-library file-id])]
(when-not (seq rows)
(ex/raise :type :not-found))
row))
(first (sequence decode-row-xf rows))))
(s/def ::file-library
(s/keys :req-un [::profile-id ::file-id]))
@ -351,8 +291,13 @@
;; --- Helpers
(defn decode-row
[{:keys [pages data] :as row}]
[{:keys [pages data changes] :as row}]
(when row
(cond-> row
changes (assoc :changes (blob/decode changes))
data (assoc :data (blob/decode data))
pages (assoc :pages (vec (.getArray pages))))))
(def decode-row-xf
(comp (map decode-row)
(map pmg/migrate-file)))

122
backend/src/app/services/queries/pages.clj
View file

@ -1,122 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2019-2020 Andrey Antukh <niwi@niwi.nz>
(ns app.services.queries.pages
(:require
[clojure.spec.alpha :as s]
[promesa.core :as p]
[app.common.spec :as us]
[app.common.exceptions :as ex]
[app.common.pages-migrations :as pmg]
[app.db :as db]
[app.services.queries :as sq]
[app.services.queries.files :as files]
[app.util.blob :as blob]))
;; --- Helpers & Specs
(declare decode-row)
(s/def ::id ::us/uuid)
(s/def ::profile-id ::us/uuid)
(s/def ::project-id ::us/uuid)
(s/def ::file-id ::us/uuid)
;; --- Query: Pages (By File ID)
(declare retrieve-pages)
(s/def ::pages
(s/keys :req-un [::profile-id ::file-id]))
(sq/defquery ::pages
[{:keys [profile-id file-id] :as params}]
(db/with-atomic [conn db/pool]
(files/check-edition-permissions! conn profile-id file-id)
(->> (retrieve-pages conn params)
(mapv #(update % :data pmg/migrate-data)))))
(def ^:private sql:pages
"select p.*
from page as p
where p.file_id = ?
and p.deleted_at is null
order by p.created_at asc")
(defn- retrieve-pages
[conn {:keys [profile-id file-id] :as params}]
(->> (db/exec! conn [sql:pages file-id])
(mapv decode-row)))
;; --- Query: Single Page (By ID)
(declare retrieve-page)
(s/def ::page
(s/keys :req-un [::profile-id ::id]))
(sq/defquery ::page
[{:keys [profile-id id] :as params}]
(with-open [conn (db/open)]
(let [page (retrieve-page conn id)]
(files/check-edition-permissions! conn profile-id (:file-id page))
(-> page
(update :data pmg/migrate-data)))))
(def ^:private sql:page
"select p.* from page as p where id=?")
(defn retrieve-page
[conn id]
(let [row (db/exec-one! conn [sql:page id])]
(when-not row
(ex/raise :type :not-found))
(decode-row row)))
;; --- Query: Page Changes
(def ^:private
sql:page-changes
"select pc.id,
pc.created_at,
pc.changes,
pc.revn
from page_change as pc
where pc.page_id=?
order by pc.revn asc
limit ?
offset ?")
(s/def ::skip ::us/integer)
(s/def ::limit ::us/integer)
(s/def ::page-changes
(s/keys :req-un [::profile-id ::id ::skip ::limit]))
(defn retrieve-page-changes
[conn id skip limit]
(->> (db/exec! conn [sql:page-changes id limit skip])
(mapv decode-row)))
(sq/defquery ::page-changes
[{:keys [profile-id id skip limit]}]
(when *assert*
(-> (db/exec! db/pool [sql:page-changes id limit skip])
(mapv decode-row))))
;; --- Helpers
(defn decode-row
[{:keys [data metadata changes] :as row}]
(when row
(cond-> row
data (assoc :data (blob/decode data))
changes (assoc :changes (blob/decode changes)))))

5
backend/src/app/services/queries/projects.clj
View file

@ -76,8 +76,9 @@
where f.project_id = p.id
and deleted_at is null)
from project as p
where team_id = ?
order by modified_at desc")
where p.team_id = ?
and p.deleted_at is null
order by p.modified_at desc")
(defn retrieve-projects
[conn team-id]

15
backend/src/app/services/queries/recent_files.clj
View file

@ -16,21 +16,13 @@
[app.services.queries :as sq]
[app.services.queries.teams :as teams]
[app.services.queries.projects :as projects :refer [retrieve-projects]]
[app.services.queries.files :refer [decode-row]]))
[app.services.queries.files :refer [decode-row-xf]]))
(def sql:project-recent-files
"select distinct
f.*,
array_agg(pg.id) over pages_w as pages,
first_value(pg.data) over pages_w as data
"select f.*
from file as f
left join page as pg on (f.id = pg.file_id)
where f.project_id = ?
and f.deleted_at is null
and pg.deleted_at is null
window pages_w as (partition by f.id order by pg.ordering
range between unbounded preceding
and unbounded following)
order by f.modified_at desc
limit 5")
@ -38,8 +30,7 @@
[conn profile-id project]
(let [project-id (:id project)]
(projects/check-edition-permissions! conn profile-id project)
(->> (db/exec! conn [sql:project-recent-files project-id])
(map decode-row))))
(into [] decode-row-xf (db/exec! conn [sql:project-recent-files project-id]))))
(s/def ::team-id ::us/uuid)
(s/def ::profile-id ::us/uuid)

66
backend/src/app/services/queries/viewer.clj
View file

@ -9,27 +9,18 @@
(ns app.services.queries.viewer
(:require
[clojure.spec.alpha :as s]
[promesa.core :as p]
[promesa.exec :as px]
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.db :as db]
[app.services.queries :as sq]
[app.services.queries.files :as files]
[app.services.queries.media :as media-queries]
[app.services.queries.pages :as pages]
[app.util.blob :as blob]
[app.util.data :as data]))
;; --- Helpers & Specs
(s/def ::id ::us/uuid)
(s/def ::page-id ::us/uuid)
[clojure.spec.alpha :as s]))
;; --- Query: Viewer Bundle (by Page ID)
(declare check-shared-token!)
(declare retrieve-shared-token)
(def ^:private
sql:project
"select p.id, p.name
@ -41,24 +32,45 @@
[conn id]
(db/exec-one! conn [sql:project id]))
(s/def ::id ::us/uuid)
(s/def ::file-id ::us/uuid)
(s/def ::page-id ::us/uuid)
(s/def ::share-token ::us/string)
(s/def ::viewer-bundle
(s/keys :req-un [::page-id]
(s/keys :req-un [::file-id ::page-id]
:opt-un [::profile-id ::share-token]))
(sq/defquery ::viewer-bundle
[{:keys [profile-id page-id share-token] :as params}]
[{:keys [profile-id file-id page-id share-token] :as params}]
(db/with-atomic [conn db/pool]
(let [page (pages/retrieve-page conn page-id)
file (files/retrieve-file conn (:file-id page))
images (media-queries/retrieve-media-objects conn (:file-id page) true)
project (retrieve-project conn (:project-id file))]
(let [file (files/retrieve-file conn file-id)
project (retrieve-project conn (:project-id file))
page (get-in file [:data :pages-index page-id])
bundle {:file (dissoc file :data)
:page (get-in file [:data :pages-index page-id])
:project project}]
(if (string? share-token)
(when (not= share-token (:share-token page))
(ex/raise :type :validation
:code :not-authorized))
(files/check-edition-permissions! conn profile-id (:file-id page)))
{:page page
:file file
:images images
:project project})))
(do
(check-shared-token! conn file-id page-id share-token)
(assoc bundle :share-token share-token))
(let [token (retrieve-shared-token conn file-id page-id)]
(files/check-edition-permissions! conn profile-id file-id)
(assoc bundle :share-token token))))))
(defn check-shared-token!
[conn file-id page-id token]
(let [sql "select exists(select 1 from file_share_token where file_id=? and page_id=? and token=?) as exists"]
(when-not (:exists (db/exec-one! conn [sql file-id page-id token]))
(ex/raise :type :validation
:code :not-authorized))))
(defn retrieve-shared-token
[conn file-id page-id]
(let [sql "select * from file_share_token where file_id=? and page_id=?"]
(db/exec-one! conn [sql file-id page-id])))