diff --git a/backend/src/app/db.clj b/backend/src/app/db.clj index 7ab340e21..8d7fbef4c 100644 --- a/backend/src/app/db.clj +++ b/backend/src/app/db.clj @@ -249,6 +249,12 @@ :code :unable-resolve-connection :hint "expected conn or system map")))) +(defn connection-map? + "Check if the provided value is a map like data structure that + contains a database connection." + [o] + (and (map? o) (connection? (::conn o)))) + (defn- get-connectable [o] (cond diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index cadac894f..663abb8c1 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -14,7 +14,7 @@ [app.common.files.changes-builder :as fcb] [app.common.files.helpers :as cfh] [app.common.files.libraries-helpers :as cflh] - [app.common.files.migrations :as pmg] + [app.common.files.migrations :as fmg] [app.common.files.shapes-helpers :as cfsh] [app.common.files.validate :as cfv] [app.common.geom.point :as gpt] @@ -32,6 +32,7 @@ [app.common.types.shape-tree :as ctst] [app.common.uuid :as uuid] [app.db :as db] + [app.features.fdata :as fdata] [app.http.sse :as sse] [app.media :as media] [app.rpc.commands.files :as files] @@ -40,7 +41,6 @@ [app.storage :as sto] [app.storage.tmp :as tmp] [app.util.blob :as blob] - [app.util.objects-map :as omap] [app.util.pointer-map :as pmap] [app.util.time :as dt] [buddy.core.codecs :as bc] @@ -703,7 +703,6 @@ (try (->> (d/zip media-group grid) (map (fn [[mobj position]] - (l/trc :hint "submit graphic processing" :file-id (str (:id fdata)) :id (str (:id mobj))) (sse/tap {:type :migration-progress :section :graphics :name (:name mobj)}) @@ -761,7 +760,7 @@ (create-media-grid fdata page-id (:id frame) grid assets) (gpt/add position (gpt/point 0 (+ height (* 2 grid-gap) frame-gap)))))))))) -(defn- migrate-file-data +(defn- migrate-fdata [fdata libs] (let [migrated? (dm/get-in fdata [:options :components-v2])] (if migrated? @@ -770,53 +769,58 @@ fdata (migrate-graphics fdata)] (update fdata :options assoc :components-v2 true))))) +(defn- process-fdata + [fdata id] + (-> fdata + (assoc :id id) + (fdata/process-pointers deref) + (fmg/migrate-data))) + (defn- process-file - [{:keys [id] :as file} & {:keys [validate? throw-on-validate?]}] - (let [conn (::db/conn *system*)] - (binding [pmap/*tracked* (atom {}) - pmap/*load-fn* (partial files/load-pointer conn id) - cfeat/*wrap-with-pointer-map-fn* - (if (contains? (:features file) "fdata/pointer-map") pmap/wrap identity) - cfeat/*wrap-with-objects-map-fn* - (if (contains? (:features file) "fdata/objectd-map") omap/wrap identity)] + [{:keys [::db/conn] :as system} id & {:keys [validate? throw-on-validate?]}] + (binding [pmap/*tracked* (pmap/create-tracked) + pmap/*load-fn* (partial fdata/load-pointer *system* id)] - (let [file (-> file - (update :data blob/decode) - (update :data assoc :id id) - (pmg/migrate-file)) + (let [file (binding [cfeat/*new* (atom #{})] + (-> (files/get-file system id :migrate? false) + (update :data process-fdata id) + (update :features into (deref cfeat/*new*)) + (update :features cfeat/migrate-legacy-features))) - libs (->> (files/get-file-libraries conn id) - (into [file] (map (fn [{:keys [id]}] - (binding [pmap/*load-fn* (partial files/load-pointer conn id)] - (-> (db/get conn :file {:id id}) - (files/decode-row) - (files/process-pointers deref) ; ensure all pointers resolved - (pmg/migrate-file)))))) - (d/index-by :id)) + libs (->> (files/get-file-libraries conn id) + (into [file] (map (fn [{:keys [id]}] + (binding [pmap/*load-fn* (partial fdata/load-pointer system id)] + (-> (files/get-file system id :migrate? false) + (update :data process-fdata id)))))) + (d/index-by :id)) - file (-> file - (update :data migrate-file-data libs) - (update :features conj "components/v2"))] + pmap? (contains? (:features file) "fdata/pointer-map") - (when (contains? (:features file) "fdata/pointer-map") - (files/persist-pointers! conn id)) + file (-> file + (update :data migrate-fdata libs) + (update :features conj "components/v2") + (cond-> pmap? (fdata/enable-pointer-map))) + ] - (db/update! conn :file - {:data (blob/encode (:data file)) - :features (db/create-array conn "text" (:features file)) - :revn (:revn file)} - {:id (:id file)}) + (when validate? + (if throw-on-validate? + (cfv/validate-file! file libs) + (doseq [error (cfv/validate-file file libs)] + (l/wrn :hint "migrate:file:validation-error" + :file-id (str (:id file)) + :file-name (:name file) + :error error)))) - (when validate? - (if throw-on-validate? - (cfv/validate-file! file libs) - (doseq [error (cfv/validate-file file libs)] - (l/wrn :hint "migrate:file:validation-error" - :file-id (str (:id file)) - :file-name (:name file) - :error error)))) + (db/update! conn :file + {:data (blob/encode (:data file)) + :features (db/create-array conn "text" (:features file)) + :revn (:revn file)} + {:id (:id file)}) - (dissoc file :data))))) + (when pmap? + (fdata/persist-pointers! system id)) + + (dissoc file :data)))) (defn migrate-file! [system file-id & {:keys [validate? throw-on-validate?]}] @@ -830,13 +834,12 @@ (let [system (update system ::sto/storage media/configure-assets-storage)] (db/tx-run! system - (fn [{:keys [::db/conn] :as system}] + (fn [system] (binding [*system* system] (fsnap/take-file-snapshot! system {:file-id file-id :label "migration/components-v2"}) - (-> (db/get conn :file {:id file-id}) - (update :features db/decode-pgarray #{}) - (process-file :validate? validate? - :throw-on-validate? throw-on-validate?)))))) + (process-file system file-id + :validate? validate? + :throw-on-validate? throw-on-validate?))))) (finally (let [elapsed (tpoint) diff --git a/backend/src/app/features/fdata.clj b/backend/src/app/features/fdata.clj index 1c8349396..832d3360d 100644 --- a/backend/src/app/features/fdata.clj +++ b/backend/src/app/features/fdata.clj @@ -7,42 +7,90 @@ (ns app.features.fdata "A `fdata/*` related feature migration helpers" (:require + [app.common.data :as d] + [app.common.exceptions :as ex] + [app.common.logging :as l] + [app.db :as db] + [app.util.blob :as blob] [app.util.objects-map :as omap] [app.util.pointer-map :as pmap])) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; OBJECTS-MAP +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + (defn enable-objects-map [file] - (-> file - (update :data (fn [data] - (-> data - (update :pages-index update-vals #(update % :objects omap/wrap)) - (update :components update-vals #(update % :objects omap/wrap))))) - (update :features conj "fdata/objects-map"))) + (let [update-fn #(d/update-when % :objects omap/wrap)] + (-> file + (update :data (fn [fdata] + (-> fdata + (update :pages-index update-vals update-fn) + (update :components update-vals update-fn)))) + (update :features conj "fdata/objects-map")))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; POINTER-MAP +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn load-pointer + "A database loader pointer helper" + [system file-id id] + (let [{:keys [content]} (db/get system :file-data-fragment + {:id id :file-id file-id} + {::db/columns [:content] + ::db/check-deleted? false})] + (when-not content + (ex/raise :type :internal + :code :fragment-not-found + :hint "fragment not found" + :file-id file-id + :fragment-id id)) + + (blob/decode content))) + +(defn persist-pointers! + "Given a database connection and the final file-id, persist all + pointers to the underlying storage (the database)." + [system file-id] + (doseq [[id item] @pmap/*tracked*] + (when (pmap/modified? item) + (l/trc :hint "persist pointer" :file-id (str file-id) :id (str id)) + (let [content (-> item deref blob/encode)] + (db/insert! system :file-data-fragment + {:id id + :file-id file-id + :content content}))))) + +(defn process-pointers + "Apply a function to all pointers on the file. Usuly used for + dereference the pointer to a plain value before some processing." + [fdata update-fn] + (cond-> fdata + (contains? fdata :pages-index) + (update :pages-index process-pointers update-fn) + + :always + (update-vals (fn [val] + (if (pmap/pointer-map? val) + (update-fn val) + val))))) + +(defn get-used-pointer-ids + "Given a file, return all pointer ids used in the data." + [fdata] + (->> (concat (vals fdata) + (vals (:pages-index fdata))) + (into #{} (comp (filter pmap/pointer-map?) + (map pmap/get-id))))) (defn enable-pointer-map + "Enable the fdata/pointer-map feature on the file." [file] (-> file - (update :data (fn [data] - (-> data + (update :data (fn [fdata] + (-> fdata (update :pages-index update-vals pmap/wrap) (update :components pmap/wrap)))) (update :features conj "fdata/pointer-map"))) - -;; (defn enable-shape-data-type -;; [file] -;; (letfn [(update-object [object] -;; (-> object -;; (d/update-when :selrect grc/make-rect) -;; (d/update-when :svg-viewbox grc/make-rect) -;; (cts/map->Shape))) - -;; (update-container [container] -;; (d/update-when container :objects update-vals update-object))] - -;; (-> file -;; (update :data (fn [data] -;; (-> data -;; (update :pages-index update-vals update-container) -;; (update :components update-vals update-container)))) -;; (update :features conj "fdata/shape-data-type")))) diff --git a/backend/src/app/rpc/commands/binfile.clj b/backend/src/app/rpc/commands/binfile.clj index a347c893f..b0c44a16a 100644 --- a/backend/src/app/rpc/commands/binfile.clj +++ b/backend/src/app/rpc/commands/binfile.clj @@ -21,8 +21,8 @@ [app.common.uuid :as uuid] [app.config :as cf] [app.db :as db] - [app.features.components-v2 :as features.components-v2] - [app.features.fdata :as features.fdata] + [app.features.components-v2 :as feat.compv2] + [app.features.fdata :as feat.fdata] [app.http.sse :as sse] [app.loggers.audit :as-alias audit] [app.loggers.webhooks :as-alias webhooks] @@ -305,25 +305,21 @@ (defn- get-files [cfg ids] - (letfn [(get-files* [{:keys [::db/conn]}] - (let [sql (str "SELECT id FROM file " - " WHERE id = ANY(?) ") - ids (db/create-array conn "uuid" ids)] - (->> (db/exec! conn [sql ids]) - (into [] (map :id)) - (not-empty))))] - - (db/run! cfg get-files*))) + (db/run! cfg (fn [{:keys [::db/conn]}] + (let [sql (str "SELECT id FROM file " + " WHERE id = ANY(?) ") + ids (db/create-array conn "uuid" ids)] + (->> (db/exec! conn [sql ids]) + (into [] (map :id)) + (not-empty)))))) (defn- get-file [cfg file-id] - (letfn [(get-file* [{:keys [::db/conn]}] - (binding [pmap/*load-fn* (partial files/load-pointer conn file-id)] - (some-> (db/get* conn :file {:id file-id} {::db/remove-deleted? false}) - (files/decode-row) - (files/process-pointers deref))))] - - (db/run! cfg get-file*))) + (db/run! cfg (fn [{:keys [::db/conn] :as cfg}] + (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)] + (some-> (db/get* conn :file {:id file-id} {::db/remove-deleted? false}) + (files/decode-row) + (update :data feat.fdata/process-pointers deref)))))) (defn- get-file-media [{:keys [::db/pool]} {:keys [data id] :as file}] @@ -666,9 +662,9 @@ (doseq [[feature file-id] (-> *state* deref :pending-to-migrate)] (case feature "components/v2" - (features.components-v2/migrate-file! options file-id - :validate? validate? - :throw-on-validate? true) + (feat.compv2/migrate-file! options file-id + :validate? validate? + :throw-on-validate? true) "fdata/shape-data-type" nil @@ -702,11 +698,11 @@ (cond-> file (and (contains? cfeat/*current* "fdata/objects-map") (not (contains? cfeat/*previous* "fdata/objects-map"))) - (features.fdata/enable-objects-map) + (feat.fdata/enable-objects-map) (and (contains? cfeat/*current* "fdata/pointer-map") (not (contains? cfeat/*previous* "fdata/pointer-map"))) - (features.fdata/enable-pointer-map))) + (feat.fdata/enable-pointer-map))) (defn- get-remaped-thumbnails [thumbnails file-id] @@ -717,7 +713,7 @@ thumbnails)) (defmethod read-section :v1/files - [{:keys [::db/conn ::input ::project-id ::enabled-features ::timestamp ::overwrite?]}] + [{:keys [::db/conn ::input ::project-id ::enabled-features ::timestamp ::overwrite?] :as system}] (doseq [expected-file-id (-> *state* deref :files)] (let [file (read-obj! input) @@ -773,7 +769,6 @@ cfeat/*previous* (:features file) pmap/*tracked* (atom {})] - (let [params (-> file (assoc :id file-id') (assoc :features features) @@ -821,7 +816,7 @@ (create-or-update-file! conn params) (db/insert! conn :file params)) - (files/persist-pointers! conn file-id') + (feat.fdata/persist-pointers! system file-id') (when overwrite? (db/delete! conn :file-thumbnail {:file-id file-id'})) diff --git a/backend/src/app/rpc/commands/comments.clj b/backend/src/app/rpc/commands/comments.clj index b8352f622..5e87884f6 100644 --- a/backend/src/app/rpc/commands/comments.clj +++ b/backend/src/app/rpc/commands/comments.clj @@ -12,6 +12,7 @@ [app.common.spec :as us] [app.common.uuid :as uuid] [app.db :as db] + [app.features.fdata :as feat.fdata] [app.loggers.audit :as-alias audit] [app.loggers.webhooks :as-alias webhooks] [app.rpc :as-alias rpc] @@ -43,15 +44,17 @@ (defn- get-file "A specialized version of get-file for comments module." - [conn file-id page-id] - (binding [pmap/*load-fn* (partial files/load-pointer conn file-id)] - (if-let [{:keys [data] :as file} (some-> (db/exec-one! conn [sql:get-file file-id]) (files/decode-row))] + [{:keys [::db/conn] :as cfg} file-id page-id] + (if-let [{:keys [data] :as file} (some-> (db/exec-one! conn [sql:get-file file-id]) + (files/decode-row))] + (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)] (-> file (assoc :page-name (dm/get-in data [:pages-index page-id :name])) - (assoc :page-id page-id)) - (ex/raise :type :not-found - :code :object-not-found - :hint "file not found")))) + (assoc :page-id page-id))) + + (ex/raise :type :not-found + :code :object-not-found + :hint "file not found"))) (defn- get-comment-thread [conn thread-id & {:as opts}] @@ -288,38 +291,37 @@ (sv/defmethod ::create-comment-thread {::doc/added "1.15" ::webhooks/event? true} - [{:keys [::db/pool] :as cfg} - {:keys [::rpc/profile-id ::rpc/request-at file-id page-id share-id position content frame-id]}] + [cfg {:keys [::rpc/profile-id ::rpc/request-at file-id page-id share-id position content frame-id]}] + (db/tx-run! cfg + (fn [{:keys [::db/conn] :as cfg}] + (files/check-comment-permissions! conn profile-id file-id share-id) + (let [{:keys [team-id project-id page-name] :as file} (get-file cfg file-id page-id)] - (db/with-atomic [conn pool] - (let [{:keys [team-id project-id page-name] :as file} (get-file conn file-id page-id)] - (files/check-comment-permissions! conn profile-id file-id share-id) + (run! (partial quotes/check-quote! conn) + (list {::quotes/id ::quotes/comment-threads-per-file + ::quotes/profile-id profile-id + ::quotes/team-id team-id + ::quotes/project-id project-id + ::quotes/file-id file-id} + {::quotes/id ::quotes/comments-per-file + ::quotes/profile-id profile-id + ::quotes/team-id team-id + ::quotes/project-id project-id + ::quotes/file-id file-id})) - (run! (partial quotes/check-quote! conn) - (list {::quotes/id ::quotes/comment-threads-per-file - ::quotes/profile-id profile-id - ::quotes/team-id team-id - ::quotes/project-id project-id - ::quotes/file-id file-id} - {::quotes/id ::quotes/comments-per-file - ::quotes/profile-id profile-id - ::quotes/team-id team-id - ::quotes/project-id project-id - ::quotes/file-id file-id})) - - (rtry/with-retry {::rtry/when rtry/conflict-exception? - ::rtry/max-retries 3 - ::rtry/label "create-comment-thread" - ::db/conn conn} - (create-comment-thread conn - {:created-at request-at - :profile-id profile-id - :file-id file-id - :page-id page-id - :page-name page-name - :position position - :content content - :frame-id frame-id}))))) + (rtry/with-retry {::rtry/when rtry/conflict-exception? + ::rtry/max-retries 3 + ::rtry/label "create-comment-thread" + ::db/conn conn} + (create-comment-thread conn + {:created-at request-at + :profile-id profile-id + :file-id file-id + :page-id page-id + :page-name page-name + :position position + :content content + :frame-id frame-id})))))) (defn- create-comment-thread @@ -402,8 +404,7 @@ ;; --- COMMAND: Add Comment -(declare get-comment-thread) -(declare create-comment) +(declare ^:private get-comment-thread) (s/def ::create-comment (s/keys :req [::rpc/profile-id] @@ -413,49 +414,51 @@ (sv/defmethod ::create-comment {::doc/added "1.15" ::webhooks/event? true} - [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at thread-id share-id content] :as params}] - (db/with-atomic [conn pool] - (let [{:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::db/for-update? true) - {:keys [team-id project-id page-name] :as file} (get-file conn file-id page-id)] + [cfg {:keys [::rpc/profile-id ::rpc/request-at thread-id share-id content]}] + (db/tx-run! cfg + (fn [{:keys [::db/conn] :as cfg}] + (let [{:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::db/for-update? true) + {:keys [team-id project-id page-name] :as file} (get-file cfg file-id page-id)] - (files/check-comment-permissions! conn profile-id (:id file) share-id) - (quotes/check-quote! conn - {::quotes/id ::quotes/comments-per-file - ::quotes/profile-id profile-id - ::quotes/team-id team-id - ::quotes/project-id project-id - ::quotes/file-id (:id file)}) + (files/check-comment-permissions! conn profile-id (:id file) share-id) + (quotes/check-quote! conn + {::quotes/id ::quotes/comments-per-file + ::quotes/profile-id profile-id + ::quotes/team-id team-id + ::quotes/project-id project-id + ::quotes/file-id (:id file)}) - ;; Update the page-name cached attribute on comment thread table. - (when (not= page-name (:page-name thread)) - (db/update! conn :comment-thread - {:page-name page-name} - {:id thread-id})) + ;; Update the page-name cached attribute on comment thread table. + (when (not= page-name (:page-name thread)) + (db/update! conn :comment-thread + {:page-name page-name} + {:id thread-id})) - (let [comment (db/insert! conn :comment - {:id (uuid/next) - :created-at request-at - :modified-at request-at - :thread-id thread-id - :owner-id profile-id - :content content}) - props {:file-id file-id - :share-id nil}] + (let [comment (db/insert! conn :comment + {:id (uuid/next) + :created-at request-at + :modified-at request-at + :thread-id thread-id + :owner-id profile-id + :content content}) + props {:file-id file-id + :share-id nil}] - ;; Update thread modified-at attribute and assoc the current - ;; profile to the participant set. - (db/update! conn :comment-thread - {:modified-at request-at - :participants (-> (:participants thread #{}) - (conj profile-id) - (db/tjson))} - {:id thread-id}) + ;; Update thread modified-at attribute and assoc the current + ;; profile to the participant set. + (db/update! conn :comment-thread + {:modified-at request-at + :participants (-> (:participants thread #{}) + (conj profile-id) + (db/tjson))} + {:id thread-id}) - ;; Update the current profile status in relation to the - ;; current thread. - (upsert-comment-thread-status! conn profile-id thread-id request-at) + ;; Update the current profile status in relation to the + ;; current thread. + (upsert-comment-thread-status! conn profile-id thread-id request-at) + + (vary-meta comment assoc ::audit/props props)))))) - (vary-meta comment assoc ::audit/props props))))) ;; --- COMMAND: Update Comment @@ -466,29 +469,31 @@ (sv/defmethod ::update-comment {::doc/added "1.15"} - [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at id share-id content] :as params}] - (db/with-atomic [conn pool] - (let [{:keys [thread-id owner-id] :as comment} (get-comment conn id ::db/for-update? true) - {:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::db/for-update? true)] + [cfg {:keys [::rpc/profile-id ::rpc/request-at id share-id content]}] - (files/check-comment-permissions! conn profile-id file-id share-id) + (db/tx-run! cfg + (fn [{:keys [::db/conn] :as cfg}] + (let [{:keys [thread-id owner-id] :as comment} (get-comment conn id ::db/for-update? true) + {:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::db/for-update? true)] - ;; Don't allow edit comments to not owners - (when-not (= owner-id profile-id) - (ex/raise :type :validation - :code :not-allowed)) + (files/check-comment-permissions! conn profile-id file-id share-id) - (let [{:keys [page-name] :as file} (get-file conn file-id page-id)] - (db/update! conn :comment - {:content content - :modified-at request-at} - {:id id}) + ;; Don't allow edit comments to not owners + (when-not (= owner-id profile-id) + (ex/raise :type :validation + :code :not-allowed)) - (db/update! conn :comment-thread - {:modified-at request-at - :page-name page-name} - {:id thread-id}) - nil)))) + (let [{:keys [page-name] :as file} (get-file cfg file-id page-id)] + (db/update! conn :comment + {:content content + :modified-at request-at} + {:id id}) + + (db/update! conn :comment-thread + {:modified-at request-at + :page-name page-name} + {:id thread-id}) + nil))))) ;; --- COMMAND: Delete Comment Thread @@ -499,7 +504,7 @@ (sv/defmethod ::delete-comment-thread {::doc/added "1.15"} - [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id share-id] :as params}] + [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id share-id]}] (db/with-atomic [conn pool] (let [{:keys [owner-id file-id] :as thread} (get-comment-thread conn id ::db/for-update? true)] (files/check-comment-permissions! conn profile-id file-id share-id) @@ -539,12 +544,12 @@ (sv/defmethod ::update-comment-thread-position {::doc/added "1.15"} - [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id position frame-id share-id] :as params}] + [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at id position frame-id share-id]}] (db/with-atomic [conn pool] (let [{:keys [file-id] :as thread} (get-comment-thread conn id ::db/for-update? true)] (files/check-comment-permissions! conn profile-id file-id share-id) (db/update! conn :comment-thread - {:modified-at (::rpc/request-at params) + {:modified-at request-at :position (db/pgpoint position) :frame-id frame-id} {:id (:id thread)}) @@ -559,12 +564,12 @@ (sv/defmethod ::update-comment-thread-frame {::doc/added "1.15"} - [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id frame-id share-id] :as params}] + [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at id frame-id share-id]}] (db/with-atomic [conn pool] (let [{:keys [file-id] :as thread} (get-comment-thread conn id ::db/for-update? true)] (files/check-comment-permissions! conn profile-id file-id share-id) (db/update! conn :comment-thread - {:modified-at (::rpc/request-at params) + {:modified-at request-at :frame-id frame-id} {:id id}) nil))) diff --git a/backend/src/app/rpc/commands/files.clj b/backend/src/app/rpc/commands/files.clj index 31fbd359f..d9e1d9b3a 100644 --- a/backend/src/app/rpc/commands/files.clj +++ b/backend/src/app/rpc/commands/files.clj @@ -20,6 +20,7 @@ [app.common.types.file :as ctf] [app.config :as cf] [app.db :as db] + [app.features.fdata :as feat.fdata] [app.loggers.audit :as-alias audit] [app.loggers.webhooks :as-alias webhooks] [app.rpc :as-alias rpc] @@ -181,62 +182,6 @@ :code :object-not-found :hint "not found")))) -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; FEATURES: pointer-map -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -(defn load-pointer - [conn file-id id] - - (dm/assert! - "expected valid connection" - (db/connection? conn)) - - (let [{:keys [content]} (db/get conn :file-data-fragment - {:id id :file-id file-id} - {::db/columns [:content] - ::db/check-deleted? false})] - (when-not content - (ex/raise :type :internal - :code :fragment-not-found - :hint "fragment not found" - :file-id file-id - :fragment-id id)) - - (blob/decode content))) - -(defn persist-pointers! - [conn file-id] - (doseq [[id item] @pmap/*tracked*] - (when (pmap/modified? item) - (let [content (-> item deref blob/encode)] - (db/insert! conn :file-data-fragment - {:id id - :file-id file-id - :content content}))))) - -(defn process-pointers - [file update-fn] - (update file :data (fn resolve-fn [data] - (cond-> data - (contains? data :pages-index) - (update :pages-index resolve-fn) - - :always - (update-vals (fn [val] - (if (pmap/pointer-map? val) - (update-fn val) - val))))))) - - -(defn get-all-pointer-ids - "Given a file, return all pointer ids used in the data." - [fdata] - (->> (concat (vals fdata) - (vals (:pages-index fdata))) - (into #{} (comp (filter pmap/pointer-map?) - (map pmap/get-id))))) - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; QUERY COMMANDS ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -277,49 +222,51 @@ [:id ::sm/uuid] [:project-id {:optional true} ::sm/uuid]])) -(defn get-file - [conn id & {:keys [project-id migrate? - include-deleted? - lock-for-update?] - :or {include-deleted? false - lock-for-update? false}}] - (dm/assert! - "expected raw connection" - (db/connection? conn)) - - (binding [pmap/*load-fn* (partial load-pointer conn id) +(defn- migrate-file + [{:keys [::db/conn] :as cfg} {:keys [id] :as file}] + (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id) pmap/*tracked* (pmap/create-tracked) cfeat/*new* (atom #{})] + (let [file (fmg/migrate-file file)] + ;; NOTE: when file is migrated, we break the rule of no perform + ;; mutations on get operations and update the file with all + ;; migrations applied + ;; + ;; NOTE: the following code will not work on read-only mode, it + ;; is a known issue; we keep is not implemented until we really + ;; need this + (if (fmg/migrated? file) + (let [file (update file :features cfeat/migrate-legacy-features) + features (set/union (deref cfeat/*new*) (:features file))] + (db/update! conn :file + {:data (blob/encode (:data file)) + :features (db/create-array conn "text" features)} + {:id id}) + (feat.fdata/persist-pointers! cfg id) + (assoc file :features features)) + file)))) - (let [params (merge {:id id} - (when (some? project-id) - {:project-id project-id})) +(defn get-file + [{:keys [::db/conn] :as cfg} id & {:keys [project-id migrate? + include-deleted? + lock-for-update?] + :or {include-deleted? false + lock-for-update? false}}] + (dm/assert! + "expected cfg with valid connection" + (db/connection-map? cfg)) - file (-> (db/get conn :file params - {::db/check-deleted? (not include-deleted?) - ::db/remove-deleted? (not include-deleted?) - ::db/for-update? lock-for-update?}) - (decode-row) - (cond-> migrate? - (fmg/migrate-file)))] - - ;; NOTE: when file is migrated, we break the rule of no perform - ;; mutations on get operations and update the file with all - ;; migrations applied - ;; - ;; NOTE: the following code will not work on read-only mode, it - ;; is a known issue; we keep is not implemented until we really - ;; need this - (if (fmg/migrated? file) - (let [file (update file :features cfeat/migrate-legacy-features) - features (set/union (deref cfeat/*new*) (:features file))] - (db/update! conn :file - {:data (blob/encode (:data file)) - :features (db/create-array conn "text" features)} - {:id id}) - (persist-pointers! conn id) - (assoc file :features features)) - file)))) + (let [params (merge {:id id} + (when (some? project-id) + {:project-id project-id})) + file (-> (db/get conn :file params + {::db/check-deleted? (not include-deleted?) + ::db/remove-deleted? (not include-deleted?) + ::db/for-update? lock-for-update?}) + (decode-row))] + (if migrate? + (migrate-file cfg file) + file))) (defn get-minimal-file [{:keys [::db/pool] :as cfg} id] @@ -345,7 +292,7 @@ :project-id project-id :file-id id) - file (-> (get-file conn id :project-id project-id) + file (-> (get-file cfg id :project-id project-id) (assoc :permissions perms) (check-version!)) @@ -358,8 +305,8 @@ ;; pointers on backend and return a complete file. file (if (and (contains? (:features file) "fdata/pointer-map") (not (contains? (:features params) "fdata/pointer-map"))) - (binding [pmap/*load-fn* (partial load-pointer conn id)] - (process-pointers file deref)) + (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)] + (update file :data feat.fdata/process-pointers deref)) file)] (vary-meta file assoc ::cond/key (get-file-etag params file))))))) @@ -498,6 +445,7 @@ (defn get-page [{:keys [::db/conn] :as cfg} {:keys [profile-id file-id page-id object-id] :as params}] + (when (and (uuid? object-id) (not (uuid? page-id))) (ex/raise :type :validation @@ -508,13 +456,13 @@ :profile-id profile-id :file-id file-id) - file (get-file conn file-id) + file (get-file cfg file-id) _ (-> (cfeat/get-team-enabled-features cf/flags team) (cfeat/check-client-features! (:features params)) (cfeat/check-file-features! (:features file) (:features params))) - page (binding [pmap/*load-fn* (partial load-pointer conn file-id)] + page (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)] (let [page-id (or page-id (-> file :data :pages first)) page (dm/get-in file [:data :pages-index page-id])] (if (pmap/pointer-map? page) @@ -573,37 +521,39 @@ and p.team_id = ? order by f.modified_at desc") -;; FIXME: i'm not sure about feature handling here... ??? -(defn get-team-shared-files - [conn team-id] + +(defn- get-library-summary + [cfg {:keys [id data] :as file}] (letfn [(assets-sample [assets limit] (let [sorted-assets (->> (vals assets) (sort-by #(str/lower (:name %))))] {:count (count sorted-assets) - :sample (into [] (take limit sorted-assets))})) + :sample (into [] (take limit sorted-assets))}))] - (library-summary [{:keys [id data] :as file}] - (binding [pmap/*load-fn* (partial load-pointer conn id)] - (let [load-objects (fn [component] - (ctf/load-component-objects data component)) - components-sample (-> (assets-sample (ctkl/components data) 4) - (update :sample #(mapv load-objects %)))] - {:components components-sample - :media (assets-sample (:media data) 3) - :colors (assets-sample (:colors data) 3) - :typographies (assets-sample (:typographies data) 3)})))] + (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)] + (let [load-objects (fn [component] + (ctf/load-component-objects data component)) + components-sample (-> (assets-sample (ctkl/components data) 4) + (update :sample #(mapv load-objects %)))] + {:components components-sample + :media (assets-sample (:media data) 3) + :colors (assets-sample (:colors data) 3) + :typographies (assets-sample (:typographies data) 3)})))) - (->> (db/exec! conn [sql:team-shared-files team-id]) - (into #{} (comp - (map decode-row) - (map (fn [row] - (if-let [media-id (:media-id row)] - (-> row - (dissoc :media-id) - (assoc :thumbnail-uri (resolve-public-uri media-id))) - (dissoc row :media-id)))) - (map #(assoc % :library-summary (library-summary %))) - (map #(dissoc % :data))))))) +(defn- get-team-shared-files + [{:keys [::db/conn] :as cfg} {:keys [team-id profile-id]}] + (teams/check-read-permissions! conn profile-id team-id) + (->> (db/exec! conn [sql:team-shared-files team-id]) + (into #{} (comp + (map decode-row) + (map (fn [row] + (if-let [media-id (:media-id row)] + (-> row + (dissoc :media-id) + (assoc :thumbnail-uri (resolve-public-uri media-id))) + (dissoc row :media-id)))) + (map #(assoc % :library-summary (get-library-summary cfg %))) + (map #(dissoc % :data)))))) (def ^:private schema:get-team-shared-files [:map {:title "get-team-shared-files"} @@ -613,10 +563,8 @@ "Get all file (libraries) for the specified team." {::doc/added "1.17" ::sm/params schema:get-team-shared-files} - [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id]}] - (dm/with-open [conn (db/open pool)] - (teams/check-read-permissions! conn profile-id team-id) - (get-team-shared-files conn team-id))) + [cfg {:keys [::rpc/profile-id] :as params}] + (db/tx-run! cfg get-team-shared-files (assoc params :profile-id profile-id))) ;; --- COMMAND QUERY: get-file-libraries @@ -744,30 +692,32 @@ ;; --- COMMAND QUERY: get-file-summary +(defn- get-file-summary + [{:keys [::db/conn] :as cfg} {:keys [profile-id id project-id] :as params}] + (check-read-permissions! conn profile-id id) + (let [team (teams/get-team conn + :profile-id profile-id + :project-id project-id + :file-id id) + + file (get-file cfg id :project-id project-id)] + + (-> (cfeat/get-team-enabled-features cf/flags team) + (cfeat/check-client-features! (:features params)) + (cfeat/check-file-features! (:features file) (:features params))) + + {:name (:name file) + :components-count (count (ctkl/components-seq (:data file))) + :graphics-count (count (get-in file [:data :media] [])) + :colors-count (count (get-in file [:data :colors] [])) + :typography-count (count (get-in file [:data :typographies] []))})) + (sv/defmethod ::get-file-summary "Retrieve a file summary by its ID. Only authenticated users." {::doc/added "1.20" ::sm/params schema:get-file} - [cfg {:keys [::rpc/profile-id id project-id] :as params}] - (db/tx-run! cfg - (fn [{:keys [::db/conn] :as cfg}] - (check-read-permissions! conn profile-id id) - (let [team (teams/get-team conn - :profile-id profile-id - :project-id project-id - :file-id id) - - file (get-file conn id :project-id project-id)] - - (-> (cfeat/get-team-enabled-features cf/flags team) - (cfeat/check-client-features! (:features params)) - (cfeat/check-file-features! (:features file) (:features params))) - - {:name (:name file) - :components-count (count (ctkl/components-seq (:data file))) - :graphics-count (count (get-in file [:data :media] [])) - :colors-count (count (get-in file [:data :colors] [])) - :typography-count (count (get-in file [:data :typographies] []))})))) + [cfg {:keys [::rpc/profile-id] :as params}] + (db/tx-run! cfg get-file-summary (assoc params :profile-id profile-id))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; MUTATION COMMANDS @@ -827,10 +777,15 @@ ORDER BY f.created_at ASC;") (defn- absorb-library-by-file! - [conn ldata file-id] - (binding [pmap/*load-fn* (partial load-pointer conn file-id) + [cfg ldata file-id] + + (dm/assert! + "expected cfg with valid connection" + (db/connection-map? cfg)) + + (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id) pmap/*tracked* (pmap/create-tracked)] - (let [file (-> (get-file conn file-id + (let [file (-> (get-file cfg file-id :include-deleted? true :lock-for-update? true) (update :data ctf/absorb-assets ldata))] @@ -839,28 +794,84 @@ :library-id (str (:id ldata)) :file-id (str file-id)) - (db/update! conn :file + (db/update! cfg :file {:revn (inc (:revn file)) :data (blob/encode (:data file)) :modified-at (dt/now)} {:id file-id}) - (persist-pointers! conn file-id)))) + (feat.fdata/persist-pointers! cfg file-id)))) (defn- absorb-library! "Find all files using a shared library, and absorb all library assets into the file local libraries" - [conn {:keys [id] :as library}] - (let [ldata (binding [pmap/*load-fn* (partial load-pointer conn id)] - (-> library (process-pointers deref) :data)) - ids (->> (db/exec! conn [sql:get-referenced-files id]) + [cfg {:keys [id] :as library}] + + (dm/assert! + "expected cfg with valid connection" + (db/connection-map? cfg)) + + (let [ldata (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)] + (-> library :data (feat.fdata/process-pointers deref))) + ids (->> (db/exec! cfg [sql:get-referenced-files id]) (map :id))] (l/trc :hint "absorbing library" :library-id (str id) :files (str/join "," (map str ids))) - (run! (partial absorb-library-by-file! conn ldata) ids))) + (run! (partial absorb-library-by-file! cfg ldata) ids))) + +(defn- set-file-shared + [{:keys [::db/conn] :as cfg} {:keys [profile-id id] :as params}] + (check-edition-permissions! conn profile-id id) + (let [file (db/get-by-id conn id {:columns [:id :name :is-shared]}) + file (cond + (and (true? (:is-shared file)) + (false? (:is-shared params))) + ;; When we disable shared flag on an already shared + ;; file, we need to perform more complex operation, + ;; so in this case we retrieve the complete file and + ;; perform all required validations. + (let [file (-> (get-file cfg id :lock-for-update? true) + (check-version!) + (assoc :is-shared false)) + team (teams/get-team conn + :profile-id profile-id + :project-id (:project-id file))] + + (-> (cfeat/get-team-enabled-features cf/flags team) + (cfeat/check-client-features! (:features params)) + (cfeat/check-file-features! (:features file))) + + (absorb-library! cfg file) + + (db/delete! conn :file-library-rel {:library-file-id id}) + (db/update! conn :file + {:is-shared false} + {:id id}) + file) + + (and (false? (:is-shared file)) + (true? (:is-shared params))) + (let [file (assoc file :is-shared true)] + (db/update! conn :file + {:is-shared false} + {:id id}) + file) + + :else + (ex/raise :type :validation + :code :invalid-shared-state + :hint "unexpected state found" + :params-is-shared (:is-shared params) + :file-is-shared (:is-shared file)))] + + (rph/with-meta + (select-keys file [:id :name :is-shared]) + {::audit/props {:name (:name file) + :project-id (:project-id file) + :is-shared (:is-shared file)}}))) (def ^:private schema:set-file-shared @@ -873,56 +884,8 @@ {::doc/added "1.17" ::webhooks/event? true ::sm/params schema:set-file-shared} - [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id] :as params}] - (db/with-atomic [conn pool] - (check-edition-permissions! conn profile-id id) - (let [file (db/get-by-id conn id {:columns [:id :name :is-shared]}) - file (cond - (and (true? (:is-shared file)) - (false? (:is-shared params))) - ;; When we disable shared flag on an already shared - ;; file, we need to perform more complex operation, - ;; so in this case we retrieve the complete file and - ;; perform all required validations. - (let [file (-> (get-file conn id :lock-for-update? true) - (check-version!) - (assoc :is-shared false)) - team (teams/get-team conn - :profile-id profile-id - :project-id (:project-id file))] - - (-> (cfeat/get-team-enabled-features cf/flags team) - (cfeat/check-client-features! (:features params)) - (cfeat/check-file-features! (:features file))) - - (absorb-library! conn file) - - (db/delete! conn :file-library-rel {:library-file-id id}) - (db/update! conn :file - {:is-shared false} - {:id id}) - file) - - (and (false? (:is-shared file)) - (true? (:is-shared params))) - (let [file (assoc file :is-shared true)] - (db/update! conn :file - {:is-shared false} - {:id id}) - file) - - :else - (ex/raise :type :validation - :code :invalid-shared-state - :hint "unexpected state found" - :params-is-shared (:is-shared params) - :file-is-shared (:is-shared file)))] - - (rph/with-meta - (select-keys file [:id :name :is-shared]) - {::audit/props {:name (:name file) - :project-id (:project-id file) - :is-shared (:is-shared file)}})))) + [cfg {:keys [::rpc/profile-id] :as params}] + (db/tx-run! cfg set-file-shared (assoc params :profile-id profile-id))) ;; --- MUTATION COMMAND: delete-file @@ -939,41 +902,44 @@ [:map {:title "delete-file"} [:id ::sm/uuid]])) +(defn- delete-file + [{:keys [::db/conn] :as cfg} {:keys [profile-id id] :as params}] + (check-edition-permissions! conn profile-id id) + (let [file (mark-file-deleted! conn id)] + + ;; NOTE: when a file is a shared library, then we proceed to load + ;; the whole file, proceed with feature checking and properly execute + ;; the absorb-library procedure + (when (:is-shared file) + (let [file (-> (get-file cfg id + :lock-for-update? true + :include-deleted? true) + (check-version!)) + + team (teams/get-team conn + :profile-id profile-id + :project-id (:project-id file))] + + + + (-> (cfeat/get-team-enabled-features cf/flags team) + (cfeat/check-client-features! (:features params)) + (cfeat/check-file-features! (:features file))) + + (absorb-library! cfg file))) + + (rph/with-meta (rph/wrap) + {::audit/props {:project-id (:project-id file) + :name (:name file) + :created-at (:created-at file) + :modified-at (:modified-at file)}}))) + (sv/defmethod ::delete-file {::doc/added "1.17" ::webhooks/event? true ::sm/params schema:delete-file} - [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id] :as params}] - (db/with-atomic [conn pool] - (check-edition-permissions! conn profile-id id) - (let [file (mark-file-deleted! conn id)] - - ;; NOTE: when a file is a shared library, then we proceed to load - ;; the whole file, proceed with feature checking and properly execute - ;; the absorb-library procedure - (when (:is-shared file) - (let [file (-> (get-file conn id - :lock-for-update? true - :include-deleted? true) - (check-version!)) - - team (teams/get-team conn - :profile-id profile-id - :project-id (:project-id file))] - - - - (-> (cfeat/get-team-enabled-features cf/flags team) - (cfeat/check-client-features! (:features params)) - (cfeat/check-file-features! (:features file))) - - (absorb-library! conn file))) - - (rph/with-meta (rph/wrap) - {::audit/props {:project-id (:project-id file) - :name (:name file) - :created-at (:created-at file) - :modified-at (:modified-at file)}})))) + [cfg {:keys [::rpc/profile-id] :as params}] + (db/tx-run! cfg delete-file (assoc params :profile-id profile-id))) ;; --- MUTATION COMMAND: link-file-to-library diff --git a/backend/src/app/rpc/commands/files_create.clj b/backend/src/app/rpc/commands/files_create.clj index b3cae1097..59273f033 100644 --- a/backend/src/app/rpc/commands/files_create.clj +++ b/backend/src/app/rpc/commands/files_create.clj @@ -7,12 +7,14 @@ (ns app.rpc.commands.files-create (:require [app.common.data :as d] + [app.common.data.macros :as dm] [app.common.features :as cfeat] [app.common.schema :as sm] [app.common.types.file :as ctf] [app.common.uuid :as uuid] [app.config :as cf] [app.db :as db] + [app.features.fdata :as feat.fdata] [app.loggers.audit :as-alias audit] [app.loggers.webhooks :as-alias webhooks] [app.rpc :as-alias rpc] @@ -44,13 +46,20 @@ :or {is-shared false revn 0 create-page true} :as params}] + (dm/assert! + "expected a valid connection" + (db/connection? conn)) + (let [id (or id (uuid/next)) - pointers (atom {}) + pointers (pmap/create-tracked) + pmap? (contains? features "fdata/pointer-map") + omap? (contains? features "fdata/objects-map") + data (binding [pmap/*tracked* pointers cfeat/*current* features - cfeat/*wrap-with-objects-map-fn* (if (features "fdata/objects-map") omap/wrap identity) - cfeat/*wrap-with-pointer-map-fn* (if (features "fdata/pointer-map") pmap/wrap identity)] + cfeat/*wrap-with-objects-map-fn* (if omap? omap/wrap identity) + cfeat/*wrap-with-pointer-map-fn* (if pmap? pmap/wrap identity)] (if create-page (ctf/make-file-data id) (ctf/make-file-data id nil))) @@ -72,7 +81,7 @@ :deleted-at deleted-at}))] (binding [pmap/*tracked* pointers] - (files/persist-pointers! conn id)) + (feat.fdata/persist-pointers! cfg id)) (->> (assoc params :file-id id :role :owner) (create-file-role! conn)) diff --git a/backend/src/app/rpc/commands/files_thumbnails.clj b/backend/src/app/rpc/commands/files_thumbnails.clj index 3eb5c5444..c60338349 100644 --- a/backend/src/app/rpc/commands/files_thumbnails.clj +++ b/backend/src/app/rpc/commands/files_thumbnails.clj @@ -16,6 +16,7 @@ [app.common.types.shape-tree :as ctt] [app.config :as cf] [app.db :as db] + [app.features.fdata :as feat.fdata] [app.loggers.audit :as-alias audit] [app.loggers.webhooks :as-alias webhooks] [app.media :as media] @@ -100,28 +101,28 @@ ;; loading all pages into memory for find the frame set for thumbnail. (defn get-file-data-for-thumbnail - [conn {:keys [data id] :as file}] + [{:keys [::db/conn] :as cfg} {:keys [data id] :as file}] (letfn [;; function responsible on finding the frame marked to be ;; used as thumbnail; the returned frame always have ;; the :page-id set to the page that it belongs. - (get-thumbnail-frame [data] + (get-thumbnail-frame [file] ;; NOTE: this is a hack for avoid perform blocking ;; operation inside the for loop, clojure lazy-seq uses ;; synchronized blocks that does not plays well with - ;; virtual threads, so we need to perform the load - ;; operation first. This operation forces all pointer maps - ;; load into the memory. - (->> (-> data :pages-index vals) - (filter pmap/pointer-map?) - (run! pmap/load!)) - - ;; Then proceed to find the frame set for thumbnail - (d/seek #(or (:use-for-thumbnail %) - (:use-for-thumbnail? %)) ; NOTE: backward comp (remove on v1.21) - (for [page (-> data :pages-index vals) - frame (-> page :objects ctt/get-frames)] - (assoc frame :page-id (:id page))))) + ;; virtual threads where all rpc methods calls are + ;; dispatched, so we need to perform the load operation + ;; first. This operation forces all pointer maps load into + ;; the memory. + ;; + ;; FIXME: this is no longer true with clojure>=1.12 + (let [{:keys [data]} (update file :data feat.fdata/process-pointers pmap/load!)] + ;; Then proceed to find the frame set for thumbnail + (d/seek #(or (:use-for-thumbnail %) + (:use-for-thumbnail? %)) ; NOTE: backward comp (remove on v1.21) + (for [page (-> data :pages-index vals) + frame (-> page :objects ctt/get-frames)] + (assoc frame :page-id (:id page)))))) ;; function responsible to filter objects data structure of ;; all unneeded shapes if a concrete frame is provided. If no @@ -165,8 +166,8 @@ objects)))] - (binding [pmap/*load-fn* (partial files/load-pointer conn id)] - (let [frame (get-thumbnail-frame data) + (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)] + (let [frame (get-thumbnail-frame file) frame-id (:id frame) page-id (or (:page-id frame) (-> data :pages first)) @@ -220,7 +221,7 @@ :profile-id profile-id :file-id file-id) - file (files/get-file conn file-id)] + file (files/get-file cfg file-id)] (-> (cfeat/get-team-enabled-features cf/flags team) (cfeat/check-client-features! (:features params)) @@ -228,7 +229,7 @@ {:file-id file-id :revn (:revn file) - :page (get-file-data-for-thumbnail conn file)})))) + :page (get-file-data-for-thumbnail cfg file)})))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; MUTATION COMMANDS diff --git a/backend/src/app/rpc/commands/files_update.clj b/backend/src/app/rpc/commands/files_update.clj index 0d94874f3..28e16ab2c 100644 --- a/backend/src/app/rpc/commands/files_update.clj +++ b/backend/src/app/rpc/commands/files_update.clj @@ -17,7 +17,7 @@ [app.common.uuid :as uuid] [app.config :as cf] [app.db :as db] - [app.features.fdata :refer [enable-pointer-map enable-objects-map]] + [app.features.fdata :as feat.fdata] [app.http.errors :as errors] [app.loggers.audit :as audit] [app.loggers.webhooks :as webhooks] @@ -106,12 +106,12 @@ (defn- wrap-with-pointer-map-context [f] - (fn [{:keys [::db/conn] :as cfg} {:keys [id] :as file}] - (binding [pmap/*tracked* (atom {}) - pmap/*load-fn* (partial files/load-pointer conn id) + (fn [cfg {:keys [id] :as file}] + (binding [pmap/*tracked* (pmap/create-tracked) + pmap/*load-fn* (partial feat.fdata/load-pointer cfg id) cfeat/*wrap-with-pointer-map-fn* pmap/wrap] (let [result (f cfg file)] - (files/persist-pointers! conn id) + (feat.fdata/persist-pointers! cfg id) result)))) (defn- wrap-with-objects-map-context @@ -236,7 +236,7 @@ ;; to be executed on a separated executor for avoid to do the ;; CPU intensive operation on vthread. - update-fdata-fn (partial update-file-data conn file changes skip-validate) + update-fdata-fn (partial update-file-data cfg file changes skip-validate) file (-> (climit/configure cfg :update-file/global) (climit/run! update-fdata-fn executor))] @@ -290,7 +290,7 @@ file) (defn- update-file-data - [conn file changes skip-validate] + [{:keys [::db/conn] :as cfg} file changes skip-validate] (let [file (update file :data (fn [data] (-> data (blob/decode) @@ -304,10 +304,10 @@ (not skip-validate)) (->> (files/get-file-libraries conn (:id file)) (into [file] (map (fn [{:keys [id]}] - (binding [pmap/*load-fn* (partial files/load-pointer conn id) + (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id) pmap/*tracked* nil] - (-> (files/get-file conn id :migrate? false) - (files/process-pointers deref) ; ensure all pointers resolved + (-> (files/get-file cfg id :migrate? false) + (feat.fdata/process-pointers deref) ; ensure all pointers resolved (fmg/migrate-file)))))) (d/index-by :id)))] @@ -332,11 +332,11 @@ (cond-> (and (contains? cfeat/*current* "fdata/objects-map") (not (contains? cfeat/*previous* "fdata/objects-map"))) - (enable-objects-map)) + (feat.fdata/enable-objects-map)) (cond-> (and (contains? cfeat/*current* "fdata/pointer-map") (not (contains? cfeat/*previous* "fdata/pointer-map"))) - (enable-pointer-map)) + (feat.fdata/enable-pointer-map)) (update :data blob/encode)))) diff --git a/backend/src/app/rpc/commands/management.clj b/backend/src/app/rpc/commands/management.clj index 00195d17e..46661029c 100644 --- a/backend/src/app/rpc/commands/management.clj +++ b/backend/src/app/rpc/commands/management.clj @@ -14,11 +14,11 @@ [app.common.schema :as sm] [app.common.uuid :as uuid] [app.db :as db] + [app.features.fdata :as feat.fdata] [app.http.sse :as sse] [app.loggers.webhooks :as-alias webhooks] [app.rpc :as-alias rpc] [app.rpc.commands.binfile :as binfile] - [app.rpc.commands.files :as files] [app.rpc.commands.projects :as proj] [app.rpc.commands.teams :as teams :refer [create-project-role create-project]] [app.rpc.doc :as-alias doc] @@ -49,9 +49,8 @@ {::doc/added "1.16" ::webhooks/event? true ::sm/params schema:duplicate-file} - [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}] - (db/with-atomic [conn pool] - (duplicate-file conn (assoc params :profile-id profile-id)))) + [cfg {:keys [::rpc/profile-id] :as params}] + (db/tx-run! cfg duplicate-file (assoc params :profile-id profile-id))) (defn- remap-id [item index key] @@ -60,7 +59,7 @@ (assoc key (get index (get item key) (get item key))))) (defn- process-file - [conn {:keys [id] :as file} index] + [cfg index {:keys [id] :as file}] (letfn [(process-form [form] (cond-> form ;; Relink library items @@ -103,26 +102,28 @@ (dissoc k)) res))) media - media))] - (-> file - (update :id #(get index %)) - (update :data - (fn [data] - (binding [pmap/*load-fn* (partial files/load-pointer conn id) - pmap/*tracked* (atom {})] - (let [file-id (get index id) - data (-> data - (blob/decode) - (assoc :id file-id) - (pmg/migrate-data) - (update :pages-index relink-shapes) - (update :components relink-shapes) - (update :media relink-media) - (d/without-nils) - (files/process-pointers pmap/clone) - (blob/encode))] - (files/persist-pointers! conn file-id) - data))))))) + media)) + + (update-fdata [fdata new-id] + (-> fdata + (blob/decode) + (assoc :id new-id) + (pmg/migrate-data) + (update :pages-index relink-shapes) + (update :components relink-shapes) + (update :media relink-media) + (d/without-nils) + (feat.fdata/process-pointers pmap/clone) + (blob/encode)))] + + (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id) + pmap/*tracked* (pmap/create-tracked)] + (let [new-id (get index id) + file (-> file + (assoc :id new-id) + (update :data update-fdata new-id))] + (feat.fdata/persist-pointers! cfg new-id) + file)))) (def sql:get-used-libraries "select flr.* @@ -139,7 +140,7 @@ and so.deleted_at is null") (defn duplicate-file* - [conn {:keys [profile-id file index project-id name flibs fmeds]} {:keys [reset-shared-flag]}] + [{:keys [::db/conn] :as cfg} {:keys [profile-id file index project-id name flibs fmeds]} {:keys [reset-shared-flag]}] (let [flibs (or flibs (db/exec! conn [sql:get-used-libraries (:id file)])) fmeds (or fmeds (db/exec! conn [sql:get-used-media-objects (:id file)])) @@ -182,7 +183,7 @@ (assoc :modified-at now) (assoc :ignore-sync-until ignore)) - file (process-file conn file index)] + file (process-file cfg index file)] (db/insert! conn :file file) (db/insert! conn :file-profile-rel @@ -201,13 +202,15 @@ file)) (defn duplicate-file - [conn {:keys [profile-id file-id] :as params}] + [{:keys [::db/conn] :as cfg} {:keys [profile-id file-id] :as params}] (let [file (db/get-by-id conn :file file-id) index {file-id (uuid/next)} params (assoc params :index index :file file)] (proj/check-edition-permissions! conn profile-id (:project-id file)) (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"]) - (-> (duplicate-file* conn params {:reset-shared-flag true}) + + ;; FIXME: why we decode the data here? + (-> (duplicate-file* cfg params {:reset-shared-flag true}) (update :data blob/decode) (update :features db/decode-pgarray #{})))) @@ -227,12 +230,11 @@ {::doc/added "1.16" ::webhooks/event? true ::sm/params schema:duplicate-project} - [{:keys [::db/pool] :as cfg} params] - (db/with-atomic [conn pool] - (duplicate-project conn (assoc params :profile-id (::rpc/profile-id params))))) + [cfg {:keys [::rpc/profile-id] :as params}] + (db/tx-run! cfg duplicate-project (assoc params :profile-id profile-id))) (defn duplicate-project - [conn {:keys [profile-id project-id name] :as params}] + [{:keys [::db/conn] :as cfg} {:keys [profile-id project-id name] :as params}] ;; Defer all constraints (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"]) @@ -270,7 +272,7 @@ (let [file (db/get-by-id conn :file id) params (assoc params :file file) opts {:reset-shared-flag false}] - (duplicate-file* conn params opts)))) + (duplicate-file* cfg params opts)))) ;; return the created project project)) diff --git a/backend/src/app/rpc/commands/viewer.clj b/backend/src/app/rpc/commands/viewer.clj index c3e7d8608..c2887ef96 100644 --- a/backend/src/app/rpc/commands/viewer.clj +++ b/backend/src/app/rpc/commands/viewer.clj @@ -29,7 +29,7 @@ (defn- get-view-only-bundle [{:keys [::db/conn] :as cfg} {:keys [profile-id file-id ::perms] :as params}] - (let [file (files/get-file conn file-id) + (let [file (files/get-file cfg file-id) project (db/get conn :project {:id (:project-id file)} diff --git a/backend/src/app/srepl/helpers.clj b/backend/src/app/srepl/helpers.clj index 9229006fd..d0a067427 100644 --- a/backend/src/app/srepl/helpers.clj +++ b/backend/src/app/srepl/helpers.clj @@ -23,6 +23,7 @@ [app.config :as cfg] [app.db :as db] [app.db.sql :as sql] + [app.features.fdata :as feat.fdata] [app.main :refer [system]] [app.rpc.commands.files :as files] [app.rpc.commands.files-update :as files-update] @@ -39,7 +40,6 @@ [promesa.exec :as px] [promesa.exec.csp :as sp])) -(def ^:dynamic *conn* nil) (def ^:dynamic *system* nil) (defn println! @@ -63,71 +63,75 @@ (defn get-file "Get the migrated data of one file." - [system id] + [system id & {:keys [migrate?] :or {migrate? true}}] (db/run! system - (fn [{:keys [::db/conn]}] - (binding [pmap/*load-fn* (partial files/load-pointer conn id)] - (-> (files/get-file conn id) - (files/process-pointers deref)))))) + (fn [system] + (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)] + (-> (files/get-file system id :migrate? migrate?) + (update :data feat.fdata/process-pointers deref)))))) (defn validate "Validate structure, referencial integrity and semantic coherence of all contents of a file. Returns a list of errors." [system id] - (db/with-atomic [conn (:app.db/pool system)] - (binding [pmap/*load-fn* (partial files/load-pointer conn id)] - (let [file (files/get-file conn id) - libraries (->> (files/get-file-libraries conn id) - (into [file] (map (fn [{:keys [id]}] - (binding [pmap/*load-fn* (partial files/load-pointer conn id)] - (-> (db/get conn :file {:id id}) - (files/decode-row) - (files/process-pointers deref) ; ensure all pointers resolved - (pmg/migrate-file)))))) - (d/index-by :id))] - (validate/validate-file file libraries))))) + (db/tx-run! system + (fn [{:keys [::db/conn] :as system}] + (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)] + (let [id (if (string? id) (parse-uuid id) id) + file (files/get-file system id) + libs (->> (files/get-file-libraries conn id) + (into [file] (map (fn [{:keys [id]}] + (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)] + (-> (files/get-file system id :migrate? false) + (feat.fdata/process-pointers deref) + (pmg/migrate-file)))))) + (d/index-by :id))] + (validate/validate-file file libs)))))) -(defn repair +(defn repair! "Repair the list of errors detected by validation." [system id] - (db/with-atomic [conn (:app.db/pool system)] - (let [file (files/get-file conn id)] - (binding [*conn* conn - pmap/*tracked* (atom {}) - pmap/*load-fn* (partial files/load-pointer conn id)] - (let [libraries (->> (files/get-file-libraries conn id) - (into [file] (map (fn [{:keys [id]}] - (binding [pmap/*load-fn* (partial files/load-pointer conn id)] - (-> (db/get conn :file {:id id}) - (files/decode-row) - (files/process-pointers deref) ; ensure all pointers resolved - (pmg/migrate-file)))))) - (d/index-by :id)) - errors (validate/validate-file file libraries) - changes (-> (repair/repair-file (:data file) libraries errors) - (get :redo-changes)) - file (-> file - (update :revn inc) - (update :data cpc/process-changes changes) - (update :data blob/encode))] + (db/tx-run! system + (fn [{:keys [::db/conn] :as system}] + (binding [pmap/*tracked* (pmap/create-tracked) + pmap/*load-fn* (partial feat.fdata/load-pointer system id)] + (let [id (if (string? id) (parse-uuid id) id) + file (files/get-file system id) + libs (->> (files/get-file-libraries conn id) + (into [file] (map (fn [{:keys [id]}] + (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)] + (-> (files/get-file system id :migrate? false) + (feat.fdata/process-pointers deref) + (pmg/migrate-file)))))) + (d/index-by :id)) + errors (validate/validate-file file libs) + changes (-> (repair/repair-file (:data file) libs errors) :redo-changes) - (files/persist-pointers! conn id) - (db/update! conn :file - {:revn (:revn file) - :data (:data file) - :data-backend nil - :modified-at (dt/now) - :has-media-trimmed false} - {:id (:id file)})))))) + file (-> file + (update :revn inc) + (update :data cpc/process-changes changes) + (update :data blob/encode))] + + (when (contains? (:features file) "fdata/pointer-map") + (feat.fdata/persist-pointers! system id)) + + (db/update! conn :file + {:revn (:revn file) + :data (:data file) + :data-backend nil + :modified-at (dt/now) + :has-media-trimmed false} + {:id (:id file)}) + :repaired))))) (defn update-file! "Apply a function to the data of one file. Optionally save the changes or not. The function receives the decoded and migrated file data." [system & {:keys [update-fn id rollback? migrate? inc-revn?] :or {rollback? true migrate? true inc-revn? true}}] - (letfn [(process-file [conn {:keys [features] :as file}] - (binding [pmap/*tracked* (atom {}) - pmap/*load-fn* (partial files/load-pointer conn id) + (letfn [(process-file [{:keys [::db/conn] :as system} {:keys [features] :as file}] + (binding [pmap/*tracked* (pmap/create-tracked) + pmap/*load-fn* (partial feat.fdata/load-pointer system id) cfeat/*wrap-with-pointer-map-fn* (if (contains? features "fdata/pointer-map") pmap/wrap identity) cfeat/*wrap-with-objects-map-fn* @@ -145,19 +149,19 @@ {:id id})) (when (contains? (:features file) "fdata/pointer-map") - (files/persist-pointers! conn id)) + (feat.fdata/persist-pointers! system id)) (dissoc file :data)))] (db/tx-run! system - (fn [{:keys [::db/conn] :as system}] - (binding [*conn* conn *system* system] + (fn [system] + (binding [*system* system] (try - (->> (files/get-file conn id :migrate? migrate?) - (process-file conn)) + (->> (files/get-file system id :migrate? migrate?) + (process-file system)) (finally (when rollback? - (db/rollback! conn))))))))) + (db/rollback! system))))))))) (defn analyze-files "Apply a function to all files in the database, reading them in @@ -187,17 +191,17 @@ (println "unexpected exception happened on processing file: " (:id file)) (strace/print-stack-trace cause)) - (process-file [conn file-id] - (let [file (binding [pmap/*load-fn* (partial files/load-pointer conn file-id)] - (-> (files/get-file conn file-id) - (files/process-pointers deref))) + (process-file [{:keys [::db/conn] :as system} file-id] + (let [file (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system file-id)] + (-> (files/get-file system file-id) + (update :data feat.fdata/process-pointers deref))) libs (when with-libraries? (->> (files/get-file-libraries conn file-id) (into [file] (map (fn [{:keys [id]}] - (binding [pmap/*load-fn* (partial files/load-pointer conn id)] - (-> (files/get-file conn id) - (files/process-pointers deref)))))) + (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)] + (-> (files/get-file system id) + (update :data feat.fdata/process-pointers deref)))))) (d/index-by :id)))] (try (if with-libraries? @@ -209,31 +213,31 @@ (db/tx-run! system (fn [{:keys [::db/conn] :as system}] (try - (binding [*conn* conn *system* system] + (binding [*system* system] (when (fn? on-init) (on-init)) - (run! (partial process-file conn) (get-candidates conn))) + (run! (partial process-file system) (get-candidates conn))) (finally (when (fn? on-end) (ex/ignoring (on-end))) - (db/rollback! conn))))))) + (db/rollback! system))))))) (defn process-files! "Apply a function to all files in the database, reading them in batches." - [{:keys [::db/pool] :as system} & {:keys [chunk-size - max-items - workers - start-at - on-file - on-error - on-end - on-init - rollback?] - :or {chunk-size 10 - max-items Long/MAX_VALUE - workers 1 - rollback? true}}] + [system & {:keys [chunk-size + max-items + workers + start-at + on-file + on-error + on-end + on-init + rollback?] + :or {chunk-size 10 + max-items Long/MAX_VALUE + workers 1 + rollback? true}}] (letfn [(get-chunk [conn cursor] (let [sql (str "SELECT id, created_at FROM file " " WHERE created_at < ? AND deleted_at is NULL " @@ -252,11 +256,11 @@ (println! "unexpected exception happened on processing file: " (:id file)) (strace/print-stack-trace cause)) - (process-file [conn file-id] + (process-file [system file-id] (try - (let [{:keys [features] :as file} (files/get-file conn file-id)] - (binding [pmap/*tracked* (atom {}) - pmap/*load-fn* (partial files/load-pointer conn file-id) + (let [{:keys [features] :as file} (files/get-file system file-id)] + (binding [pmap/*tracked* (pmap/create-tracked) + pmap/*load-fn* (partial feat.fdata/load-pointer system file-id) cfeat/*wrap-with-pointer-map-fn* (if (contains? features "fdata/pointer-map") pmap/wrap identity) cfeat/*wrap-with-objects-map-fn* @@ -265,30 +269,30 @@ (on-file file) (when (contains? features "fdata/pointer-map") - (files/persist-pointers! conn file-id)))) + (feat.fdata/persist-pointers! system file-id)))) (catch Throwable cause ((or on-error on-error*) cause file-id)))) (run-worker [in index] (db/tx-run! system - (fn [{:keys [::db/conn] :as system}] - (binding [*conn* conn *system* system] + (fn [system] + (binding [*system* system] (loop [i 0] (when-let [file-id (sp/take! in)] (println! "=> worker: index:" index "| loop:" i "| file:" (str file-id) "|" (px/get-name)) - (process-file conn file-id) + (process-file system file-id) (recur (inc i))))) (when rollback? - (db/rollback! conn))))) + (db/rollback! system))))) (run-producer [input] - (db/with-atomic [conn pool] - (doseq [file-id (get-candidates conn)] - (println! "=> producer:" file-id "|" (px/get-name)) - (sp/put! input file-id)) - (sp/close! input)))] + (db/tx-run! system (fn [{:keys [::db/conn]}] + (doseq [file-id (get-candidates conn)] + (println! "=> producer:" file-id "|" (px/get-name)) + (sp/put! input file-id)) + (sp/close! input))))] (when (fn? on-init) (on-init)) diff --git a/backend/src/app/tasks/file_gc.clj b/backend/src/app/tasks/file_gc.clj index e7eb3a232..5e97bdabc 100644 --- a/backend/src/app/tasks/file_gc.clj +++ b/backend/src/app/tasks/file_gc.clj @@ -19,8 +19,8 @@ [app.common.types.shape-tree :as ctt] [app.config :as cf] [app.db :as db] + [app.features.fdata :as feat.fdata] [app.media :as media] - [app.rpc.commands.files :as files] [app.storage :as sto] [app.util.blob :as blob] [app.util.pointer-map :as pmap] @@ -271,9 +271,9 @@ " limit 1;") rows (db/exec! conn [sql file-id cursor])] [(some-> rows peek :created-at) - (mapcat (comp files/get-all-pointer-ids blob/decode :data) rows)]))] + (mapcat (comp feat.fdata/get-used-pointer-ids blob/decode :data) rows)]))] - (let [used (into (files/get-all-pointer-ids data) + (let [used (into (feat.fdata/get-used-pointer-ids data) (d/iteration get-pointers-chunk :vf second :kf first @@ -290,10 +290,10 @@ (defn- process-file [{:keys [::db/conn] :as cfg} {:keys [id data revn modified-at features] :as file}] - (l/dbg :hint "processing file" :id id :modified-at modified-at) + (l/dbg :hint "processing file" :file-id (str id) :modified-at modified-at) - (binding [pmap/*load-fn* (partial files/load-pointer conn id) - pmap/*tracked* (atom {})] + (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id) + pmap/*tracked* (pmap/create-tracked)] (let [data (-> (blob/decode data) (assoc :id id) (pmg/migrate-data))] @@ -311,4 +311,4 @@ {:has-media-trimmed true} {:id id}) - (files/persist-pointers! conn id)))) + (feat.fdata/persist-pointers! cfg id))))