🎉 Integrate storage/pointer-map file feature

backend/src/app/http/debug.clj

@@ -16,7 +16,7 @@
    [app.http.middleware :as mw]
    [app.http.session :as session]
    [app.rpc.commands.binfile :as binf]
-   [app.rpc.mutations.files :refer [create-file]]
+   [app.rpc.commands.files.create :refer [create-file]]
    [app.rpc.queries.profile :as profile]
    [app.util.blob :as blob]
    [app.util.template :as tmpl]

backend/src/app/migrations.clj

@@ -257,6 +257,11 @@
    {:name "0082-add-features-column-to-file-table"
     :fn (mg/resource "app/migrations/sql/0082-add-features-column-to-file-table.sql")}
 
+   {:name "0083-add-file-data-fragment-table"
+    :fn (mg/resource "app/migrations/sql/0083-add-file-data-fragment-table.sql")}
+
+   {:name "0084-add-features-column-to-file-change-table"
+    :fn (mg/resource "app/migrations/sql/0084-add-features-column-to-file-change-table.sql")}
    ])
 
 

backend/src/app/migrations/sql/0083-add-file-data-fragment-table.sql

@@ -0,0 +1,15 @@
+CREATE TABLE file_data_fragment (
+  id uuid NOT NULL,
+  file_id uuid NOT NULL REFERENCES file(id) ON DELETE CASCADE DEFERRABLE,
+
+  created_at timestamptz NOT NULL DEFAULT now(),
+
+  metadata jsonb NULL,
+  content bytea NOT NULL,
+
+  PRIMARY KEY (file_id, id)
+);
+
+ALTER TABLE file_data_fragment
+  ALTER COLUMN metadata SET STORAGE external,
+  ALTER COLUMN content SET STORAGE external;

backend/src/app/migrations/sql/0084-add-features-column-to-file-change-table.sql

@@ -0,0 +1,8 @@
+ALTER TABLE file_change
+  ADD COLUMN features text[] DEFAULT NULL;
+
+ALTER TABLE file_change
+  ALTER COLUMN features SET STORAGE external;
+
+ALTER TABLE file
+  ALTER COLUMN features SET STORAGE external;

backend/src/app/rpc.clj

@@ -230,7 +230,10 @@
                      'app.rpc.commands.auth
                      'app.rpc.commands.ldap
                      'app.rpc.commands.demo
-                     'app.rpc.commands.files)
+                     'app.rpc.commands.files
+                     'app.rpc.commands.files.update
+                     'app.rpc.commands.files.create
+                     'app.rpc.commands.files.temp)
          (map (partial process-method cfg))
          (into {}))))
 

backend/src/app/rpc/commands/binfile.clj

@@ -17,14 +17,15 @@
    [app.db :as db]
    [app.media :as media]
    [app.rpc :as-alias rpc]
+   [app.rpc.commands.files :as files]
    [app.rpc.doc :as-alias doc]
-   [app.rpc.queries.files :as files]
    [app.rpc.queries.projects :as projects]
    [app.storage :as sto]
    [app.storage.tmp :as tmp]
    [app.tasks.file-gc]
    [app.util.blob :as blob]
    [app.util.fressian :as fres]
+   [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
    [app.util.time :as dt]
    [clojure.spec.alpha :as s]
@@ -290,9 +291,11 @@
 
 (defn- retrieve-file
   [pool file-id]
-  (->> (db/query pool :file {:id file-id})
+  (with-open [conn (db/open pool)]
-       (map files/decode-row)
+    (binding [pmap/*load-fn* (partial files/load-pointer conn file-id)]
-       (first)))
+      (some-> (db/get* conn :file {:id file-id})
+              (files/decode-row)
+              (update :data files/process-pointers deref)))))
 
 (def ^:private sql:file-media-objects
   "SELECT * FROM file_media_object WHERE id = ANY(?)")

backend/src/app/rpc/commands/comments.clj

@@ -10,8 +10,8 @@
    [app.common.geom.point :as gpt]
    [app.common.spec :as us]
    [app.db :as db]
+   [app.rpc.commands.files :as files]
    [app.rpc.doc :as-alias doc]
-   [app.rpc.queries.files :as files]
    [app.rpc.queries.teams :as teams]
    [app.rpc.retry :as retry]
    [app.util.blob :as blob]

backend/src/app/rpc/commands/files.clj

@@ -6,18 +6,312 @@
 
 (ns app.rpc.commands.files
   (:require
+   [app.common.data :as d]
+   [app.common.data.macros :as dm]
+   [app.common.exceptions :as ex]
+   [app.common.geom.shapes :as gsh]
+   [app.common.pages.helpers :as cph]
+   [app.common.pages.migrations :as pmg]
    [app.common.spec :as us]
+   [app.common.types.file :as ctf]
+   [app.common.types.shape-tree :as ctt]
    [app.db :as db]
+   [app.db.sql :as sql]
+   [app.rpc :as-alias rpc]
+   [app.rpc.commands.files.thumbnails :as-alias thumbs]
    [app.rpc.doc :as-alias doc]
-   [app.rpc.queries.files :as files]
+   [app.rpc.helpers :as rpch]
+   [app.rpc.permissions :as perms]
+   [app.rpc.queries.projects :as projects]
+   [app.rpc.queries.share-link :refer [retrieve-share-link]]
+   [app.rpc.queries.teams :as teams]
+   [app.util.blob :as blob]
+   [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
-   [clojure.spec.alpha :as s]))
+   [app.util.time :as dt]
+   [clojure.set :as set]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]))
+
+;; --- FEATURES
+
+(def supported-features
+  #{"storage/objects-map"
+    "storage/pointer-map"
+    "components/v2"})
+
+(def default-features #{})
+
+;; --- SPECS
+
+(s/def ::features ::us/set-of-strings)
+(s/def ::file-id ::us/uuid)
+(s/def ::frame-id ::us/uuid)
+(s/def ::id ::us/uuid)
+(s/def ::is-shared ::us/boolean)
+(s/def ::name ::us/string)
+(s/def ::profile-id ::us/uuid)
+(s/def ::project-id ::us/uuid)
+(s/def ::search-term ::us/string)
+(s/def ::team-id ::us/uuid)
+
+(defn decode-row
+  [{:keys [data changes features] :as row}]
+  (when row
+    (cond-> row
+      features (assoc :features (db/decode-pgarray features #{}))
+      changes  (assoc :changes (blob/decode changes))
+      data     (assoc :data (blob/decode data)))))
+
+;; --- FILE PERMISSIONS
+
+(def ^:private sql:file-permissions
+  "select fpr.is_owner,
+          fpr.is_admin,
+          fpr.can_edit
+     from file_profile_rel as fpr
+    where fpr.file_id = ?
+      and fpr.profile_id = ?
+   union all
+   select tpr.is_owner,
+          tpr.is_admin,
+          tpr.can_edit
+     from team_profile_rel as tpr
+    inner join project as p on (p.team_id = tpr.team_id)
+    inner join file as f on (p.id = f.project_id)
+    where f.id = ?
+      and tpr.profile_id = ?
+   union all
+   select ppr.is_owner,
+          ppr.is_admin,
+          ppr.can_edit
+     from project_profile_rel as ppr
+    inner join file as f on (f.project_id = ppr.project_id)
+    where f.id = ?
+      and ppr.profile_id = ?")
+
+(defn retrieve-file-permissions
+  [conn profile-id file-id]
+  (when (and profile-id file-id)
+    (db/exec! conn [sql:file-permissions
+                    file-id profile-id
+                    file-id profile-id
+                    file-id profile-id])))
+
+(defn get-permissions
+  ([conn profile-id file-id]
+   (let [rows     (retrieve-file-permissions conn profile-id file-id)
+         is-owner (boolean (some :is-owner rows))
+         is-admin (boolean (some :is-admin rows))
+         can-edit (boolean (some :can-edit rows))]
+     (when (seq rows)
+       {:type :membership
+        :is-owner is-owner
+        :is-admin (or is-owner is-admin)
+        :can-edit (or is-owner is-admin can-edit)
+        :can-read true
+        :is-logged (some? profile-id)})))
+  ([conn profile-id file-id share-id]
+   (let [perms  (get-permissions conn profile-id file-id)
+         ldata  (retrieve-share-link conn file-id share-id)]
+
+     ;; NOTE: in a future when share-link becomes more powerful and
+     ;; will allow us specify which parts of the app is available, we
+     ;; will probably need to tweak this function in order to expose
+     ;; this flags to the frontend.
+     (cond
+       (some? perms) perms
+       (some? ldata) {:type :share-link
+                      :can-read true
+                      :is-logged (some? profile-id)
+                      :who-comment (:who-comment ldata)
+                      :who-inspect (:who-inspect ldata)}))))
+
+(def has-edit-permissions?
+  (perms/make-edition-predicate-fn get-permissions))
+
+(def has-read-permissions?
+  (perms/make-read-predicate-fn get-permissions))
+
+(def has-comment-permissions?
+  (perms/make-comment-predicate-fn get-permissions))
+
+(def check-edition-permissions!
+  (perms/make-check-fn has-edit-permissions?))
+
+(def check-read-permissions!
+  (perms/make-check-fn has-read-permissions?))
+
+;; A user has comment permissions if she has read permissions, or comment permissions
+(defn check-comment-permissions!
+  [conn profile-id file-id share-id]
+  (let [can-read (has-read-permissions? conn profile-id file-id)
+        can-comment  (has-comment-permissions? conn profile-id file-id share-id)]
+    (when-not (or can-read can-comment)
+      (ex/raise :type :not-found
+                :code :object-not-found
+                :hint "not found"))))
+
+;; --- HELPERS
+
+(defn retrieve-team-id
+  [conn project-id]
+  (:team-id (db/get-by-id conn :project project-id {:columns [:team-id]})))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; FEATURES: pointer-map
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn check-features-compatibility!
+  [features]
+  (let [not-supported (set/difference features supported-features)]
+    (when (seq not-supported)
+      (ex/raise :type :restriction
+                :code :features-not-supported
+                :feature (first not-supported)
+                :hint (format "features %s not supported" (str/join "," not-supported))))
+    features))
+
+(defn load-pointer
+  [conn file-id id]
+  (let [row (db/get conn :file-data-fragment
+                    {:id id :file-id file-id}
+                    {:columns [:content]
+                     :check-deleted? false})]
+    (blob/decode (:content row))))
+
+(defn persist-pointers!
+  [conn file-id]
+  (doseq [[id item] @pmap/*tracked*]
+    (when (pmap/modified? item)
+      (let [content (-> item deref blob/encode)]
+        (db/insert! conn :file-data-fragment
+                    {:id id
+                     :file-id file-id
+                     :content content})))))
+
+(defn process-pointers
+  [file update-fn]
+  (update file :data (fn resolve-fn [data]
+                       (cond-> data
+                         (contains? data :pages-index)
+                         (update :pages-index resolve-fn)
+
+                         :always
+                         (update-vals (fn [val]
+                                        (if (pmap/pointer-map? val)
+                                          (update-fn val)
+                                          val)))))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; QUERY COMMANDS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-;; --- Query: File Libraries used by a File
+;; --- COMMAND QUERY: get-file (by id)
+
+(defn retrieve-object-thumbnails
+  ([conn file-id]
+   (let [sql (str/concat
+              "select object_id, data "
+              "  from file_object_thumbnail"
+              " where file_id=?")]
+     (->> (db/exec! conn [sql file-id])
+          (d/index-by :object-id :data))))
+
+  ([conn file-id object-ids]
+   (let [sql (str/concat
+              "select object_id, data "
+              "  from file_object_thumbnail"
+              " where file_id=? and object_id = ANY(?)")
+         ids (db/create-array conn "text" (seq object-ids))]
+     (->> (db/exec! conn [sql file-id ids])
+          (d/index-by :object-id :data)))))
+
+(defn retrieve-file
+  [conn id client-features]
+  ;; here we check if client requested features are supported
+  (check-features-compatibility! client-features)
+
+  (binding [pmap/*load-fn* (partial load-pointer conn id)]
+    (let [file     (->> (db/get-by-id conn :file id)
+                        (decode-row)
+                        (pmg/migrate-file))
+          features (:features file)
+          file     (cond-> file
+                     (and (contains? client-features "components/v2")
+                          (not (contains? features "components/v2")))
+                     (update :data ctf/migrate-to-components-v2)
+
+                     (and (contains? features "storage/pointer-map")
+                          (not (contains? client-features "storage/pointer-map")))
+                     (process-pointers deref))]
+
+      (when (and (contains? features "components/v2")
+                 (not (contains? client-features "components/v2")))
+        (ex/raise :type :restriction
+                  :code :feature-mismatch
+                  :feature "components/v2"
+                  :hint "file has 'components/v2' feature enabled but frontend didn't specifies it"))
+
+      file)))
+
+(defn get-file
+  [conn id features]
+  (let [file   (retrieve-file conn id features)
+        thumbs (retrieve-object-thumbnails conn id)]
+    (assoc file :thumbnails thumbs)
+    #_file))
+
+(s/def ::get-file
+  (s/keys :req-un [::profile-id ::id]
+          :opt-un [::features]))
+
+
+;; TODO: this should be changed probably because thumbnails will not be included
+
+(sv/defmethod ::get-file
+  "Retrieve a file by its ID. Only authenticated users."
+  {::doc/added "1.17"}
+  [{:keys [pool] :as cfg} {:keys [profile-id id features] :as params}]
+  (with-open [conn (db/open pool)]
+    (let [perms (get-permissions conn profile-id id)]
+      (check-read-permissions! perms)
+      (-> (get-file conn id features)
+          (assoc :permissions perms)))))
+
+
+;; --- COMMAND QUERY: get-project-files
+
+(def ^:private sql:project-files
+  "select f.id,
+          f.project_id,
+          f.created_at,
+          f.modified_at,
+          f.name,
+          f.revn,
+          f.is_shared
+     from file as f
+    where f.project_id = ?
+      and f.deleted_at is null
+    order by f.modified_at desc")
+
+(s/def ::get-project-files
+  (s/keys :req-un [::profile-id ::project-id]))
+
+(defn get-project-files
+  [conn project-id]
+  (db/exec! conn [sql:project-files project-id]))
+
+(sv/defmethod ::get-project-files
+  "Get all files for the specified project."
+  {::doc/added "1.17"}
+  [{:keys [pool] :as cfg} {:keys [profile-id project-id] :as params}]
+  (with-open [conn (db/open pool)]
+    (projects/check-read-permissions! conn profile-id project-id)
+    (get-project-files conn project-id)))
+
+
+;; --- COMMAND QUERY: has-file-libraries
 
 (declare retrieve-has-file-libraries)
 
@@ -32,7 +326,7 @@
   {::doc/added "1.15.1"}
   [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}]
   (with-open [conn (db/open pool)]
-    (files/check-read-permissions! pool profile-id file-id)
+    (check-read-permissions! pool profile-id file-id)
     (retrieve-has-file-libraries conn params)))
 
 (def ^:private sql:has-file-libraries
@@ -48,3 +342,586 @@
   (let [row (db/exec-one! conn [sql:has-file-libraries file-id])]
     (:has-libraries row)))
 
+
+;; --- QUERY COMMAND: get-page
+
+(defn- prune-objects
+  "Given the page data and the object-id returns the page data with all
+  other not needed objects removed from the `:objects` data
+  structure."
+  [{:keys [objects] :as page} object-id]
+  (let [objects (cph/get-children-with-self objects object-id)]
+     (assoc page :objects (d/index-by :id objects))))
+
+(defn- prune-thumbnails
+  "Given the page data, removes the `:thumbnail` prop from all
+  shapes."
+  [page]
+  (update page :objects d/update-vals #(dissoc % :thumbnail)))
+
+(defn get-page
+  [conn {:keys [file-id page-id object-id features]}]
+  (let [file     (retrieve-file conn file-id features)
+        page-id  (or page-id (-> file :data :pages first))
+        page     (dm/get-in file [:data :pages-index page-id])]
+    (cond-> (prune-thumbnails page)
+      (uuid? object-id)
+      (prune-objects object-id))))
+
+(s/def ::page-id ::us/uuid)
+(s/def ::object-id ::us/uuid)
+(s/def ::get-page
+  (s/and
+   (s/keys :req-un [::profile-id ::file-id]
+           :opt-un [::page-id ::object-id ::features])
+   (fn [obj]
+     (if (contains? obj :object-id)
+       (contains? obj :page-id)
+       true))))
+
+(sv/defmethod ::get-page
+  "Retrieves the page data from file and returns it. If no page-id is
+  specified, the first page will be returned. If object-id is
+  specified, only that object and its children will be returned in the
+  page objects data structure.
+
+  If you specify the object-id, the page-id parameter becomes
+  mandatory.
+
+  Mainly used for rendering purposes."
+  {::doc/added "1.17"}
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}]
+  (with-open [conn (db/open pool)]
+    (check-read-permissions! conn profile-id file-id)
+    (get-page conn params)))
+
+
+;; --- COMMAND QUERY: get-team-shared-files
+
+(def ^:private sql:team-shared-files
+  "select f.id,
+          f.revn,
+          f.data,
+          f.project_id,
+          f.created_at,
+          f.modified_at,
+          f.name,
+          f.is_shared
+     from file as f
+    inner join project as p on (p.id = f.project_id)
+    where f.is_shared = true
+      and f.deleted_at is null
+      and p.deleted_at is null
+      and p.team_id = ?
+    order by f.modified_at desc")
+
+(defn get-team-shared-files
+  [conn {:keys [team-id] :as params}]
+  (let [assets-sample
+        (fn [assets limit]
+          (let [sorted-assets (->> (vals assets)
+                                   (sort-by #(str/lower (:name %))))]
+
+            {:count (count sorted-assets)
+             :sample (into [] (take limit sorted-assets))}))
+
+        library-summary
+        (fn [data]
+          {:components (assets-sample (:components data) 4)
+           :colors (assets-sample (:colors data) 3)
+           :typographies (assets-sample (:typographies data) 3)})
+
+        xform (comp
+               (map decode-row)
+               (map #(assoc % :library-summary (library-summary (:data %))))
+               (map #(dissoc % :data)))]
+
+    (into #{} xform (db/exec! conn [sql:team-shared-files team-id]))))
+
+(s/def ::get-team-shared-files
+  (s/keys :req-un [::profile-id ::team-id]))
+
+(sv/defmethod ::get-team-shared-files
+  "Get all file (libraries) for the specified team."
+  {::doc/added "1.17"}
+  [{:keys [pool] :as cfg} params]
+  (with-open [conn (db/open pool)]
+    (get-team-shared-files conn params)))
+
+
+;; --- COMMAND QUERY: get-file-libraries
+
+(def ^:private sql:file-libraries
+  "WITH RECURSIVE libs AS (
+     SELECT fl.*, flr.synced_at
+       FROM file AS fl
+       JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
+      WHERE flr.file_id = ?::uuid
+    UNION
+     SELECT fl.*, flr.synced_at
+       FROM file AS fl
+       JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
+       JOIN libs AS l ON (flr.file_id = l.id)
+   )
+   SELECT l.id,
+          l.data,
+          l.project_id,
+          l.created_at,
+          l.modified_at,
+          l.deleted_at,
+          l.name,
+          l.revn,
+          l.synced_at
+     FROM libs AS l
+    WHERE l.deleted_at IS NULL OR l.deleted_at > now();")
+
+(defn get-file-libraries
+  [conn is-indirect file-id]
+  (let [xform (comp
+               (map #(assoc % :is-indirect is-indirect))
+               (map decode-row))]
+    (into #{} xform (db/exec! conn [sql:file-libraries file-id]))))
+
+(s/def ::get-file-libraries
+  (s/keys :req-un [::profile-id ::file-id]))
+
+(sv/defmethod ::get-file-libraries
+  "Get libraries used by the specified file."
+  {::doc/added "1.17"}
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}]
+  (with-open [conn (db/open pool)]
+    (check-read-permissions! conn profile-id file-id)
+    (get-file-libraries conn false file-id)))
+
+
+;; --- COMMAND QUERY: Files that use this File library
+
+(def ^:private sql:library-using-files
+  "SELECT f.id,
+          f.name
+     FROM file_library_rel AS flr
+     JOIN file AS f ON (f.id = flr.file_id)
+    WHERE flr.library_file_id = ?
+      AND (f.deleted_at IS NULL OR f.deleted_at > now())")
+
+(defn get-library-file-references
+  [conn file-id]
+  (db/exec! conn [sql:library-using-files file-id]))
+
+(s/def ::get-library-file-references
+  (s/keys :req-un [::profile-id ::file-id]))
+
+(sv/defmethod ::get-library-file-references
+  "Returns all the file references that use specified file (library) id."
+  {::doc/added "1.17"}
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}]
+  (with-open [conn (db/open pool)]
+    (check-read-permissions! conn profile-id file-id)
+    (get-library-file-references conn file-id)))
+
+
+;; --- COMMAND QUERY: get-team-recent-files
+
+(def sql:team-recent-files
+  "with recent_files as (
+     select f.id,
+            f.revn,
+            f.project_id,
+            f.created_at,
+            f.modified_at,
+            f.name,
+            f.is_shared,
+            row_number() over w as row_num
+       from file as f
+       join project as p on (p.id = f.project_id)
+      where p.team_id = ?
+        and p.deleted_at is null
+        and f.deleted_at is null
+     window w as (partition by f.project_id order by f.modified_at desc)
+      order by f.modified_at desc
+   )
+   select * from recent_files where row_num <= 10;")
+
+(defn get-team-recent-files
+  [conn team-id]
+  (db/exec! conn [sql:team-recent-files team-id]))
+
+(s/def ::get-team-recent-files
+  (s/keys :req-un [::profile-id ::team-id]))
+
+(sv/defmethod ::get-team-recent-files
+  {::doc/added "1.17"}
+  [{:keys [pool] :as cfg} {:keys [profile-id team-id]}]
+  (with-open [conn (db/open pool)]
+    (teams/check-read-permissions! conn profile-id team-id)
+    (get-team-recent-files conn team-id)))
+
+
+;; --- COMMAND QUERY: get-file-thumbnail
+
+(defn get-file-thumbnail
+  [conn file-id revn]
+  (let [sql (sql/select :file-thumbnail
+                        (cond-> {:file-id file-id}
+                          revn (assoc :revn revn))
+                        {:limit 1
+                         :order-by [[:revn :desc]]})
+        row (db/exec-one! conn sql)]
+    (when-not row
+      (ex/raise :type :not-found
+                :code :file-thumbnail-not-found))
+
+    {:data (:data row)
+     :props (some-> (:props row) db/decode-transit-pgobject)
+     :revn (:revn row)
+     :file-id (:file-id row)}))
+
+(s/def ::revn ::us/integer)
+
+(s/def ::get-file-thumbnail
+  (s/keys :req-un [::profile-id ::file-id]
+          :opt-un [::revn]))
+
+(sv/defmethod ::get-file-thumbnail
+  {::doc/added "1.17"}
+  [{:keys [pool]} {:keys [profile-id file-id revn]}]
+  (with-open [conn (db/open pool)]
+    (check-read-permissions! conn profile-id file-id)
+    (-> (get-file-thumbnail conn file-id revn)
+        (with-meta {::rpc/transform-response (rpch/http-cache {:max-age (* 1000 60 60)})}))))
+
+
+
+;; --- COMMAND QUERY: get-file-data-for-thumbnail
+
+(defn get-file-data-for-thumbnail
+  [conn {:keys [data id] :as file}]
+  (letfn [;; function responsible on finding the frame marked to be
+          ;; used as thumbnail; the returned frame always have
+          ;; the :page-id set to the page that it belongs.
+          (get-thumbnail-frame [data]
+            (d/seek :use-for-thumbnail?
+                    (for [page  (-> data :pages-index vals)
+                          frame (-> page :objects ctt/get-frames)]
+                      (assoc frame :page-id (:id page)))))
+
+          ;; function responsible to filter objects data structure of
+          ;; all unneeded shapes if a concrete frame is provided. If no
+          ;; frame, the objects is returned untouched.
+          (filter-objects [objects frame-id]
+            (d/index-by :id (cph/get-children-with-self objects frame-id)))
+
+          ;; function responsible of assoc available thumbnails
+          ;; to frames and remove all children shapes from objects if
+          ;; thumbnails is available
+          (assoc-thumbnails [objects page-id thumbnails]
+            (loop [objects objects
+                   frames  (filter cph/frame-shape? (vals objects))]
+
+              (if-let [frame  (-> frames first)]
+                (let [frame-id (:id frame)
+                      object-id (str page-id frame-id)
+                      frame (if-let [thumb (get thumbnails object-id)]
+                              (assoc frame :thumbnail thumb :shapes [])
+                              (dissoc frame :thumbnail))
+
+                      children-ids
+                      (cph/get-children-ids objects frame-id)
+
+                      bounds
+                      (when (:show-content frame)
+                        (gsh/selection-rect (concat [frame] (->> children-ids (map (d/getf objects))))))
+
+                      frame
+                      (cond-> frame
+                        (some? bounds)
+                        (assoc :children-bounds bounds))]
+
+                  (if (:thumbnail frame)
+                    (recur (-> objects
+                               (assoc frame-id frame)
+                               (d/without-keys children-ids))
+                           (rest frames))
+                    (recur (assoc objects frame-id frame)
+                           (rest frames))))
+
+                objects)))]
+
+    (let [frame     (get-thumbnail-frame data)
+          frame-id  (:id frame)
+          page-id   (or (:page-id frame)
+                        (-> data :pages first))
+
+          page      (dm/get-in data [:pages-index page-id])
+          frame-ids (if (some? frame) (list frame-id) (map :id (ctt/get-frames (:objects page))))
+
+          obj-ids   (map #(str page-id %) frame-ids)
+          thumbs    (retrieve-object-thumbnails conn id obj-ids)]
+
+      (cond-> page
+        ;; If we have frame, we need to specify it on the page level
+        ;; and remove the all other unrelated objects.
+        (some? frame-id)
+        (-> (assoc :thumbnail-frame-id frame-id)
+            (update :objects filter-objects frame-id))
+
+        ;; Assoc the available thumbnails and prune not visible shapes
+        ;; for avoid transfer unnecessary data.
+        :always
+        (update :objects assoc-thumbnails page-id thumbs)))))
+
+(s/def ::get-file-data-for-thumbnail
+  (s/keys :req-un [::profile-id ::file-id]
+          :opt-un [::features]))
+
+(sv/defmethod ::get-file-data-for-thumbnail
+  "Retrieves the data for generate the thumbnail of the file. Used
+  mainly for render thumbnails on dashboard."
+  {::doc/added "1.17"}
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id features] :as props}]
+  (with-open [conn (db/open pool)]
+    (check-read-permissions! conn profile-id file-id)
+    (let [file (retrieve-file conn file-id features)]
+      {:file-id file-id
+       :revn (:revn file)
+       :page (get-file-data-for-thumbnail conn file)})))
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; MUTATION COMMANDS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; --- MUTATION COMMAND: rename-file
+
+(defn rename-file
+  [conn {:keys [id name] :as params}]
+  (-> (db/update! conn :file
+                  {:name name
+                   :modified-at (dt/now)}
+                  {:id id})
+      (select-keys [:id :name :created-at :modified-at])))
+
+(s/def ::rename-file
+  (s/keys :req-un [::profile-id ::name ::id]))
+
+(sv/defmethod ::rename-file
+  {::doc/added "1.17"}
+  [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}]
+  (db/with-atomic [conn pool]
+    (check-edition-permissions! conn profile-id id)
+    (rename-file conn params)))
+
+
+;; --- MUTATION COMMAND: set-file-shared
+
+(defn unlink-files
+  [conn {:keys [id] :as params}]
+  (db/delete! conn :file-library-rel {:library-file-id id}))
+
+(defn set-file-shared
+  [conn {:keys [id is-shared] :as params}]
+  (-> (db/update! conn :file
+                  {:is-shared is-shared}
+                  {:id id})
+      (select-keys [:id :name :is-shared])))
+
+(defn absorb-library
+  "Find all files using a shared library, and absorb all library assets
+  into the file local libraries"
+  [conn {:keys [id] :as params}]
+  (let [library (db/get-by-id conn :file id)]
+    (when (:is-shared library)
+      (let [ldata (-> library decode-row pmg/migrate-file :data)]
+        (->> (db/query conn :file-library-rel {:library-file-id id})
+             (map :file-id)
+             (keep #(db/get-by-id conn :file % {:check-deleted? false}))
+             (map decode-row)
+             (map pmg/migrate-file)
+             (run! (fn [{:keys [id data revn] :as file}]
+                     (let [data (ctf/absorb-assets data ldata)]
+                       (db/update! conn :file
+                                   {:revn (inc revn)
+                                    :data (blob/encode data)
+                                    :modified-at (dt/now)}
+                                   {:id id})))))))))
+
+(s/def ::set-file-shared
+  (s/keys :req-un [::profile-id ::id ::is-shared]))
+
+(sv/defmethod ::set-file-shared
+  {::doc/added "1.17"}
+  [{:keys [pool] :as cfg} {:keys [id profile-id is-shared] :as params}]
+  (db/with-atomic [conn pool]
+    (check-edition-permissions! conn profile-id id)
+    (when-not is-shared
+      (absorb-library conn params)
+      (unlink-files conn params))
+    (set-file-shared conn params)))
+
+
+;; --- MUTATION COMMAND: delete-file
+
+(defn mark-file-deleted
+  [conn {:keys [id] :as params}]
+  (db/update! conn :file
+              {:deleted-at (dt/now)}
+              {:id id})
+  nil)
+
+(s/def ::delete-file
+  (s/keys :req-un [::id ::profile-id]))
+
+(sv/defmethod ::delete-file
+  {::doc/added "1.17"}
+  [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}]
+  (db/with-atomic [conn pool]
+    (check-edition-permissions! conn profile-id id)
+    (absorb-library conn params)
+    (mark-file-deleted conn params)))
+
+
+;; --- MUTATION COMMAND: link-file-to-library
+
+(def sql:link-file-to-library
+  "insert into file_library_rel (file_id, library_file_id)
+   values (?, ?)
+       on conflict do nothing;")
+
+(defn link-file-to-library
+  [conn {:keys [file-id library-id] :as params}]
+  (db/exec-one! conn [sql:link-file-to-library file-id library-id]))
+
+(s/def ::link-file-to-library
+  (s/keys :req-un [::profile-id ::file-id ::library-id]))
+
+(sv/defmethod ::link-file-to-library
+  {::doc/added "1.17"}
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id library-id] :as params}]
+  (when (= file-id library-id)
+    (ex/raise :type :validation
+              :code :invalid-library
+              :hint "A file cannot be linked to itself"))
+  (db/with-atomic [conn pool]
+    (check-edition-permissions! conn profile-id file-id)
+    (check-edition-permissions! conn profile-id library-id)
+    (link-file-to-library conn params)))
+
+
+;; --- MUTATION COMMAND: unlink-file-from-library
+
+(defn unlink-file-from-library
+  [conn {:keys [file-id library-id] :as params}]
+  (db/delete! conn :file-library-rel
+              {:file-id file-id
+               :library-file-id library-id}))
+
+(s/def ::unlink-file-from-library
+  (s/keys :req-un [::profile-id ::file-id ::library-id]))
+
+(sv/defmethod ::unlink-file-from-library
+  {::doc/added "1.17"}
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}]
+  (db/with-atomic [conn pool]
+    (check-edition-permissions! conn profile-id file-id)
+    (unlink-file-from-library conn params)))
+
+
+;; --- MUTATION COMMAND: update-sync
+
+(defn update-sync
+  [conn {:keys [file-id library-id] :as params}]
+  (db/update! conn :file-library-rel
+              {:synced-at (dt/now)}
+              {:file-id file-id
+               :library-file-id library-id}))
+
+(s/def ::update-file-library-sync-status
+  (s/keys :req-un [::profile-id ::file-id ::library-id]))
+
+;; TODO: improve naming
+
+(sv/defmethod ::update-file-library-sync-status
+  "Update the synchronization statos of a file->library link"
+  {::doc/added "1.17"}
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}]
+  (db/with-atomic [conn pool]
+    (check-edition-permissions! conn profile-id file-id)
+    (update-sync conn params)))
+
+
+;; --- MUTATION COMMAND: ignore-sync
+
+(defn ignore-sync
+  [conn {:keys [file-id date] :as params}]
+  (db/update! conn :file
+              {:ignore-sync-until date}
+              {:id file-id}))
+
+(s/def ::ignore-file-library-sync-status
+  (s/keys :req-un [::profile-id ::file-id ::date]))
+
+;; TODO: improve naming
+(sv/defmethod ::ignore-file-library-sync-status
+  "Ignore updates in linked files"
+  {::doc/added "1.17"}
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}]
+  (db/with-atomic [conn pool]
+    (check-edition-permissions! conn profile-id file-id)
+    (ignore-sync conn params)))
+
+
+;; --- MUTATION COMMAND: upsert-file-object-thumbnail
+
+(def sql:upsert-object-thumbnail
+  "insert into file_object_thumbnail(file_id, object_id, data)
+   values (?, ?, ?)
+       on conflict(file_id, object_id) do
+          update set data = ?;")
+
+(defn upsert-file-object-thumbnail!
+  [conn {:keys [file-id object-id data]}]
+  (if data
+    (db/exec-one! conn [sql:upsert-object-thumbnail file-id object-id data data])
+    (db/delete! conn :file-object-thumbnail {:file-id file-id :object-id object-id})))
+
+(s/def ::data (s/nilable ::us/string))
+(s/def ::thumbs/object-id ::us/string)
+(s/def ::upsert-file-object-thumbnail
+  (s/keys :req-un [::profile-id ::file-id ::thumbs/object-id ::data]))
+
+(sv/defmethod ::upsert-file-object-thumbnail
+  {::doc/added "1.17"}
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}]
+  (db/with-atomic [conn pool]
+    (check-edition-permissions! conn profile-id file-id)
+    (upsert-file-object-thumbnail! conn params)
+    nil))
+
+;; --- MUTATION COMMAND: upsert-file-thumbnail
+
+(def sql:upsert-file-thumbnail
+  "insert into file_thumbnail (file_id, revn, data, props)
+   values (?, ?, ?, ?::jsonb)
+       on conflict(file_id, revn) do
+          update set data = ?, props=?, updated_at=now();")
+
+(defn upsert-file-thumbnail
+  [conn {:keys [file-id revn data props]}]
+  (let [props (db/tjson (or props {}))]
+    (db/exec-one! conn [sql:upsert-file-thumbnail
+                        file-id revn data props data props])))
+
+(s/def ::revn ::us/integer)
+(s/def ::props map?)
+(s/def ::upsert-file-thumbnail
+  (s/keys :req-un [::profile-id ::file-id ::revn ::data ::props]))
+
+(sv/defmethod ::upsert-file-thumbnail
+  "Creates or updates the file thumbnail. Mainly used for paint the
+  grid thumbnails."
+  {::doc/added "1.17"}
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}]
+  (db/with-atomic [conn pool]
+    (check-edition-permissions! conn profile-id file-id)
+    (upsert-file-thumbnail conn params)
+    nil))

backend/src/app/rpc/commands/files/create.clj

@@ -0,0 +1,83 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.rpc.commands.files.create
+  (:require
+   [app.common.data :as d]
+   [app.common.files.features :as ffeat]
+   [app.common.types.file :as ctf]
+   [app.common.uuid :as uuid]
+   [app.db :as db]
+   [app.loggers.audit :as audit]
+   [app.rpc.commands.files :as files]
+   [app.rpc.doc :as-alias doc]
+   [app.rpc.permissions :as perms]
+   [app.rpc.queries.projects :as proj]
+   [app.util.blob :as blob]
+   [app.util.objects-map :as omap]
+   [app.util.pointer-map :as pmap]
+   [app.util.services :as sv]
+   [clojure.spec.alpha :as s]))
+
+(defn create-file-role!
+  [conn {:keys [file-id profile-id role]}]
+  (let [params {:file-id file-id
+                :profile-id profile-id}]
+    (->> (perms/assign-role-flags params role)
+         (db/insert! conn :file-profile-rel))))
+
+(defn create-file
+  [conn {:keys [id name project-id is-shared data revn
+                modified-at deleted-at
+                ignore-sync-until features]
+         :or {is-shared false revn 0}
+         :as params}]
+  (let [id       (or id (:id data) (uuid/next))
+        features (-> (into files/default-features features)
+                     (files/check-features-compatibility!))
+
+        data     (or data
+                     (binding [ffeat/*current* features
+                               ffeat/*wrap-with-objects-map-fn* (if (features "storate/objects-map") omap/wrap identity)
+                               ffeat/*wrap-with-pointer-map-fn* (if (features "storage/pointer-map") pmap/wrap identity)]
+                       (ctf/make-file-data id)))
+
+        features (db/create-array conn "text" features)
+        file     (db/insert! conn :file
+                             (d/without-nils
+                              {:id id
+                               :project-id project-id
+                               :name name
+                               :revn revn
+                               :is-shared is-shared
+                               :data (blob/encode data)
+                               :features features
+                               :ignore-sync-until ignore-sync-until
+                               :modified-at modified-at
+                               :deleted-at deleted-at}))]
+
+    (->> (assoc params :file-id id :role :owner)
+         (create-file-role! conn))
+
+    (files/decode-row file)))
+
+(s/def ::create-file
+  (s/keys :req-un [::files/profile-id
+                   ::files/name
+                   ::files/project-id]
+          :opt-un [::files/id
+                   ::files/is-shared
+                   ::files/features]))
+
+(sv/defmethod ::create-file
+  {::doc/added "1.17"}
+  [{:keys [pool] :as cfg} {:keys [profile-id project-id] :as params}]
+  (db/with-atomic [conn pool]
+    (proj/check-edition-permissions! conn profile-id project-id)
+    (let [team-id (files/retrieve-team-id conn project-id)]
+      (-> (create-file conn params)
+          (vary-meta assoc ::audit/props {:team-id team-id})))))
+

backend/src/app/rpc/commands/files/temp.clj

@@ -0,0 +1,96 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.rpc.commands.files.temp
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.pages :as cp]
+   [app.common.uuid :as uuid]
+   [app.db :as db]
+   [app.rpc.commands.files :as files]
+   [app.rpc.commands.files.create :as files.create]
+   [app.rpc.commands.files.update :as files.update]
+   [app.rpc.doc :as-alias doc]
+   [app.rpc.queries.projects :as proj]
+   [app.util.blob :as blob]
+   [app.util.services :as sv]
+   [app.util.time :as dt]
+   [clojure.spec.alpha :as s]))
+
+;; --- MUTATION COMMAND: create-temp-file
+
+(s/def ::create-temp-file ::files.create/create-file)
+
+(sv/defmethod ::create-temp-file
+  {::doc/added "1.17"}
+  [{:keys [pool] :as cfg} {:keys [profile-id project-id] :as params}]
+  (db/with-atomic [conn pool]
+    (proj/check-edition-permissions! conn profile-id project-id)
+    (files.create/create-file conn (assoc params :deleted-at (dt/in-future {:days 1})))))
+
+;; --- MUTATION COMMAND: update-temp-file
+
+(defn update-temp-file
+  [conn {:keys [profile-id session-id id revn changes] :as params}]
+  (db/insert! conn :file-change
+              {:id (uuid/next)
+               :session-id session-id
+               :profile-id profile-id
+               :created-at (dt/now)
+               :file-id id
+               :revn revn
+               :data nil
+               :changes (blob/encode changes)}))
+
+(s/def ::update-temp-file
+  (s/keys :req-un [::files.update/changes
+                   ::files.update/revn
+                   ::files.update/session-id
+                   ::files/id]))
+
+(sv/defmethod ::update-temp-file
+  {::doc/added "1.17"}
+  [{:keys [pool] :as cfg} params]
+  (db/with-atomic [conn pool]
+    (update-temp-file conn params)
+    nil))
+
+;; --- MUTATION COMMAND: persist-temp-file
+
+(defn persist-temp-file
+  [conn {:keys [id] :as params}]
+  (let [file (db/get-by-id conn :file id)
+        revs (db/query conn :file-change
+                       {:file-id id}
+                       {:order-by [[:revn :asc]]})
+        revn (count revs)]
+
+    (when (nil? (:deleted-at file))
+      (ex/raise :type :validation
+                :code :cant-persist-already-persisted-file))
+
+    (loop [revs (seq revs)
+           data (blob/decode (:data file))]
+      (if-let [rev (first revs)]
+        (recur (rest revs)
+               (->> rev :changes blob/decode (cp/process-changes data)))
+        (db/update! conn :file
+                    {:deleted-at nil
+                     :revn revn
+                     :data (blob/encode data)}
+                    {:id id})))
+    nil))
+
+(s/def ::persist-temp-file
+  (s/keys :req-un [::files/id
+                   ::files/profile-id]))
+
+(sv/defmethod ::persist-temp-file
+  {::doc/added "1.17"}
+  [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}]
+  (db/with-atomic [conn pool]
+    (files/check-edition-permissions! conn profile-id id)
+    (persist-temp-file conn params)))

backend/src/app/rpc/commands/files/update.clj

@@ -0,0 +1,295 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.rpc.commands.files.update
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.files.features :as ffeat]
+   [app.common.logging :as l]
+   [app.common.pages :as cp]
+   [app.common.pages.migrations :as pmg]
+   [app.common.spec :as us]
+   [app.common.types.file :as ctf]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.loggers.audit :as audit]
+   [app.metrics :as mtx]
+   [app.msgbus :as mbus]
+   [app.rpc :as-alias rpc]
+   [app.rpc.climit :as-alias climit]
+   [app.rpc.commands.files :as files]
+   [app.rpc.doc :as-alias doc]
+   [app.util.blob :as blob]
+   [app.util.objects-map :as omap]
+   [app.util.pointer-map :as pmap]
+   [app.util.services :as sv]
+   [app.util.time :as dt]
+   [clojure.spec.alpha :as s]))
+
+;; --- SPECS
+
+(s/def ::changes
+  (s/coll-of map? :kind vector?))
+
+(s/def ::hint-origin ::us/keyword)
+(s/def ::hint-events
+  (s/every ::us/keyword :kind vector?))
+
+(s/def ::change-with-metadata
+  (s/keys :req-un [::changes]
+          :opt-un [::hint-origin
+                   ::hint-events]))
+
+(s/def ::changes-with-metadata
+  (s/every ::change-with-metadata :kind vector?))
+
+(s/def ::session-id ::us/uuid)
+(s/def ::revn ::us/integer)
+(s/def ::update-file
+  (s/and
+   (s/keys :req-un [::files/id ::files/profile-id ::session-id ::revn]
+           :opt-un [::changes ::changes-with-metadata ::features])
+   (fn [o]
+     (or (contains? o :changes)
+         (contains? o :changes-with-metadata)))))
+
+;; --- HELPERS
+
+;; File changes that affect to the library, and must be notified
+;; to all clients using it.
+
+(def ^:private library-change-types
+  #{:add-color :mod-color :del-color
+    :add-media :mod-media :del-media
+    :add-component :mod-component :del-component
+    :add-typography :mod-typography :del-typography})
+
+(def ^:private file-change-types
+  #{:add-obj :mod-obj :del-obj
+    :reg-objects :mov-objects})
+
+(defn- library-change?
+  [{:keys [type] :as change}]
+  (or (contains? library-change-types type)
+      (and (contains? file-change-types type)
+           (some? (:component-id change)))))
+
+(def ^:private sql:get-file
+  "SELECT f.*, p.team_id
+     FROM file AS f
+     JOIN project AS p ON (p.id = f.project_id)
+    WHERE f.id = ?
+      AND (f.deleted_at IS NULL OR
+           f.deleted_at > now())
+      FOR KEY SHARE")
+
+(defn get-file
+  [conn id]
+  (let [file (db/exec-one! conn [sql:get-file id])]
+    (when-not file
+      (ex/raise :type :not-found
+                :code :object-not-found
+                :hint (format "file with id '%s' does not exists" id)))
+    (update file :features db/decode-pgarray #{})))
+
+(defn- wrap-with-pointer-map-context
+  [f]
+  (fn [{:keys [conn] :as cfg} {:keys [id] :as file}]
+    (binding [pmap/*tracked* (atom {})
+              pmap/*load-fn* (partial files/load-pointer conn id)
+              ffeat/*wrap-with-pointer-map-fn* pmap/wrap]
+      (let [result (f cfg file)]
+        (files/persist-pointers! conn id)
+        result))))
+
+(defn- wrap-with-objects-map-context
+  [f]
+  (fn [cfg file]
+    (binding [ffeat/*wrap-with-objects-map-fn* omap/wrap]
+      (f cfg file))))
+
+(declare get-lagged-changes)
+(declare send-notifications!)
+(declare update-file)
+(declare update-file*)
+(declare take-snapshot?)
+
+;; If features are specified from params and the final feature
+;; set is different than the persisted one, update it on the
+;; database.
+
+(sv/defmethod ::update-file
+  {::climit/queue :update-file
+   ::climit/key-fn :id
+   ::doc/added "1.17"}
+  [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}]
+
+  (db/with-atomic [conn pool]
+    (files/check-edition-permissions! conn profile-id id)
+    (db/xact-lock! conn id)
+
+    (let [cfg    (assoc cfg :conn conn)
+          tpoint (dt/tpoint)]
+      (-> (update-file cfg params)
+          (vary-meta assoc ::rpc/before-complete
+                     (fn []
+                       (let [elapsed (tpoint)]
+                         (l/trace :hint "update-file" :time (dt/format-duration elapsed)))))))))
+
+(defn update-file
+  [{:keys [conn metrics] :as cfg} {:keys [id profile-id changes changes-with-metadata] :as params}]
+  (let [file     (get-file conn id)
+        features (->> (concat (:features file)
+                              (:features params))
+                      (into files/default-features)
+                      (files/check-features-compatibility!))]
+
+    (files/check-edition-permissions! conn profile-id (:id file))
+
+    (binding [ffeat/*current*  features
+              ffeat/*previous* (:features file)]
+      (let [update-fn (cond-> update-file*
+                        (contains? features "storage/pointer-map")
+                        (wrap-with-pointer-map-context)
+
+                        (contains? features "storage/objects-map")
+                        (wrap-with-objects-map-context))
+
+            file      (assoc file :features features)
+            changes   (if changes-with-metadata
+                        (->> changes-with-metadata (mapcat :changes) vec)
+                        (vec changes))
+
+            params    (assoc params :file file :changes changes)]
+
+        (mtx/run! metrics {:id :update-file-changes :inc (count changes)})
+
+        (when (not= features (:features file))
+          (let [features (db/create-array conn "text" features)]
+            (db/update! conn :file
+                        {:features features}
+                        {:id id})))
+
+        (-> (update-fn cfg params)
+            (vary-meta assoc ::audit/props {:project-id (:project-id file)
+                                            :team-id    (:team-id file)}))))))
+
+(defn- update-file*
+  [{:keys [conn] :as cfg} {:keys [file changes session-id profile-id] :as params}]
+  (when (> (:revn params)
+           (:revn file))
+    (ex/raise :type :validation
+              :code :revn-conflict
+              :hint "The incoming revision number is greater that stored version."
+              :context {:incoming-revn (:revn params)
+                        :stored-revn (:revn file)}))
+
+  (let [ts   (dt/now)
+        file (-> file
+                 (update :revn inc)
+                 (update :data (fn [data]
+                                 (cond-> data
+                                   :always
+                                   (-> (blob/decode)
+                                       (assoc :id (:id file))
+                                       (pmg/migrate-data))
+
+                                   (and (contains? ffeat/*current* "components/v2")
+                                        (not (contains? ffeat/*previous* "components/v2")))
+                                   (ctf/migrate-to-components-v2)
+
+                                   :always
+                                   (-> (cp/process-changes changes)
+                                       (blob/encode))))))]
+    (db/insert! conn :file-change
+                {:id (uuid/next)
+                 :session-id session-id
+                 :profile-id profile-id
+                 :created-at ts
+                 :file-id (:id file)
+                 :revn (:revn file)
+                 :features (db/create-array conn "text" (:features file))
+                 :data (when (take-snapshot? file)
+                         (:data file))
+                 :changes (blob/encode changes)})
+
+    (db/update! conn :file
+                {:revn (:revn file)
+                 :data (:data file)
+                 :data-backend nil
+                 :modified-at ts
+                 :has-media-trimmed false}
+                {:id (:id file)})
+
+    (db/update! conn :project
+                {:modified-at ts}
+                {:id (:project-id file)})
+
+    (let [params (assoc params :file file)]
+      ;; Send asynchronous notifications
+      (send-notifications! cfg params)
+
+      ;; Retrieve and return lagged data
+      (get-lagged-changes conn params))))
+
+(defn- take-snapshot?
+  "Defines the rule when file `data` snapshot should be saved."
+  [{:keys [revn modified-at] :as file}]
+  (let [freq    (or (cf/get :file-change-snapshot-every) 20)
+        timeout (or (cf/get :file-change-snapshot-timeout)
+                    (dt/duration {:hours 1}))]
+    (or (= 1 freq)
+        (zero? (mod revn freq))
+        (> (inst-ms (dt/diff modified-at (dt/now)))
+           (inst-ms timeout)))))
+
+(def ^:private
+  sql:lagged-changes
+  "select s.id, s.revn, s.file_id,
+          s.session_id, s.changes
+     from file_change as s
+    where s.file_id = ?
+      and s.revn > ?
+    order by s.created_at asc")
+
+(defn- get-lagged-changes
+  [conn params]
+  (->> (db/exec! conn [sql:lagged-changes (:id params) (:revn params)])
+       (into [] (comp (map files/decode-row)
+                      (map (fn [row]
+                             (cond-> row
+                               (= (:revn row) (:revn (:file params)))
+                               (assoc :changes []))))))))
+
+(defn- send-notifications!
+  [{:keys [conn] :as cfg} {:keys [file changes session-id] :as params}]
+  (let [lchanges (filter library-change? changes)
+        msgbus   (:msgbus cfg)]
+
+    ;; Asynchronously publish message to the msgbus
+    (mbus/pub! msgbus
+               :topic (:id file)
+               :message {:type :file-change
+                         :profile-id (:profile-id params)
+                         :file-id (:id file)
+                         :session-id (:session-id params)
+                         :revn (:revn file)
+                         :changes changes})
+
+    (when (and (:is-shared file) (seq lchanges))
+      (let [team-id (or (:team-id file)
+                        (files/retrieve-team-id conn (:project-id file)))]
+        ;; Asynchronously publish message to the msgbus
+        (mbus/pub! msgbus
+                   :topic team-id
+                   :message {:type :library-change
+                             :profile-id (:profile-id params)
+                             :file-id (:id file)
+                             :session-id session-id
+                             :revn (:revn file)
+                             :modified-at (dt/now)
+                             :changes lchanges})))))

backend/src/app/rpc/commands/management.clj

@@ -14,11 +14,13 @@
    [app.common.uuid :as uuid]
    [app.db :as db]
    [app.rpc.commands.binfile :as binfile]
+   [app.rpc.commands.files :as files]
    [app.rpc.doc :as-alias doc]
    [app.rpc.mutations.projects :refer [create-project-role create-project]]
    [app.rpc.queries.projects :as proj]
    [app.rpc.queries.teams :as teams]
    [app.util.blob :as blob]
+   [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
    [app.util.time :as dt]
    [clojure.spec.alpha :as s]
@@ -53,7 +55,7 @@
     (assoc key (get index (get item key) (get item key)))))
 
 (defn- process-file
-  [file index]
+  [conn {:keys [id] :as file} index]
   (letfn [(process-form [form]
             (cond-> form
               ;; Relink library items
@@ -97,18 +99,25 @@
                              res)))
                        media
                        media))]
-
+    (-> file
-    (update file :data
+        (update :id #(get index %))
+        (update :data
                 (fn [data]
-              (-> data
+                  (binding [pmap/*load-fn* (partial files/load-pointer conn id)
+                            pmap/*tracked* (atom {})]
+                    (let [file-id (get index id)
+                          data    (-> data
                                       (blob/decode)
-                  (assoc :id (:id file))
+                                      (assoc :id file-id)
                                       (pmg/migrate-data)
                                       (update :pages-index relink-shapes)
                                       (update :components relink-shapes)
                                       (update :media relink-media)
                                       (d/without-nils)
-                  (blob/encode))))))
+                                      (files/process-pointers pmap/clone)
+                                      (blob/encode))]
+                      (files/persist-pointers! conn file-id)
+                      data)))))))
 
 (def sql:retrieve-used-libraries
   "select flr.*
@@ -166,9 +175,9 @@
         file    (-> file
                     (assoc :created-at now)
                     (assoc :modified-at now)
-                    (assoc :ignore-sync-until ignore)
+                    (assoc :ignore-sync-until ignore))
-                    (update :id #(get index %))
+
-                    (process-file index))]
+        file    (process-file conn file index)]
 
     (db/insert! conn :file file)
     (db/insert! conn :file-profile-rel

backend/src/app/rpc/commands/viewer.clj

@@ -0,0 +1,89 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.rpc.commands.viewer
+  (:require
+   [app.common.exceptions :as ex]
+   [app.db :as db]
+   [app.rpc.commands.comments :as comments]
+   [app.rpc.commands.files :as files]
+   [app.rpc.doc :as-alias doc]
+   [app.rpc.queries.share-link :as slnk]
+   [app.util.services :as sv]
+   [clojure.spec.alpha :as s]))
+
+;; --- Query: View Only Bundle
+
+(defn- get-project
+  [conn id]
+  (db/get-by-id conn :project id {:columns [:id :name :team-id]}))
+
+(defn- get-bundle
+  [conn file-id profile-id features]
+  (let [file    (files/get-file conn file-id features)
+        project (get-project conn (:project-id file))
+        libs    (files/get-file-libraries conn false file-id)
+        users   (comments/get-file-comments-users conn file-id profile-id)
+
+        links   (->> (db/query conn :share-link {:file-id file-id})
+                     (mapv slnk/decode-share-link-row))
+
+        fonts   (db/query conn :team-font-variant
+                          {:team-id (:team-id project)
+                           :deleted-at nil})]
+    {:file file
+     :users users
+     :fonts fonts
+     :project project
+     :share-links links
+     :libraries libs}))
+
+(defn get-view-only-bundle
+  [conn {:keys [profile-id file-id share-id features] :as params}]
+  (let [slink  (slnk/retrieve-share-link conn file-id share-id)
+        perms  (files/get-permissions conn profile-id file-id share-id)
+        thumbs (files/retrieve-object-thumbnails conn file-id)
+        bundle (-> (get-bundle conn file-id profile-id features)
+                   (assoc :permissions perms)
+                   (assoc-in [:file :thumbnails] thumbs))]
+
+    ;; When we have neither profile nor share, we just return a not
+    ;; found response to the user.
+    (when (and (not profile-id)
+               (not slink))
+      (ex/raise :type :not-found
+                :code :object-not-found))
+
+    ;; When we have only profile, we need to check read permissions
+    ;; on file.
+    (when (and profile-id (not slink))
+      (files/check-read-permissions! conn profile-id file-id))
+
+    (cond-> bundle
+      (some? slink)
+      (assoc :share slink)
+
+      (and (some? slink)
+           (not (contains? (:flags slink) "view-all-pages")))
+      (update-in [:file :data] (fn [data]
+                                 (let [allowed-pages (:pages slink)]
+                                   (-> data
+                                       (update :pages (fn [pages] (filterv #(contains? allowed-pages %) pages)))
+                                       (update :pages-index (fn [index] (select-keys index allowed-pages))))))))))
+
+(s/def ::get-view-only-bundle
+  (s/keys :req-un [::files/file-id]
+          :opt-un [::files/profile-id
+                   ::files/share-id
+                   ::files/features]))
+
+(sv/defmethod ::get-view-only-bundle
+  {:auth false
+   ::doc/added "1.17"}
+  [{:keys [pool]} params]
+  (with-open [conn (db/open pool)]
+    (get-view-only-bundle conn params)))
+

backend/src/app/rpc/mutations/comments.clj

@@ -10,8 +10,8 @@
    [app.common.spec :as us]
    [app.db :as db]
    [app.rpc.commands.comments :as cmd.comments]
+   [app.rpc.commands.files :as cmd.files]
    [app.rpc.doc :as-alias doc]
-   [app.rpc.queries.files :as files]
    [app.rpc.retry :as retry]
    [app.util.services :as sv]
    [clojure.spec.alpha :as s]))
@@ -27,7 +27,7 @@
    ::doc/deprecated "1.15"}
   [{:keys [pool] :as cfg} {:keys [profile-id file-id share-id] :as params}]
   (db/with-atomic [conn pool]
-    (files/check-comment-permissions! conn profile-id file-id share-id)
+    (cmd.files/check-comment-permissions! conn profile-id file-id share-id)
     (cmd.comments/create-comment-thread conn params)))
 
 ;; --- Mutation: Update Comment Thread Status
@@ -44,7 +44,7 @@
   (db/with-atomic [conn pool]
     (let [cthr (db/get-by-id conn :comment-thread id {:for-update true})]
       (when-not cthr (ex/raise :type :not-found))
-      (files/check-comment-permissions! conn profile-id (:file-id cthr) share-id)
+      (cmd.files/check-comment-permissions! conn profile-id (:file-id cthr) share-id)
       (cmd.comments/upsert-comment-thread-status! conn profile-id (:id cthr)))))
 
 
@@ -61,7 +61,7 @@
       (when-not thread
         (ex/raise :type :not-found))
 
-      (files/check-comment-permissions! conn profile-id (:file-id thread) share-id)
+      (cmd.files/check-comment-permissions! conn profile-id (:file-id thread) share-id)
       (db/update! conn :comment-thread
                   {:is-resolved is-resolved}
                   {:id id})

backend/src/app/rpc/mutations/files.clj

@@ -6,641 +6,233 @@
 
 (ns app.rpc.mutations.files
   (:require
-   [app.common.data :as d]
    [app.common.exceptions :as ex]
-   [app.common.files.features :as ffeat]
    [app.common.logging :as l]
-   [app.common.pages :as cp]
-   [app.common.pages.migrations :as pmg]
    [app.common.spec :as us]
-   [app.common.types.file :as ctf]
-   [app.common.uuid :as uuid]
-   [app.config :as cf]
    [app.db :as db]
    [app.loggers.audit :as audit]
-   [app.metrics :as mtx]
-   [app.msgbus :as mbus]
    [app.rpc :as-alias rpc]
    [app.rpc.climit :as-alias climit]
+   [app.rpc.commands.files :as cmd.files]
+   [app.rpc.commands.files.create :as cmd.files.create]
+   [app.rpc.commands.files.temp :as cmd.files.temp]
+   [app.rpc.commands.files.update :as cmd.files.update]
    [app.rpc.doc :as-alias doc]
-   [app.rpc.permissions :as perms]
-   [app.rpc.queries.files :as files]
    [app.rpc.queries.projects :as proj]
-   [app.storage.impl :as simpl]
-   [app.util.blob :as blob]
-   [app.util.objects-map :as omap]
    [app.util.services :as sv]
    [app.util.time :as dt]
-   [clojure.spec.alpha :as s]
+   [clojure.spec.alpha :as s]))
-   [promesa.core :as p]))
-
-(declare create-file)
-(declare retrieve-team-id)
-
-;; --- Helpers & Specs
-
-(s/def ::frame-id ::us/uuid)
-(s/def ::file-id ::us/uuid)
-(s/def ::id ::us/uuid)
-(s/def ::name ::us/string)
-(s/def ::profile-id ::us/uuid)
-(s/def ::project-id ::us/uuid)
-(s/def ::url ::us/url)
 
 ;; --- Mutation: Create File
 
-(s/def ::features ::us/set-of-strings)
+(s/def ::create-file ::cmd.files.create/create-file)
-(s/def ::is-shared ::us/boolean)
-(s/def ::create-file
-  (s/keys :req-un [::profile-id ::name ::project-id]
-          :opt-un [::id ::is-shared ::features ::components-v2]))
 
 (sv/defmethod ::create-file
-  {::doc/added "1.0"}
+  {::doc/added "1.0"
-  [{:keys [pool] :as cfg} {:keys [profile-id project-id] :as params}]
+   ::doc/deprecated "1.17"}
+  [{:keys [pool] :as cfg} {:keys [profile-id project-id features components-v2] :as params}]
   (db/with-atomic [conn pool]
-    (let [team-id (retrieve-team-id conn project-id)]
     (proj/check-edition-permissions! conn profile-id project-id)
-      (with-meta
+    (let [team-id  (cmd.files/retrieve-team-id conn project-id)
-        (create-file conn params)
-        {::audit/props {:team-id team-id}}))))
-
-(defn create-file-role
-  [conn {:keys [file-id profile-id role]}]
-  (let [params {:file-id file-id
-                :profile-id profile-id}]
-    (->> (perms/assign-role-flags params role)
-         (db/insert! conn :file-profile-rel))))
-
-(defn create-file
-  [conn {:keys [id name project-id is-shared data revn
-                modified-at deleted-at ignore-sync-until
-                components-v2 features]
-         :or {is-shared false revn 0}
-         :as params}]
-  (let [id       (or id (:id data) (uuid/next))
-
-        ;; BACKWARD COMPATIBILITY with the components-v2 param
           features (cond-> (or features #{})
+                     ;; BACKWARD COMPATIBILITY with the components-v2 param
                      components-v2 (conj "components/v2"))
+          params   (assoc params :features features)]
+      (-> (cmd.files.create/create-file conn params)
+          (vary-meta assoc ::audit/props {:team-id team-id})))))
 
-        data     (or data
-                     (binding [ffeat/*current* features]
-                       (ctf/make-file-data id)))
-
-        features (db/create-array conn "text" features)
-        file     (db/insert! conn :file
-                             (d/without-nils
-                              {:id id
-                               :project-id project-id
-                               :name name
-                               :revn revn
-                               :is-shared is-shared
-                               :data (blob/encode data)
-                               :features features
-                               :ignore-sync-until ignore-sync-until
-                               :modified-at modified-at
-                               :deleted-at deleted-at}))]
-
-    (->> (assoc params :file-id id :role :owner)
-         (create-file-role conn))
-
-    (-> file files/decode-row)))
 
 ;; --- Mutation: Rename File
 
-(declare rename-file)
+(s/def ::rename-file ::cmd.files/rename-file)
-
-(s/def ::rename-file
-  (s/keys :req-un [::profile-id ::name ::id]))
 
 (sv/defmethod ::rename-file
-  {::doc/added "1.0"}
+  {::doc/added "1.0"
+   ::doc/deprecated "1.17"}
   [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}]
   (db/with-atomic [conn pool]
-    (files/check-edition-permissions! conn profile-id id)
+    (cmd.files/check-edition-permissions! conn profile-id id)
-    (rename-file conn params)))
+    (cmd.files/rename-file conn params)))
 
-(defn- rename-file
-  [conn {:keys [id name] :as params}]
-  (-> (db/update! conn :file
-                  {:name name
-                   :modified-at (dt/now)}
-                  {:id id})
-      (select-keys [:id :name :created-at :modified-at])))
 
 ;; --- Mutation: Set File shared
 
-(declare set-file-shared)
+(s/def ::set-file-shared ::cmd.files/set-file-shared)
-(declare unlink-files)
-(declare absorb-library)
-
-(s/def ::set-file-shared
-  (s/keys :req-un [::profile-id ::id ::is-shared]))
 
 (sv/defmethod ::set-file-shared
-  {::doc/added "1.2"}
+  {::doc/added "1.2"
+   ::doc/deprecated "1.17"}
   [{:keys [pool] :as cfg} {:keys [id profile-id is-shared] :as params}]
   (db/with-atomic [conn pool]
-    (files/check-edition-permissions! conn profile-id id)
+    (cmd.files/check-edition-permissions! conn profile-id id)
     (when-not is-shared
-      (absorb-library conn params)
+      (cmd.files/absorb-library conn params)
-      (unlink-files conn params))
+      (cmd.files/unlink-files conn params))
-    (-> (set-file-shared conn params)
+    (cmd.files/set-file-shared conn params)))
-        (update :features db/decode-pgarray #{}))))
-
-(defn- unlink-files
-  [conn {:keys [id] :as params}]
-  (db/delete! conn :file-library-rel {:library-file-id id}))
-
-(defn- set-file-shared
-  [conn {:keys [id is-shared] :as params}]
-  (db/update! conn :file
-              {:is-shared is-shared}
-              {:id id}))
 
 ;; --- Mutation: Delete File
 
-(declare mark-file-deleted)
+(s/def ::delete-file ::cmd.files/delete-file)
-
-(s/def ::delete-file
-  (s/keys :req-un [::id ::profile-id]))
 
 (sv/defmethod ::delete-file
-  {::doc/added "1.0"}
+  {::doc/added "1.0"
+   ::doc/deprecated "1.17"}
   [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}]
   (db/with-atomic [conn pool]
-    (files/check-edition-permissions! conn profile-id id)
+    (cmd.files/check-edition-permissions! conn profile-id id)
-    (absorb-library conn params)
+    (cmd.files/absorb-library conn params)
-    (mark-file-deleted conn params)))
+    (cmd.files/mark-file-deleted conn params)))
-
-(defn mark-file-deleted
-  [conn {:keys [id] :as params}]
-  (db/update! conn :file
-              {:deleted-at (dt/now)}
-              {:id id})
-  nil)
-
-(defn absorb-library
-  "Find all files using a shared library, and absorb all library assets
-  into the file local libraries"
-  [conn {:keys [id] :as params}]
-  (let [library (db/get-by-id conn :file id)]
-    (when (:is-shared library)
-      (let [ldata (-> library files/decode-row pmg/migrate-file :data)]
-        (->> (db/query conn :file-library-rel {:library-file-id id})
-             (keep (fn [{:keys [file-id]}]
-                    (some->> (db/get-by-id conn :file file-id {:check-not-found false})
-                             (files/decode-row)
-                             (pmg/migrate-file))))
-             (run! (fn [{:keys [id data revn] :as file}]
-                     (let [data (ctf/absorb-assets data ldata)]
-                       (db/update! conn :file
-                                   {:revn (inc revn)
-                                    :data (blob/encode data)
-                                    :modified-at (dt/now)}
-                                   {:id id})))))))))
 
 ;; --- Mutation: Link file to library
 
-(declare link-file-to-library)
+(s/def ::link-file-to-library ::cmd.files/link-file-to-library)
-
-(s/def ::link-file-to-library
-  (s/keys :req-un [::profile-id ::file-id ::library-id]))
 
 (sv/defmethod ::link-file-to-library
+  {::doc/added "1.3"
+   ::doc/deprecated "1.17"}
   [{:keys [pool] :as cfg} {:keys [profile-id file-id library-id] :as params}]
   (when (= file-id library-id)
     (ex/raise :type :validation
               :code :invalid-library
               :hint "A file cannot be linked to itself"))
   (db/with-atomic [conn pool]
-    (files/check-edition-permissions! conn profile-id file-id)
+    (cmd.files/check-edition-permissions! conn profile-id file-id)
-    (files/check-edition-permissions! conn profile-id library-id)
+    (cmd.files/check-edition-permissions! conn profile-id library-id)
-    (link-file-to-library conn params)))
+    (cmd.files/link-file-to-library conn params)))
-
-(def sql:link-file-to-library
-  "insert into file_library_rel (file_id, library_file_id)
-   values (?, ?)
-       on conflict do nothing;")
-
-(defn- link-file-to-library
-  [conn {:keys [file-id library-id] :as params}]
-  (db/exec-one! conn [sql:link-file-to-library file-id library-id]))
-
 
 ;; --- Mutation: Unlink file from library
 
-(declare unlink-file-from-library)
+(s/def ::unlink-file-from-library ::cmd.files/unlink-file-from-library)
-
-(s/def ::unlink-file-from-library
-  (s/keys :req-un [::profile-id ::file-id ::library-id]))
 
 (sv/defmethod ::unlink-file-from-library
+  {::doc/added "1.3"
+   ::doc/deprecated "1.17"}
   [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}]
   (db/with-atomic [conn pool]
-    (files/check-edition-permissions! conn profile-id file-id)
+    (cmd.files/check-edition-permissions! conn profile-id file-id)
-    (unlink-file-from-library conn params)))
+    (cmd.files/unlink-file-from-library conn params)))
-
-(defn- unlink-file-from-library
-  [conn {:keys [file-id library-id] :as params}]
-  (db/delete! conn :file-library-rel
-              {:file-id file-id
-               :library-file-id library-id}))
 
 
 ;; --- Mutation: Update synchronization status of a link
 
-(declare update-sync)
+(s/def ::update-sync ::cmd.files/update-file-library-sync-status)
-
-(s/def ::update-sync
-  (s/keys :req-un [::profile-id ::file-id ::library-id]))
 
 (sv/defmethod ::update-sync
+  {::doc/added "1.10"
+   ::doc/deprecated "1.17"}
   [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}]
   (db/with-atomic [conn pool]
-    (files/check-edition-permissions! conn profile-id file-id)
+    (cmd.files/check-edition-permissions! conn profile-id file-id)
-    (update-sync conn params)))
+    (cmd.files/update-sync conn params)))
 
-(defn- update-sync
-  [conn {:keys [file-id library-id] :as params}]
-  (db/update! conn :file-library-rel
-              {:synced-at (dt/now)}
-              {:file-id file-id
-               :library-file-id library-id}))
 
 ;; --- Mutation: Ignore updates in linked files
 
 (declare ignore-sync)
 
-(s/def ::ignore-sync
+(s/def ::ignore-sync ::cmd.files/ignore-file-library-sync-status)
-  (s/keys :req-un [::profile-id ::file-id ::date]))
 
 (sv/defmethod ::ignore-sync
+  {::doc/added "1.10"
+   ::doc/deprecated "1.17"}
   [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}]
   (db/with-atomic [conn pool]
-    (files/check-edition-permissions! conn profile-id file-id)
+    (cmd.files/check-edition-permissions! conn profile-id file-id)
-    (ignore-sync conn params)))
+    (cmd.files/ignore-sync conn params)))
-
-(defn- ignore-sync
-  [conn {:keys [file-id date] :as params}]
-  (db/update! conn :file
-              {:ignore-sync-until date}
-              {:id file-id}))
 
 
 ;; --- MUTATION: update-file
 
-;; A generic, Changes based (granular) file update method.
-
-;; File changes that affect to the library, and must be notified
-;; to all clients using it.
-(defn library-change?
-  [change]
-  (or (#{:add-color :mod-color :del-color
-         :add-media :mod-media :del-media
-         :add-component :mod-component :del-component
-         :add-typography :mod-typography :del-typography} (:type change))
-      (and (#{:add-obj :mod-obj :del-obj
-              :reg-objects :mov-objects} (:type change))
-           (some? (:component-id change)))))
-
-(declare insert-change)
-(declare retrieve-lagged-changes)
-(declare send-notifications)
-(declare update-file)
-
-(s/def ::changes
-  (s/coll-of map? :kind vector?))
-
-(s/def ::hint-origin ::us/keyword)
-(s/def ::hint-events
-  (s/every ::us/keyword :kind vector?))
-
-(s/def ::change-with-metadata
-  (s/keys :req-un [::changes]
-          :opt-un [::hint-origin
-                   ::hint-events]))
-
-(s/def ::changes-with-metadata
-  (s/every ::change-with-metadata :kind vector?))
-
-(s/def ::session-id ::us/uuid)
-(s/def ::revn ::us/integer)
 (s/def ::components-v2 ::us/boolean)
 (s/def ::update-file
-  (s/and
+  (s/and ::cmd.files.update/update-file
-   (s/keys :req-un [::id ::session-id ::profile-id ::revn]
+         (s/keys :opt-un [::components-v2])))
-           :opt-un [::changes ::changes-with-metadata ::components-v2 ::features])
-   (fn [o]
-     (or (contains? o :changes)
-         (contains? o :changes-with-metadata)))))
-
-(def ^:private sql:retrieve-file
-  "SELECT f.*, p.team_id
-     FROM file AS f
-     JOIN project AS p ON (p.id = f.project_id)
-    WHERE f.id = ?
-      AND (f.deleted_at IS NULL OR
-           f.deleted_at > now())
-      FOR KEY SHARE")
 
 (sv/defmethod ::update-file
   {::climit/queue :update-file
-   ::climit/key-fn :id}
+   ::climit/key-fn :id
-  [{:keys [pool] :as cfg} {:keys [id profile-id components-v2] :as params}]
+   ::doc/added "1.0"
+   ::doc/deprecated "1.17"}
+  [{:keys [pool] :as cfg} {:keys [id profile-id features components-v2] :as params}]
   (db/with-atomic [conn pool]
     (db/xact-lock! conn id)
-    (let [file      (db/exec-one! conn [sql:retrieve-file id])
+    (cmd.files/check-edition-permissions! conn profile-id id)
-          features' (:features params #{})
-          features  (db/decode-pgarray (:features file) features')
 
-          ;; BACKWARD COMPATIBILITY with the components-v2 parameter
+    (let [;; BACKWARD COMPATIBILITY with the components-v2 parameter
-          features  (cond-> features
+          features  (cond-> (or features #{})
                       components-v2 (conj "components/v2"))
+          tpoint    (dt/tpoint)
+          params    (assoc params :features features)
+          cfg       (assoc cfg :conn conn)]
 
-          file      (assoc file :features features)
+      (-> (cmd.files.update/update-file cfg params)
-          tpoint    (dt/tpoint)]
+          (vary-meta assoc ::rpc/before-complete
-
-      (when-not file
-        (ex/raise :type :not-found
-                  :code :object-not-found
-                  :hint (format "file with id '%s' does not exists" id)))
-
-      ;; If features are specified from params and the final feature
-      ;; set is different than the persisted one, update it on the
-      ;; database.
-      (when (not= features features')
-        (let [features (db/create-array conn "text" features)]
-          (db/update! conn :file
-                      {:features features}
-                      {:id id})))
-
-      (binding [ffeat/*current* features
-                ffeat/*wrap-objects-fn* (if (features "storate/objects-map")
-                                         omap/wrap
-                                         identity)]
-        (files/check-edition-permissions! conn profile-id (:id file))
-        (with-meta
-          (update-file (assoc cfg :conn conn)
-                       (assoc params :file file))
-          {::audit/props
-           {:project-id (:project-id file)
-            :team-id    (:team-id file)}
-
-           ::rpc/before-complete
                      (fn []
                        (let [elapsed (tpoint)]
-               (l/trace :hint "update-file" :time (dt/format-duration elapsed))))})))))
+                         (l/trace :hint "update-file" :time (dt/format-duration elapsed)))))))))
-
-
-(defn- take-snapshot?
-  "Defines the rule when file `data` snapshot should be saved."
-  [{:keys [revn modified-at] :as file}]
-  (let [freq    (or (cf/get :file-change-snapshot-every) 20)
-        timeout (or (cf/get :file-change-snapshot-timeout)
-                    (dt/duration {:hours 1}))]
-    (or (= 1 freq)
-        (zero? (mod revn freq))
-        (> (inst-ms (dt/diff modified-at (dt/now)))
-           (inst-ms timeout)))))
-
-(defn- delete-from-storage
-  [{:keys [storage] :as cfg} file]
-  (p/do
-    (when-let [backend (simpl/resolve-backend storage (:data-backend file))]
-      (simpl/del-object backend file))))
-
-(defn- update-file
-  [{:keys [conn metrics] :as cfg}
-   {:keys [file changes changes-with-metadata session-id profile-id] :as params}]
-  (when (> (:revn params)
-           (:revn file))
-
-    (ex/raise :type :validation
-              :code :revn-conflict
-              :hint "The incoming revision number is greater that stored version."
-              :context {:incoming-revn (:revn params)
-                        :stored-revn (:revn file)}))
-
-  (let [changes (if changes-with-metadata
-                  (mapcat :changes changes-with-metadata)
-                  changes)
-
-        changes (vec changes)
-
-        ;; Trace the number of changes processed
-        _       (mtx/run! metrics {:id :update-file-changes :inc (count changes)})
-
-        ts      (dt/now)
-        file    (-> file
-                    (update :revn inc)
-                    (update :data (fn [data]
-                                    ;; Trace the length of bytes of processed data
-                                    (mtx/run! metrics {:id :update-file-bytes-processed :inc (alength data)})
-                                    (cond-> data
-                                      :always
-                                      (-> (blob/decode)
-                                          (assoc :id (:id file))
-                                          (pmg/migrate-data))
-
-
-                                      (contains? ffeat/*current* "components/v2")
-                                      (ctf/migrate-to-components-v2)
-
-                                      :always
-                                      (-> (cp/process-changes changes)
-                                          (blob/encode))))))]
-    ;; Insert change to the xlog
-    (db/insert! conn :file-change
-                {:id (uuid/next)
-                 :session-id session-id
-                 :profile-id profile-id
-                 :created-at ts
-                 :file-id (:id file)
-                 :revn (:revn file)
-                 :data (when (take-snapshot? file)
-                         (:data file))
-                 :changes (blob/encode changes)})
-
-    ;; Update file
-    (db/update! conn :file
-                {:revn (:revn file)
-                 :data (:data file)
-                 :data-backend nil
-                 :modified-at ts
-                 :has-media-trimmed false}
-                {:id (:id file)})
-
-    ;; We need to delete the data from external storage backend
-    (when-not (nil? (:data-backend file))
-      @(delete-from-storage cfg file))
-
-    (db/update! conn :project
-                {:modified-at ts}
-                {:id (:project-id file)})
-
-    (let [params (assoc params :file file :changes changes)]
-      ;; Send asynchronous notifications
-      (send-notifications cfg params)
-
-      ;; Retrieve and return lagged data
-      (retrieve-lagged-changes conn params))))
-
-(def ^:private
-  sql:lagged-changes
-  "select s.id, s.revn, s.file_id,
-          s.session_id, s.changes
-     from file_change as s
-    where s.file_id = ?
-      and s.revn > ?
-    order by s.created_at asc")
-
-(defn- retrieve-lagged-changes
-  [conn params]
-  (->> (db/exec! conn [sql:lagged-changes (:id params) (:revn params)])
-       (into [] (comp (map files/decode-row)
-                      (map (fn [row]
-                             (cond-> row
-                               (= (:revn row) (:revn (:file params)))
-                               (assoc :changes []))))))))
-
-(defn- send-notifications
-  [{:keys [conn] :as cfg} {:keys [file changes session-id] :as params}]
-  (let [lchanges (filter library-change? changes)
-        msgbus   (:msgbus cfg)]
-
-
-    ;; Asynchronously publish message to the msgbus
-    (mbus/pub! msgbus
-               :topic (:id file)
-               :message {:type :file-change
-                         :profile-id (:profile-id params)
-                         :file-id (:id file)
-                         :session-id (:session-id params)
-                         :revn (:revn file)
-                         :changes changes})
-
-    (when (and (:is-shared file) (seq lchanges))
-      (let [team-id (or (:team-id file)
-                        (retrieve-team-id conn (:project-id file)))]
-        ;; Asynchronously publish message to the msgbus
-        (mbus/pub! msgbus
-                   :topic team-id
-                   :message {:type :library-change
-                             :profile-id (:profile-id params)
-                             :file-id (:id file)
-                             :session-id session-id
-                             :revn (:revn file)
-                             :modified-at (dt/now)
-                             :changes lchanges})))))
-
-(defn- retrieve-team-id
-  [conn project-id]
-  (:team-id (db/get-by-id conn :project project-id {:columns [:team-id]})))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; TEMPORARY FILES (behaves differently)
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(s/def ::create-temp-file ::create-file)
-
-(sv/defmethod ::create-temp-file
-  [{:keys [pool] :as cfg} {:keys [profile-id project-id] :as params}]
-  (db/with-atomic [conn pool]
-    (proj/check-edition-permissions! conn profile-id project-id)
-    (create-file conn (assoc params :deleted-at (dt/in-future {:days 1})))))
-
-(s/def ::update-temp-file
-  (s/keys :req-un [::changes ::revn ::session-id ::id]))
-
-(sv/defmethod ::update-temp-file
-  {::doc/added "1.7"}
-  [{:keys [pool] :as cfg} {:keys [profile-id session-id id revn changes] :as params}]
-  (db/with-atomic [conn pool]
-    (db/insert! conn :file-change
-                {:id (uuid/next)
-                 :session-id session-id
-                 :profile-id profile-id
-                 :created-at (dt/now)
-                 :file-id id
-                 :revn revn
-                 :data nil
-                 :changes (blob/encode changes)})
-    nil))
-
-(s/def ::persist-temp-file
-  (s/keys :req-un [::id ::profile-id]))
-
-(sv/defmethod ::persist-temp-file
-  {::doc/added "1.7"}
-  [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}]
-  (db/with-atomic [conn pool]
-    (files/check-edition-permissions! conn profile-id id)
-    (let [file (db/get-by-id conn :file id)
-          revs (db/query conn :file-change
-                         {:file-id id}
-                         {:order-by [[:revn :asc]]})
-          revn (count revs)]
-
-      (when (nil? (:deleted-at file))
-        (ex/raise :type :validation
-                  :code :cant-persist-already-persisted-file))
-
-      (loop [revs (seq revs)
-             data (blob/decode (:data file))]
-        (if-let [rev (first revs)]
-          (recur (rest revs)
-                 (->> rev :changes blob/decode (cp/process-changes data)))
-          (db/update! conn :file
-                      {:deleted-at nil
-                       :revn revn
-                       :data (blob/encode data)}
-                      {:id id})))
-      nil)))
 
 ;; --- Mutation: upsert object thumbnail
 
-(def sql:upsert-object-thumbnail
+(s/def ::upsert-file-object-thumbnail ::cmd.files/upsert-file-object-thumbnail)
-  "insert into file_object_thumbnail(file_id, object_id, data)
-   values (?, ?, ?)
-       on conflict(file_id, object_id) do
-          update set data = ?;")
-
-(s/def ::data (s/nilable ::us/string))
-(s/def ::object-id ::us/string)
-(s/def ::upsert-file-object-thumbnail
-  (s/keys :req-un [::profile-id ::file-id ::object-id ::data]))
 
 (sv/defmethod ::upsert-file-object-thumbnail
-  [{:keys [pool] :as cfg} {:keys [profile-id file-id object-id data]}]
+  {::doc/added "1.13"
+   ::doc/deprecated "1.17"}
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}]
   (db/with-atomic [conn pool]
-    (files/check-edition-permissions! conn profile-id file-id)
+    (cmd.files/check-edition-permissions! conn profile-id file-id)
-    (if data
+    (cmd.files/upsert-file-object-thumbnail! conn params)
-      (db/exec-one! conn [sql:upsert-object-thumbnail file-id object-id data data])
-      (db/delete! conn :file-object-thumbnail {:file-id file-id :object-id object-id}))
     nil))
 
+
 ;; --- Mutation: upsert file thumbnail
 
-(def sql:upsert-file-thumbnail
+(s/def ::upsert-file-thumbnail ::cmd.files/upsert-file-thumbnail)
-  "insert into file_thumbnail (file_id, revn, data, props)
-   values (?, ?, ?, ?::jsonb)
-       on conflict(file_id, revn) do
-          update set data = ?, props=?, updated_at=now();")
-
-(s/def ::revn ::us/integer)
-(s/def ::props map?)
-(s/def ::upsert-file-thumbnail
-  (s/keys :req-un [::profile-id ::file-id ::revn ::data ::props]))
 
 (sv/defmethod ::upsert-file-thumbnail
   "Creates or updates the file thumbnail. Mainly used for paint the
   grid thumbnails."
-  [{:keys [pool] :as cfg} {:keys [profile-id file-id revn data props]}]
+  {::doc/added "1.13"
+   ::doc/deprecated "1.17"}
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}]
   (db/with-atomic [conn pool]
-    (files/check-edition-permissions! conn profile-id file-id)
+    (cmd.files/check-edition-permissions! conn profile-id file-id)
-    (let [props (db/tjson (or props {}))]
+    (cmd.files/upsert-file-thumbnail conn params)
-      (db/exec-one! conn [sql:upsert-file-thumbnail
+    nil))
-                          file-id revn data props data props])
+
-      nil)))
+
+;; --- MUTATION COMMAND: create-temp-file
+
+(s/def ::create-temp-file ::cmd.files.temp/create-temp-file)
+
+(sv/defmethod ::create-temp-file
+  {::doc/added "1.7"
+   ::doc/deprecated "1.17"}
+  [{:keys [pool] :as cfg} {:keys [profile-id project-id] :as params}]
+  (db/with-atomic [conn pool]
+    (proj/check-edition-permissions! conn profile-id project-id)
+    (cmd.files.create/create-file conn (assoc params :deleted-at (dt/in-future {:days 1})))))
+
+;; --- MUTATION COMMAND: update-temp-file
+
+(s/def ::update-temp-file ::cmd.files.temp/update-temp-file)
+
+(sv/defmethod ::update-temp-file
+  {::doc/added "1.7"
+   ::doc/deprecated "1.17"}
+  [{:keys [pool] :as cfg} params]
+  (db/with-atomic [conn pool]
+    (cmd.files.temp/update-temp-file conn params)
+    nil))
+
+;; --- MUTATION COMMAND: persist-temp-file
+
+(s/def ::persist-temp-file ::cmd.files.temp/persist-temp-file)
+
+(sv/defmethod ::persist-temp-file
+  {::doc/added "1.7"
+   ::doc/deprecated "1.17"}
+  [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}]
+  (db/with-atomic [conn pool]
+    (cmd.files/check-edition-permissions! conn profile-id id)
+    (cmd.files.temp/persist-temp-file conn params)))

backend/src/app/rpc/mutations/share_link.clj

@@ -10,7 +10,7 @@
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
    [app.db :as db]
-   [app.rpc.queries.files :as files]
+   [app.rpc.commands.files :as files]
    [app.util.services :as sv]
    [clojure.spec.alpha :as s]))
 

backend/src/app/rpc/queries/comments.clj

@@ -8,8 +8,8 @@
   (:require
    [app.db :as db]
    [app.rpc.commands.comments :as cmd.comments]
+   [app.rpc.commands.files :as cmd.files]
    [app.rpc.doc :as-alias doc]
-   [app.rpc.queries.files :as files]
    [app.rpc.queries.teams :as teams]
    [app.util.services :as sv]
    [clojure.spec.alpha :as s]))
@@ -52,7 +52,7 @@
    ::doc/deprecated "1.15"}
   [{:keys [pool] :as cfg} {:keys [profile-id file-id share-id] :as params}]
   (with-open [conn (db/open pool)]
-    (files/check-comment-permissions! conn profile-id file-id share-id)
+    (cmd.files/check-comment-permissions! conn profile-id file-id share-id)
     (cmd.comments/get-comment-thread conn params)))
 
 ;; --- QUERY: Comments
@@ -65,7 +65,7 @@
   [{:keys [pool] :as cfg} {:keys [profile-id thread-id share-id] :as params}]
   (with-open [conn (db/open pool)]
     (let [thread (db/get-by-id conn :comment-thread thread-id)]
-      (files/check-comment-permissions! conn profile-id (:file-id thread) share-id))
+      (cmd.files/check-comment-permissions! conn profile-id (:file-id thread) share-id))
     (cmd.comments/get-comments conn thread-id)))
 
 
@@ -78,5 +78,5 @@
    ::doc/added "1.13"}
   [{:keys [pool] :as cfg} {:keys [profile-id file-id share-id]}]
   (with-open [conn (db/open pool)]
-    (files/check-comment-permissions! conn profile-id file-id share-id)
+    (cmd.files/check-comment-permissions! conn profile-id file-id share-id)
     (cmd.comments/get-file-comments-users conn file-id profile-id)))

backend/src/app/rpc/queries/files.clj

@@ -6,289 +6,56 @@
 
 (ns app.rpc.queries.files
   (:require
-   [app.common.data :as d]
-   [app.common.data.macros :as dm]
-   [app.common.exceptions :as ex]
-   [app.common.geom.shapes :as gsh]
-   [app.common.pages.helpers :as cph]
-   [app.common.pages.migrations :as pmg]
    [app.common.spec :as us]
-   [app.common.types.file :as ctf]
-   [app.common.types.shape-tree :as ctt]
    [app.db :as db]
-   [app.db.sql :as sql]
    [app.rpc :as-alias rpc]
+   [app.rpc.commands.files :as cmd.files]
+   [app.rpc.doc :as-alias doc]
    [app.rpc.helpers :as rpch]
-   [app.rpc.permissions :as perms]
    [app.rpc.queries.projects :as projects]
-   [app.rpc.queries.share-link :refer [retrieve-share-link]]
    [app.rpc.queries.teams :as teams]
-   [app.util.blob :as blob]
    [app.util.services :as sv]
-   [clojure.spec.alpha :as s]
+   [clojure.spec.alpha :as s]))
-   [cuerdas.core :as str]))
-
-(declare decode-row)
-
-;; --- Helpers & Specs
-
-(s/def ::frame-id ::us/uuid)
-(s/def ::id ::us/uuid)
-(s/def ::name ::us/string)
-(s/def ::project-id ::us/uuid)
-(s/def ::file-id ::us/uuid)
-(s/def ::profile-id ::us/uuid)
-(s/def ::team-id ::us/uuid)
-(s/def ::search-term ::us/string)
-(s/def ::components-v2 ::us/boolean)
-
-;; --- Query: File Permissions
-
-(def ^:private sql:file-permissions
-  "select fpr.is_owner,
-          fpr.is_admin,
-          fpr.can_edit
-     from file_profile_rel as fpr
-    where fpr.file_id = ?
-      and fpr.profile_id = ?
-   union all
-   select tpr.is_owner,
-          tpr.is_admin,
-          tpr.can_edit
-     from team_profile_rel as tpr
-    inner join project as p on (p.team_id = tpr.team_id)
-    inner join file as f on (p.id = f.project_id)
-    where f.id = ?
-      and tpr.profile_id = ?
-   union all
-   select ppr.is_owner,
-          ppr.is_admin,
-          ppr.can_edit
-     from project_profile_rel as ppr
-    inner join file as f on (f.project_id = ppr.project_id)
-    where f.id = ?
-      and ppr.profile_id = ?")
-
-(defn retrieve-file-permissions
-  [conn profile-id file-id]
-  (when (and profile-id file-id)
-    (db/exec! conn [sql:file-permissions
-                    file-id profile-id
-                    file-id profile-id
-                    file-id profile-id])))
-
-(defn get-permissions
-  ([conn profile-id file-id]
-   (let [rows     (retrieve-file-permissions conn profile-id file-id)
-         is-owner (boolean (some :is-owner rows))
-         is-admin (boolean (some :is-admin rows))
-         can-edit (boolean (some :can-edit rows))]
-     (when (seq rows)
-       {:type :membership
-        :is-owner is-owner
-        :is-admin (or is-owner is-admin)
-        :can-edit (or is-owner is-admin can-edit)
-        :can-read true
-        :is-logged (some? profile-id)})))
-  ([conn profile-id file-id share-id]
-   (let [perms  (get-permissions conn profile-id file-id)
-         ldata  (retrieve-share-link conn file-id share-id)]
-
-     ;; NOTE: in a future when share-link becomes more powerful and
-     ;; will allow us specify which parts of the app is available, we
-     ;; will probably need to tweak this function in order to expose
-     ;; this flags to the frontend.
-     (cond
-       (some? perms) perms
-       (some? ldata) {:type :share-link
-                      :can-read true
-                      :is-logged (some? profile-id)
-                      :who-comment (:who-comment ldata)
-                      :who-inspect (:who-inspect ldata)}))))
-
-(def has-edit-permissions?
-  (perms/make-edition-predicate-fn get-permissions))
-
-(def has-read-permissions?
-  (perms/make-read-predicate-fn get-permissions))
-
-(def has-comment-permissions?
-  (perms/make-comment-predicate-fn get-permissions))
-
-(def check-edition-permissions!
-  (perms/make-check-fn has-edit-permissions?))
-
-(def check-read-permissions!
-  (perms/make-check-fn has-read-permissions?))
-
-;; A user has comment permissions if she has read permissions, or comment permissions
-(defn check-comment-permissions!
-  [conn profile-id file-id share-id]
-   (let [can-read (has-read-permissions? conn profile-id file-id)
-         can-comment  (has-comment-permissions? conn profile-id file-id share-id)]
-     (when-not (or can-read can-comment)
-       (ex/raise :type :not-found
-                 :code :object-not-found
-                 :hint "not found"))))
-
-;; --- Query: Files search
-
-;; TODO: this query need to a good refactor
-
-(def ^:private sql:search-files
-  "with projects as (
-     select p.*
-       from project as p
-      inner join team_profile_rel as tpr on (tpr.team_id = p.team_id)
-      where tpr.profile_id = ?
-        and p.team_id = ?
-        and p.deleted_at is null
-        and (tpr.is_admin = true or
-             tpr.is_owner = true or
-             tpr.can_edit = true)
-      union
-     select p.*
-       from project as p
-      inner join project_profile_rel as ppr on (ppr.project_id = p.id)
-      where ppr.profile_id = ?
-        and p.team_id = ?
-        and p.deleted_at is null
-        and (ppr.is_admin = true or
-             ppr.is_owner = true or
-             ppr.can_edit = true)
-   )
-   select distinct
-          f.id,
-          f.project_id,
-          f.created_at,
-          f.modified_at,
-          f.name,
-          f.is_shared
-     from file as f
-    inner join projects as pr on (f.project_id = pr.id)
-    where f.name ilike ('%' || ? || '%')
-      and f.deleted_at is null
-    order by f.created_at asc")
-
-(s/def ::search-files
-  (s/keys :req-un [::profile-id ::team-id]
-          :opt-un [::search-term]))
-
-(sv/defmethod ::search-files
-  [{:keys [pool] :as cfg} {:keys [profile-id team-id search-term] :as params}]
-  (when search-term
-    (db/exec! pool [sql:search-files
-                    profile-id team-id
-                    profile-id team-id
-                    search-term])))
 
 ;; --- Query: Project Files
 
-(def ^:private sql:project-files
+(s/def ::project-files ::cmd.files/get-project-files)
-  "select f.id,
-          f.project_id,
-          f.created_at,
-          f.modified_at,
-          f.name,
-          f.revn,
-          f.is_shared
-     from file as f
-    where f.project_id = ?
-      and f.deleted_at is null
-    order by f.modified_at desc")
-
-(s/def ::project-files
-  (s/keys :req-un [::profile-id ::project-id]))
 
 (sv/defmethod ::project-files
+  {::doc/added "1.1"
+   ::doc/deprecated "1.17"}
   [{:keys [pool] :as cfg} {:keys [profile-id project-id] :as params}]
   (with-open [conn (db/open pool)]
     (projects/check-read-permissions! conn profile-id project-id)
-    (db/exec! conn [sql:project-files project-id])))
+    (cmd.files/get-project-files conn project-id)))
 
 ;; --- Query: File (By ID)
 
-(defn retrieve-object-thumbnails
+(s/def ::components-v2 ::us/boolean)
-  ([{:keys [pool]} file-id]
-   (let [sql (str/concat
-              "select object_id, data "
-              "  from file_object_thumbnail"
-              " where file_id=?")]
-     (->> (db/exec! pool [sql file-id])
-          (d/index-by :object-id :data))))
-
-  ([{:keys [pool]} file-id object-ids]
-   (with-open [conn (db/open pool)]
-     (let [sql (str/concat
-                "select object_id, data "
-                "  from file_object_thumbnail"
-                " where file_id=? and object_id = ANY(?)")
-           ids (db/create-array conn "text" (seq object-ids))]
-       (->> (db/exec! conn [sql file-id ids])
-            (d/index-by :object-id :data))))))
-
-(defn retrieve-file
-  [{:keys [pool] :as cfg} id features]
-  (let [file (->> (db/get-by-id pool :file id)
-                  (decode-row)
-                  (pmg/migrate-file))]
-
-    (if (contains? features "components/v2")
-      (update file :data ctf/migrate-to-components-v2)
-      (if (dm/get-in file [:data :options :components-v2])
-        (ex/raise :type :restriction
-                  :code :feature-disabled
-                  :hint "tried to open a components/v2 file with feature disabled")
-        file))))
-
-(s/def ::features ::us/set-of-strings)
 (s/def ::file
-  (s/keys :req-un [::profile-id ::id]
+  (s/and ::cmd.files/get-file
-          :opt-un [::features ::components-v2]))
+         (s/keys :opt-un [::components-v2])))
 
 (sv/defmethod ::file
   "Retrieve a file by its ID. Only authenticated users."
+  {::doc/added "1.0"
+   ::doc/deprecated "1.17"}
   [{:keys [pool] :as cfg} {:keys [profile-id id features components-v2] :as params}]
-  (let [perms    (get-permissions pool profile-id id)
+  (with-open [conn (db/open pool)]
-
+    (let [perms    (cmd.files/get-permissions pool profile-id id)
           ;; BACKWARD COMPATIBILTY with the components-v2 parameter
           features (cond-> (or features #{})
-                   components-v2 (conj features "components/v2"))]
+                     components-v2 (conj "components/v2"))]
-    (check-read-permissions! perms)
-    (let [file   (retrieve-file cfg id features)
-          thumbs (retrieve-object-thumbnails cfg id)]
-      (-> file
-          (assoc :thumbnails thumbs)
-          (assoc :permissions perms)))))
 
+      (cmd.files/check-read-permissions! perms)
+      (-> (cmd.files/get-file conn id features)
+          (assoc :permissions perms)))))
 
 ;; --- QUERY: page
 
-(defn- prune-objects
-  "Given the page data and the object-id returns the page data with all
-  other not needed objects removed from the `:objects` data
-  structure."
-  [{:keys [objects] :as page} object-id]
-  (let [objects (cph/get-children-with-self objects object-id)]
-     (assoc page :objects (d/index-by :id objects))))
-
-(defn- prune-thumbnails
-  "Given the page data, removes the `:thumbnail` prop from all
-  shapes."
-  [page]
-  (update page :objects d/update-vals #(dissoc % :thumbnail)))
-
-(s/def ::page-id ::us/uuid)
-(s/def ::object-id ::us/uuid)
-
 (s/def ::page
-  (s/and
+  (s/and ::cmd.files/get-page
-   (s/keys :req-un [::profile-id ::file-id]
+         (s/keys :opt-un [::components-v2])))
-           :opt-un [::page-id ::object-id ::features ::components-v2])
-   (fn [obj]
-     (if (contains? obj :object-id)
-       (contains? obj :page-id)
-       true))))
 
 (sv/defmethod ::page
   "Retrieves the page data from file and returns it. If no page-id is
@@ -300,288 +67,100 @@
   mandatory.
 
   Mainly used for rendering purposes."
-  [{:keys [pool] :as cfg} {:keys [profile-id file-id page-id object-id features components-v2] :as props}]
+  {::doc/added "1.5"
-  (check-read-permissions! pool profile-id file-id)
+   ::doc/deprecated "1.17"}
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id features components-v2] :as params}]
+  (with-open [conn (db/open pool)]
+    (cmd.files/check-read-permissions! conn profile-id file-id)
     (let [;; BACKWARD COMPATIBILTY with the components-v2 parameter
           features (cond-> (or features #{})
-                   components-v2 (conj features "components/v2"))
+                     components-v2 (conj "components/v2"))
+          params   (assoc params :features features)]
 
-        file     (retrieve-file cfg file-id features)
+      (cmd.files/get-page conn params))))
-        page-id  (or page-id (-> file :data :pages first))
-        page     (dm/get-in file [:data :pages-index page-id])]
-
-    (cond-> (prune-thumbnails page)
-      (uuid? object-id)
-      (prune-objects object-id))))
 
 ;; --- QUERY: file-data-for-thumbnail
 
-(defn- get-file-thumbnail-data
-  [cfg {:keys [data id] :as file}]
-  (letfn [;; function responsible on finding the frame marked to be
-          ;; used as thumbnail; the returned frame always have
-          ;; the :page-id set to the page that it belongs.
-          (get-thumbnail-frame [data]
-            (d/seek :use-for-thumbnail?
-                    (for [page  (-> data :pages-index vals)
-                          frame (-> page :objects ctt/get-frames)]
-                      (assoc frame :page-id (:id page)))))
-
-          ;; function responsible to filter objects data structure of
-          ;; all unneeded shapes if a concrete frame is provided. If no
-          ;; frame, the objects is returned untouched.
-          (filter-objects [objects frame-id]
-            (d/index-by :id (cph/get-children-with-self objects frame-id)))
-
-          ;; function responsible of assoc available thumbnails
-          ;; to frames and remove all children shapes from objects if
-          ;; thumbnails is available
-          (assoc-thumbnails [objects page-id thumbnails]
-            (loop [objects objects
-                   frames  (filter cph/frame-shape? (vals objects))]
-
-              (if-let [frame  (-> frames first)]
-                (let [frame-id (:id frame)
-                      object-id (str page-id frame-id)
-                      frame (if-let [thumb (get thumbnails object-id)]
-                              (assoc frame :thumbnail thumb :shapes [])
-                              (dissoc frame :thumbnail))
-
-                      children-ids
-                      (cph/get-children-ids objects frame-id)
-
-                      bounds
-                      (when (:show-content frame)
-                        (gsh/selection-rect (concat [frame] (->> children-ids (map (d/getf objects))))))
-
-                      frame
-                      (cond-> frame
-                        (some? bounds)
-                        (assoc :children-bounds bounds))]
-
-                  (if (:thumbnail frame)
-                    (recur (-> objects
-                               (assoc frame-id frame)
-                               (d/without-keys children-ids))
-                           (rest frames))
-                    (recur (assoc objects frame-id frame)
-                           (rest frames))))
-
-                objects)))]
-
-    (let [frame     (get-thumbnail-frame data)
-          frame-id  (:id frame)
-          page-id   (or (:page-id frame)
-                        (-> data :pages first))
-
-          page      (dm/get-in data [:pages-index page-id])
-          frame-ids (if (some? frame) (list frame-id) (map :id (ctt/get-frames (:objects page))))
-
-          obj-ids   (map #(str page-id %) frame-ids)
-          thumbs    (retrieve-object-thumbnails cfg id obj-ids)]
-
-      (cond-> page
-        ;; If we have frame, we need to specify it on the page level
-        ;; and remove the all other unrelated objects.
-        (some? frame-id)
-        (-> (assoc :thumbnail-frame-id frame-id)
-            (update :objects filter-objects frame-id))
-
-        ;; Assoc the available thumbnails and prune not visible shapes
-        ;; for avoid transfer unnecessary data.
-        :always
-        (update :objects assoc-thumbnails page-id thumbs)))))
-
 (s/def ::file-data-for-thumbnail
-  (s/keys :req-un [::profile-id ::file-id]
+  (s/and ::cmd.files/get-file-data-for-thumbnail
-          :opt-un [::components-v2 ::features]))
+         (s/keys :opt-un [::components-v2])))
 
 (sv/defmethod ::file-data-for-thumbnail
   "Retrieves the data for generate the thumbnail of the file. Used
   mainly for render thumbnails on dashboard."
+  {::doc/added "1.11"
+   ::doc/deprecated "1.17"}
   [{:keys [pool] :as cfg} {:keys [profile-id file-id features components-v2] :as props}]
-  (check-read-permissions! pool profile-id file-id)
+  (with-open [conn (db/open pool)]
+    (cmd.files/check-read-permissions! conn profile-id file-id)
     (let [;; BACKWARD COMPATIBILTY with the components-v2 parameter
           features (cond-> (or features #{})
-                   components-v2 (conj features "components/v2"))
+                     components-v2 (conj "components/v2"))
-        file     (retrieve-file cfg file-id features)]
+          file     (cmd.files/retrieve-file conn file-id features)]
       {:file-id file-id
        :revn (:revn file)
-     :page (get-file-thumbnail-data cfg file)}))
+       :page (cmd.files/get-file-data-for-thumbnail conn file)})))
-
 
 ;; --- Query: Shared Library Files
 
-(def ^:private sql:team-shared-files
+(s/def ::team-shared-files ::cmd.files/get-team-shared-files)
-  "select f.id,
-          f.revn,
-          f.data,
-          f.project_id,
-          f.created_at,
-          f.modified_at,
-          f.name,
-          f.is_shared
-     from file as f
-    inner join project as p on (p.id = f.project_id)
-    where f.is_shared = true
-      and f.deleted_at is null
-      and p.deleted_at is null
-      and p.team_id = ?
-    order by f.modified_at desc")
-
-(s/def ::team-shared-files
-  (s/keys :req-un [::profile-id ::team-id]))
 
 (sv/defmethod ::team-shared-files
-  [{:keys [pool] :as cfg} {:keys [team-id] :as params}]
+  {::doc/added "1.3"
-  (let [assets-sample
+   ::doc/deprecated "1.17"}
-        (fn [assets limit]
+  [{:keys [pool] :as cfg} {:keys [profile-id team-id] :as params}]
-          (let [sorted-assets (->> (vals assets)
+  (with-open [conn (db/open pool)]
-                                   (sort-by #(str/lower (:name %))))]
+    (teams/check-read-permissions! conn profile-id team-id)
-
+    (cmd.files/get-team-shared-files conn params)))
-          {:count (count sorted-assets)
-           :sample (into [] (take limit sorted-assets))}))
-
-        library-summary
-        (fn [data]
-          {:components (assets-sample (:components data) 4)
-           :colors (assets-sample (:colors data) 3)
-           :typographies (assets-sample (:typographies data) 3)})
-
-        xform (comp
-                (map decode-row)
-                (map #(assoc % :library-summary (library-summary (:data %))))
-                (map #(dissoc % :data)))]
-
-    (into #{} xform (db/exec! pool [sql:team-shared-files team-id]))))
 
 
 ;; --- Query: File Libraries used by a File
 
-(def ^:private sql:file-libraries
+(s/def ::file-libraries ::cmd.files/get-file-libraries)
-  "WITH RECURSIVE libs AS (
-     SELECT fl.*, flr.synced_at
-       FROM file AS fl
-       JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
-      WHERE flr.file_id = ?::uuid
-    UNION
-     SELECT fl.*, flr.synced_at
-       FROM file AS fl
-       JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
-       JOIN libs AS l ON (flr.file_id = l.id)
-   )
-   SELECT l.id,
-          l.data,
-          l.project_id,
-          l.created_at,
-          l.modified_at,
-          l.deleted_at,
-          l.name,
-          l.revn,
-          l.synced_at
-     FROM libs AS l
-    WHERE l.deleted_at IS NULL OR l.deleted_at > now();")
-
-(defn retrieve-file-libraries
-  [{:keys [pool] :as cfg} is-indirect file-id]
-  (let [xform (comp
-               (map #(assoc % :is-indirect is-indirect))
-               (map decode-row))]
-    (into #{} xform (db/exec! pool [sql:file-libraries file-id]))))
-
-(s/def ::file-libraries
-  (s/keys :req-un [::profile-id ::file-id]))
 
 (sv/defmethod ::file-libraries
+  {::doc/added "1.3"
+   ::doc/deprecated "1.17"}
   [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}]
-  (check-read-permissions! pool profile-id file-id)
+  (with-open [conn (db/open pool)]
-  (retrieve-file-libraries cfg false file-id))
+    (cmd.files/check-read-permissions! conn profile-id file-id)
+    (cmd.files/get-file-libraries conn false file-id)))
 
 
 ;; --- Query: Files that use this File library
 
-(def ^:private sql:library-using-files
+(s/def ::library-using-files ::cmd.files/get-library-file-references)
-  "SELECT f.id,
-          f.name
-     FROM file_library_rel AS flr
-     JOIN file AS f ON (f.id = flr.file_id)
-    WHERE flr.library_file_id = ?
-      AND (f.deleted_at IS NULL OR f.deleted_at > now())")
-
-(s/def ::library-using-files
-  (s/keys :req-un [::profile-id ::file-id]))
 
 (sv/defmethod ::library-using-files
+  {::doc/added "1.13"
+   ::doc/deprecated "1.17"}
   [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}]
-  (check-read-permissions! pool profile-id file-id)
+  (with-open [conn (db/open pool)]
-  (db/exec! pool [sql:library-using-files file-id]))
+    (cmd.files/check-read-permissions! conn profile-id file-id)
+    (cmd.files/get-library-file-references conn file-id)))
 
 ;; --- QUERY: team-recent-files
 
-(def sql:team-recent-files
+(s/def ::team-recent-files ::cmd.files/get-team-recent-files)
-  "with recent_files as (
-     select f.id,
-            f.revn,
-            f.project_id,
-            f.created_at,
-            f.modified_at,
-            f.name,
-            f.is_shared,
-            row_number() over w as row_num
-       from file as f
-       join project as p on (p.id = f.project_id)
-      where p.team_id = ?
-        and p.deleted_at is null
-        and f.deleted_at is null
-     window w as (partition by f.project_id order by f.modified_at desc)
-      order by f.modified_at desc
-   )
-   select * from recent_files where row_num <= 10;")
-
-
-(s/def ::team-recent-files
-  (s/keys :req-un [::profile-id ::team-id]))
 
 (sv/defmethod ::team-recent-files
+  {::doc/added "1.0"
+   ::doc/deprecated "1.17"}
   [{:keys [pool] :as cfg} {:keys [profile-id team-id]}]
-  (teams/check-read-permissions! pool profile-id team-id)
+  (with-open [conn (db/open pool)]
-  (db/exec! pool [sql:team-recent-files team-id]))
+    (teams/check-read-permissions! conn profile-id team-id)
+    (cmd.files/get-team-recent-files conn team-id)))
+
 
 ;; --- QUERY: get file thumbnail
 
-(s/def ::revn ::us/integer)
+(s/def ::file-thumbnail ::cmd.files/get-file-thumbnail)
-
-(s/def ::file-thumbnail
-  (s/keys :req-un [::profile-id ::file-id]
-          :opt-un [::revn]))
 
 (sv/defmethod ::file-thumbnail
+  {::doc/added "1.13"
+   ::doc/deprecated "1.17"}
   [{:keys [pool]} {:keys [profile-id file-id revn]}]
-  (check-read-permissions! pool profile-id file-id)
+  (with-open [conn (db/open pool)]
-  (let [sql (sql/select :file-thumbnail
+    (cmd.files/check-read-permissions! conn profile-id file-id)
-                        (cond-> {:file-id file-id}
+    (-> (cmd.files/get-file-thumbnail conn file-id revn)
-                          revn (assoc :revn revn))
+        (with-meta {::rpc/transform-response (rpch/http-cache {:max-age (* 1000 60 60)})}))))
-                        {:limit 1
-                         :order-by [[:revn :desc]]})
-
-        row (db/exec-one! pool sql)]
-
-    (when-not row
-      (ex/raise :type :not-found
-                :code :file-thumbnail-not-found))
-
-    (with-meta {:data (:data row)
-                :props (some-> (:props row) db/decode-transit-pgobject)
-                :revn (:revn row)
-                :file-id (:file-id row)}
-      {::rpc/transform-response (rpch/http-cache {:max-age (* 1000 60 60)})})))
-
-;; --- Helpers
-
-(defn decode-row
-  [{:keys [data changes features] :as row}]
-  (when row
-    (cond-> row
-      features (assoc :features (db/decode-pgarray features))
-      changes  (assoc :changes (blob/decode changes))
-      data     (assoc :data (blob/decode data)))))

backend/src/app/rpc/queries/fonts.clj

@@ -8,7 +8,7 @@
   (:require
    [app.common.spec :as us]
    [app.db :as db]
-   [app.rpc.queries.files :as files]
+   [app.rpc.commands.files :as files]
    [app.rpc.queries.projects :as projects]
    [app.rpc.queries.teams :as teams]
    [app.util.services :as sv]

backend/src/app/rpc/queries/viewer.clj

@@ -6,88 +6,26 @@
 
 (ns app.rpc.queries.viewer
   (:require
-   [app.common.exceptions :as ex]
    [app.common.spec :as us]
    [app.db :as db]
-   [app.rpc.commands.comments :as comments]
+   [app.rpc.commands.viewer :as viewer]
-   [app.rpc.queries.files :as files]
+   [app.rpc.doc :as-alias doc]
-   [app.rpc.queries.share-link :as slnk]
    [app.util.services :as sv]
-   [clojure.spec.alpha :as s]
+   [clojure.spec.alpha :as s]))
-   [promesa.core :as p]))
 
-;; --- Query: View Only Bundle
-
-(defn- retrieve-project
-  [pool id]
-  (db/get-by-id pool :project id {:columns [:id :name :team-id]}))
-
-(defn- retrieve-bundle
-  [{:keys [pool] :as cfg} file-id profile-id features]
-  (p/let [file    (files/retrieve-file cfg file-id features)
-          project (retrieve-project pool (:project-id file))
-          libs    (files/retrieve-file-libraries cfg false file-id)
-          users   (comments/get-file-comments-users pool file-id profile-id)
-
-          links   (->> (db/query pool :share-link {:file-id file-id})
-                       (mapv slnk/decode-share-link-row))
-
-          fonts   (db/query pool :team-font-variant
-                            {:team-id (:team-id project)
-                             :deleted-at nil})]
-    {:file file
-     :users users
-     :fonts fonts
-     :project project
-     :share-links links
-     :libraries libs}))
-
-(s/def ::file-id ::us/uuid)
-(s/def ::profile-id ::us/uuid)
-(s/def ::share-id ::us/uuid)
-(s/def ::features ::us/set-of-strings)
-
-;; TODO: deprecated, should be removed when version >= 1.18
 (s/def ::components-v2 ::us/boolean)
-
 (s/def ::view-only-bundle
-  (s/keys :req-un [::file-id]
+  (s/and ::viewer/get-view-only-bundle
-          :opt-un [::profile-id ::share-id ::features ::components-v2]))
+         (s/keys :opt-un [::components-v2])))
 
-(sv/defmethod ::view-only-bundle {:auth false}
+(sv/defmethod ::view-only-bundle
-  [{:keys [pool] :as cfg} {:keys [profile-id file-id share-id features components-v2] :as params}]
+  {:auth false
-  (p/let [;; BACKWARD COMPATIBILTY with the components-v2 parameter
+   ::doc/added "1.3"
+   ::doc/deprecated "1.17"}
+  [{:keys [pool] :as cfg} {:keys [features components-v2] :as params}]
+  (with-open [conn (db/open pool)]
+    (let [;; BACKWARD COMPATIBILTY with the components-v2 parameter
           features (cond-> (or features #{})
-                     components-v2 (conj features "components/v2"))
+                     components-v2 (conj "components/v2"))
-
+          params   (assoc params :features features)]
-          slink  (slnk/retrieve-share-link pool file-id share-id)
+      (viewer/get-view-only-bundle conn params))))
-          perms  (files/get-permissions pool profile-id file-id share-id)
-          thumbs (files/retrieve-object-thumbnails cfg file-id)
-          bundle (p/-> (retrieve-bundle cfg file-id profile-id features)
-                       (assoc :permissions perms)
-                       (assoc-in [:file :thumbnails] thumbs))]
-    ;; When we have neither profile nor share, we just return a not
-    ;; found response to the user.
-    (do
-      (when (and (not profile-id)
-                 (not slink))
-        (ex/raise :type :not-found
-                  :code :object-not-found))
-
-      ;; When we have only profile, we need to check read permissions
-      ;; on file.
-      (when (and profile-id (not slink))
-        (files/check-read-permissions! pool profile-id file-id))
-
-      (cond-> bundle
-        (some? slink)
-        (assoc :share slink)
-
-        (and (some? slink)
-           (not (contains? (:flags slink) "view-all-pages")))
-        (update-in [:file :data] (fn [data]
-                                   (let [allowed-pages (:pages slink)]
-                                     (-> data
-                                         (update :pages (fn [pages] (filterv #(contains? allowed-pages %) pages)))
-                                         (update :pages-index (fn [index] (select-keys index allowed-pages)))))))))))
-

backend/src/app/srepl/main.clj

@@ -135,8 +135,7 @@
                   (update :features conj "storage/pointer-map"))))
 
           (migrate-to-omap [data file-id]
-            (binding [pmap/*tracked* (atom {})
+            (binding [pmap/*tracked* (atom {})]
-                      pmap/*metadata* {:file-id file-id}]
               (let [data (-> data
                              (update :pages-index update-vals pmap/wrap)
                              (update :components pmap/wrap))]
@@ -144,7 +143,6 @@
                   (db/insert! h/*conn* :file-data-fragment
                               {:id id
                                :file-id file-id
-                               :metadata (-> item meta db/tjson)
                                :content (-> item deref blob/encode)}))
                 data)))]
 

backend/src/app/tasks/file_gc.clj

@@ -17,11 +17,12 @@
    [app.common.types.shape-tree :as ctt]
    [app.config :as cf]
    [app.db :as db]
+   [app.rpc.commands.files :as files]
    [app.util.blob :as blob]
+   [app.util.pointer-map :as pmap]
    [app.util.time :as dt]
    [clojure.set :as set]
    [clojure.spec.alpha :as s]
-   [cuerdas.core :as str]
    [integrant.core :as ig]))
 
 (declare ^:private retrieve-candidates)
@@ -55,7 +56,7 @@
               (recur (inc total)
                      (rest files)))
             (do
-              (l/info :hint "task finished" :min-age (dt/format-duration min-age) :total total)
+              (l/info :hint "task finished" :min-age (dt/format-duration min-age) :processed total)
 
               ;; Allow optional rollback passed by params
               (when (:rollback? params)
@@ -72,6 +73,7 @@
   "select f.id,
           f.data,
           f.revn,
+          f.features,
           f.modified_at
      from file as f
     where f.has_media_trimmed is false
@@ -86,17 +88,22 @@
   (if id
     (do
       (l/warn :hint "explicit file id passed on params" :id id)
-      (db/query conn :file {:id id}))
+      (->> (db/query conn :file {:id id})
+           (map #(update % :features db/decode-pgarray #{}))))
     (let [interval  (db/interval min-age)
           get-chunk (fn [cursor]
                       (let [rows (db/exec! conn [sql:retrieve-candidates-chunk interval cursor])]
-                        [(some->> rows peek :modified-at) (seq rows)]))]
+                        [(some->> rows peek :modified-at)
+                         (map #(update % :features db/decode-pgarray #{}) rows)]))]
+
       (d/iteration get-chunk
                    :vf second
                    :kf first
                    :initk (dt/now)))))
 
 (defn collect-used-media
+  "Analyzes the file data and collects all references to external
+  assets. Returns a set of ids."
   [data]
   (let [xform (comp
                (map :objects)
@@ -152,8 +159,7 @@
         unused (set/difference stored using)]
 
     (when (seq unused)
-      (let [sql (str/concat
+      (let [sql (str "delete from file_object_thumbnail "
-                 "delete from file_object_thumbnail "
                      " where file_id=? and object_id=ANY(?)")
             res (db/exec-one! conn [sql file-id (db/create-array conn "text" unused)])]
         (l/debug :hint "delete file object thumbnails" :file-id file-id :total (:next.jdbc/update-count res))))))
@@ -233,10 +239,27 @@
                     {:data new-data}
                     {:id library-id})))))
 
+(def ^:private sql:get-unused-fragments
+  "SELECT id FROM file_data_fragment
+    WHERE file_id = ? AND id != ALL(?::uuid[])")
+
+(defn- clean-data-fragments!
+  [conn file-id data]
+  (let [used (->> (concat (vals data)
+                          (vals (:pages-index data)))
+                  (into #{} (comp (filter pmap/pointer-map?)
+                                  (map pmap/get-id)))
+                  (db/create-array conn "uuid"))
+        rows (db/exec! conn [sql:get-unused-fragments file-id used])]
+    (doseq [fragment-id (map :id rows)]
+      (l/trace :hint "remove unused file data fragment" :id (str fragment-id))
+      (db/delete! conn :file-data-fragment {:id fragment-id :file-id file-id}))))
+
 (defn- process-file
-  [{:keys [conn] :as cfg} {:keys [id data revn modified-at] :as file}]
+  [{:keys [conn] :as cfg} {:keys [id data revn modified-at features] :as file}]
   (l/debug :hint "processing file" :id id :modified-at modified-at)
 
+  (binding [pmap/*load-fn* (partial files/load-pointer conn id)]
     (let [data (-> (blob/decode data)
                    (assoc :id id)
                    (pmg/migrate-data))]
@@ -246,8 +269,11 @@
       (clean-file-thumbnails! conn id revn)
       (clean-deleted-components! conn id data)
 
+      (when (contains? features "storage/pointer-map")
+        (clean-data-fragments! conn id data))
+
       ;; Mark file as trimmed
       (db/update! conn :file
                   {:has-media-trimmed true}
                   {:id id})
-    nil))
+      nil)))

backend/src/app/util/fressian.clj

@@ -86,17 +86,25 @@
   (write-tag! w tag 1)
   (write-list! w o))
 
+(defn begin-closed-list!
+  [^StreamingWriter w]
+  (.beginClosedList w))
+
+(defn end-list!
+  [^StreamingWriter w]
+  (.endList w))
+
 (defn write-map-like
   "Writes a map as Fressian with the tag 'map' and all keys cached."
   [tag ^Writer w m]
   (write-tag! w tag 1)
-  (.beginClosedList ^StreamingWriter w)
+  (begin-closed-list! w)
   (loop [items (seq m)]
     (when-let [^clojure.lang.MapEntry item (first items)]
       (write-object! w (.key item) true)
       (write-object! w (.val item))
       (recur (rest items))))
-  (.endList ^StreamingWriter w))
+  (end-list! w))
 
 (defn read-map-like
   [^Reader rdr]

backend/test/app/services_files_test.clj

@@ -496,7 +496,8 @@
         (t/is (contains? (:objects result) shape1-id))
         (t/is (contains? (:objects result) frame2-id))
         (t/is (contains? (:objects result) shape2-id))
-        (t/is (contains? (:objects result) uuid/zero)))
+        (t/is (contains? (:objects result) uuid/zero))
+        )
 
       ;; Query :page RPC method with page-id
       (let [data {::th/type :page
@@ -523,6 +524,7 @@
                   :components-v2 true}
             {:keys [error result] :as out} (th/query! data)]
         ;; (th/print-result! out)
+        (t/is (nil? error))
         (t/is (map? result))
         (t/is (contains? result :objects))
         (t/is (contains? (:objects result) frame1-id))
@@ -542,7 +544,9 @@
         (t/is (not (th/success? out)))
         (let [{:keys [type code]} (-> out :error ex-data)]
           (t/is (= :validation type))
-          (t/is (= :spec-validation code)))))
+          (t/is (= :spec-validation code))))
+
+      )
 
     (t/testing "RPC :file-data-for-thumbnail"
       ;; Insert a thumbnail data for the frame-id

backend/test/app/test_helpers.clj

@@ -18,7 +18,9 @@
    [app.media]
    [app.migrations]
    [app.rpc.commands.auth :as cmd.auth]
-   [app.rpc.mutations.files :as files]
+   [app.rpc.commands.files :as files]
+   [app.rpc.commands.files.create :as files.create]
+   [app.rpc.commands.files.update :as files.update]
    [app.rpc.mutations.profile :as profile]
    [app.rpc.mutations.projects :as projects]
    [app.rpc.mutations.teams :as teams]
@@ -178,7 +180,7 @@
    (us/assert uuid? profile-id)
    (us/assert uuid? project-id)
    (with-open [conn (db/open pool)]
-     (#'files/create-file conn
+     (files.create/create-file conn
                                (merge {:id (mk-uuid "file" i)
                                        :name (str "file" i)
                                        :components-v2 true}
@@ -259,7 +261,7 @@
   ([params] (create-file-role* *pool* params))
   ([pool {:keys [file-id profile-id role] :or {role :owner}}]
    (with-open [conn (db/open pool)]
-     (#'files/create-file-role conn {:file-id file-id
+     (files.create/create-file-role! conn {:file-id file-id
                                            :profile-id profile-id
                                            :role role}))))
 
@@ -268,15 +270,15 @@
   ([pool {:keys [file-id changes session-id profile-id revn]
           :or {session-id (uuid/next) revn 0}}]
    (with-open [conn (db/open pool)]
-     (let [file    (db/get-by-id conn :file file-id)
+     (let [msgbus    (:app.msgbus/msgbus *system*)
-           msgbus  (:app.msgbus/msgbus *system*)
+           metrics   (:app.metrics/metrics *system*)
-           metrics (:app.metrics/metrics *system*)]
+           features  #{"components/v2"}]
-       (#'files/update-file {:conn conn
+       (files.update/update-file {:conn conn
                                   :msgbus msgbus
                                   :metrics metrics}
-                            {:file file
+                                 {:id file-id
                                   :revn revn
-                             :components-v2 true
+                                  :features features
                                   :changes changes
                                   :session-id session-id
                                   :profile-id profile-id})))))

common/src/app/common/files/features.cljc

@@ -6,5 +6,12 @@
 
 (ns app.common.files.features)
 
+;; A set of enabled by default file features. Will be used in feature
+;; negotiation on obtaining files from backend.
+
+(def enabled #{})
+
+(def ^:dynamic *previous* #{})
 (def ^:dynamic *current* #{})
-(def ^:dynamic *wrap-objects-fn* identity)
+(def ^:dynamic *wrap-with-objects-map-fn* identity)
+(def ^:dynamic *wrap-with-pointer-map-fn* identity)

common/src/app/common/transit.cljc

@@ -33,6 +33,10 @@
 (def write-handler-map (atom nil))
 (def read-handler-map (atom nil))
 
+;; A generic pointer; mainly used for deserialize backend pointer-map
+;; instances that serializes to pointer but may in other ways.
+(defrecord Pointer [id])
+
 ;; --- HELPERS
 
 #?(:clj
@@ -133,6 +137,11 @@
                     (.fromMillis ^js lxn/DateTime ms))))
   :wfn (comp str inst-ms)}
 
+ {:id "penpot/pointer"
+  :class Pointer
+  :rfn (fn [[id meta]]
+         (Pointer. id meta {}))}
+
  #?(:clj
     {:id "m"
      :class OffsetDateTime

common/src/app/common/types/components_list.cljc

@@ -17,7 +17,7 @@
 (defn add-component
   [file-data {:keys [id name path main-instance-id main-instance-page shapes]}]
   (let [components-v2  (dm/get-in file-data [:options :components-v2])
-        wrap-object-fn feat/*wrap-objects-fn*]
+        wrap-object-fn feat/*wrap-with-objects-map-fn*]
     (cond-> file-data
       :always
       (assoc-in [:components id]
@@ -35,7 +35,7 @@
 
 (defn mod-component
   [file-data {:keys [id name path objects]}]
-  (let [wrap-objects-fn feat/*wrap-objects-fn*]
+  (let [wrap-objects-fn feat/*wrap-with-objects-map-fn*]
     (update-in file-data [:components id]
                (fn [component]
                  (let [objects (some-> objects wrap-objects-fn)]

common/src/app/common/types/page.cljc

@@ -50,11 +50,13 @@
 
 (defn make-empty-page
   [id name]
-  (let [wrap-fn ffeat/*wrap-objects-fn*]
+  (let [wrap-objects-fn ffeat/*wrap-with-objects-map-fn*
+        wrap-pointer-fn ffeat/*wrap-with-pointer-map-fn*]
     (-> empty-page-data
         (assoc :id id)
         (assoc :name name)
-        (update :objects wrap-fn))))
+        (update :objects wrap-objects-fn)
+        (wrap-pointer-fn))))
 
 ;; --- Helpers for flow
 

frontend/src/app/main/data/dashboard.cljs

@@ -110,7 +110,7 @@
     ptk/WatchEvent
     (watch [_ state _]
       (let [team-id (:current-team-id state)]
-        (->> (rp/query :team-members {:team-id team-id})
+        (->> (rp/query! :team-members {:team-id team-id})
              (rx/map team-members-fetched))))))
 
 ;; --- EVENT: fetch-team-stats
@@ -128,7 +128,7 @@
     ptk/WatchEvent
     (watch [_ state _]
       (let [team-id (:current-team-id state)]
-        (->> (rp/query :team-stats {:team-id team-id})
+        (->> (rp/query! :team-stats {:team-id team-id})
              (rx/map team-stats-fetched))))))
 
 ;; --- EVENT: fetch-team-invitations
@@ -146,7 +146,7 @@
     ptk/WatchEvent
     (watch [_ state _]
       (let [team-id (:current-team-id state)]
-        (->> (rp/query :team-invitations {:team-id team-id})
+        (->> (rp/query! :team-invitations {:team-id team-id})
              (rx/map team-invitations-fetched))))))
 
 ;; --- EVENT: fetch-projects
@@ -165,7 +165,7 @@
     ptk/WatchEvent
     (watch [_ state _]
       (let [team-id (:current-team-id state)]
-        (->> (rp/query :projects {:team-id team-id})
+        (->> (rp/query! :projects {:team-id team-id})
              (rx/map projects-fetched))))))
 
 ;; --- EVENT: search
@@ -193,7 +193,7 @@
     (watch [_ state _]
       (let [team-id (:current-team-id state)
             params  (assoc params :team-id team-id)]
-        (->> (rp/query :search-files params)
+        (->> (rp/query! :search-files params)
              (rx/map search-result-fetched))))))
 
 ;; --- EVENT: files
@@ -222,7 +222,7 @@
   (ptk/reify ::fetch-files
     ptk/WatchEvent
     (watch [_ _ _]
-      (->> (rp/query :project-files {:project-id project-id})
+      (->> (rp/cmd! :get-project-files {:project-id project-id})
            (rx/map #(files-fetched project-id %))))))
 
 ;; --- EVENT: shared-files
@@ -243,7 +243,7 @@
     ptk/WatchEvent
     (watch [_ state _]
       (let [team-id (:current-team-id state)]
-        (->> (rp/query :team-shared-files {:team-id team-id})
+        (->> (rp/cmd! :get-team-shared-files {:team-id team-id})
              (rx/map shared-files-fetched))))))
 
 ;; --- EVENT: Get files that use this shared-file
@@ -269,7 +269,8 @@
     ptk/WatchEvent
     (watch [_ _ _]
       (->> (rx/from files)
-           (rx/mapcat (fn [file]  (rp/query :library-using-files {:file-id (:id file)})))
+           (rx/map :id)
+           (rx/mapcat #(rp/cmd! :get-library-file-references {:file-id %}))
            (rx/reduce into [])
            (rx/map library-using-files-fetched)))))
 
@@ -292,7 +293,7 @@
      ptk/WatchEvent
      (watch [_ state _]
        (let [team-id (or team-id (:current-team-id state))]
-         (->> (rp/query :team-recent-files {:team-id team-id})
+         (->> (rp/cmd! :get-team-recent-files {:team-id team-id})
               (rx/map recent-files-fetched)))))))
 
 
@@ -588,7 +589,7 @@
 
             new-name (str name " " (tr "dashboard.copy-suffix"))]
 
-        (->> (rp/command! :duplicate-project {:project-id id :name new-name})
+        (->> (rp/cmd! :duplicate-project {:project-id id :name new-name})
              (rx/tap on-success)
              (rx/map project-duplicated)
              (rx/catch on-error))))))
@@ -608,7 +609,7 @@
              :or {on-success identity
                   on-error rx/throw}} (meta params)]
 
-        (->> (rp/command! :move-project {:project-id id :team-id team-id})
+        (->> (rp/cmd! :move-project {:project-id id :team-id team-id})
              (rx/tap on-success)
              (rx/catch on-error))))))
 
@@ -683,7 +684,7 @@
     ptk/WatchEvent
     (watch [_ state _]
       (let [team-id (uuid/uuid (get-in state [:route :path-params :team-id]))]
-        (->> (rp/mutation :delete-file {:id id})
+        (->> (rp/cmd! :delete-file {:id id})
              (rx/map #(file-deleted team-id project-id)))))))
 
 ;; --- Rename File
@@ -706,7 +707,7 @@
     ptk/WatchEvent
     (watch [_ _ _]
       (let [params (select-keys params [:id :name])]
-        (->> (rp/mutation :rename-file params)
+        (->> (rp/cmd! :rename-file params)
              (rx/ignore))))))
 
 ;; --- Set File shared
@@ -731,7 +732,7 @@
     ptk/WatchEvent
     (watch [_ _ _]
       (let [params {:id id :is-shared is-shared}]
-        (->> (rp/mutation :set-file-shared params)
+        (->> (rp/cmd! :set-file-shared params)
              (rx/ignore))))))
 
 ;; --- EVENT: create-file
@@ -774,7 +775,7 @@
                          (assoc :name name)
                          (assoc :features features))]
 
-        (->> (rp/mutation! :create-file params)
+        (->> (rp/cmd! :create-file params)
              (rx/tap on-success)
              (rx/map #(with-meta (file-created %) (meta it)))
              (rx/catch on-error))))))
@@ -794,7 +795,7 @@
 
             new-name (str name " " (tr "dashboard.copy-suffix"))]
 
-        (->> (rp/command! :duplicate-file {:file-id id :name new-name})
+        (->> (rp/cmd! :duplicate-file {:file-id id :name new-name})
              (rx/tap on-success)
              (rx/map file-created)
              (rx/catch on-error))))))
@@ -816,7 +817,7 @@
       (let [{:keys [on-success on-error]
              :or {on-success identity
                   on-error rx/throw}} (meta params)]
-        (->> (rp/command! :move-files {:ids ids :project-id project-id})
+        (->> (rp/cmd! :move-files {:ids ids :project-id project-id})
              (rx/tap on-success)
              (rx/catch on-error))))))
 
@@ -836,8 +837,7 @@
       (let [{:keys [on-success on-error]
              :or {on-success identity
                   on-error rx/throw}} (meta params)]
-
+        (->> (rp/cmd! :clone-template {:project-id project-id :template-id template-id})
-        (->> (rp/command! :clone-template {:project-id project-id :template-id template-id})
              (rx/tap on-success)
              (rx/catch on-error))))))
 

frontend/src/app/main/data/workspace.cljs

@@ -357,7 +357,7 @@
     ptk/WatchEvent
     (watch [_ _ _]
       (let [params {:id id :name name}]
-        (->> (rp/mutation :rename-file params)
+        (->> (rp/cmd! :rename-file params)
              (rx/ignore))))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

frontend/src/app/main/data/workspace/libraries.cljs

@@ -7,6 +7,7 @@
 (ns app.main.data.workspace.libraries
   (:require
    [app.common.data :as d]
+   [app.common.files.features :as ffeat]
    [app.common.geom.point :as gpt]
    [app.common.logging :as log]
    [app.common.pages :as cp]
@@ -739,9 +740,9 @@
               ;; TODO: look for a more precise way of syncing this.
               ;; Maybe by using the stream (second argument passed to watch)
               ;; to wait for the corresponding changes-committed and then proceed
-               ;; with the :update-sync mutation.
+              ;; with the :update-file-library-sync-status mutation.
               (rx/concat (rx/timer 3000)
-                          (rp/mutation :update-sync
+                         (rp/cmd! :update-file-library-sync-status
                                   {:file-id file-id
                                    :library-id library-id})))
             (when (and (seq (:redo-changes library-changes))
@@ -788,7 +789,7 @@
 
     ptk/WatchEvent
     (watch [_ state _]
-      (rp/mutation :ignore-sync
+      (rp/cmd! :ignore-file-library-sync-status
                {:file-id (get-in state [:workspace-file :id])
                 :date (dt/now)}))))
 
@@ -880,7 +881,7 @@
     ptk/WatchEvent
     (watch [_ _ _]
       (let [params {:id id :is-shared is-shared}]
-        (->> (rp/mutation :set-file-shared params)
+        (->> (rp/cmd! :set-file-shared params)
              (rx/ignore))))))
 
 (defn- shared-files-fetched
@@ -898,7 +899,7 @@
   (ptk/reify ::fetch-shared-files
     ptk/WatchEvent
     (watch [_ _ _]
-      (->> (rp/query :team-shared-files {:team-id team-id})
+      (->> (rp/cmd! :get-team-shared-files {:team-id team-id})
            (rx/map shared-files-fetched)))))
 
 ;; --- Link and unlink Files
@@ -908,13 +909,16 @@
   (ptk/reify ::attach-library
     ptk/WatchEvent
     (watch [_ state _]
-      (let [components-v2 (features/active-feature? state :components-v2)
+      (let [features (cond-> ffeat/enabled
-            fetched       #(assoc-in %2 [:workspace-libraries (:id %1)] %1)
+                       (features/active-feature? state :components-v2)
-            params        {:file-id file-id
+                       (conj "components/v2"))]
-                           :library-id library-id}]
+        (rx/concat
-        (->> (rp/mutation :link-file-to-library params)
+         (->> (rp/cmd! :link-file-to-library {:file-id file-id :library-id library-id})
-             (rx/mapcat #(rp/query :file {:id library-id :components-v2 components-v2}))
+              (rx/ignore))
-             (rx/map #(partial fetched %)))))))
+         (->> (rp/cmd! :get-file {:id library-id :features features})
+              (rx/map (fn [file]
+                        (fn [state]
+                          (assoc-in state [:workspace-libraries library-id] file))))))))))
 
 (defn unlink-file-from-library
   [file-id library-id]
@@ -927,5 +931,5 @@
     (watch [_ _ _]
       (let [params {:file-id file-id
                     :library-id library-id}]
-        (->> (rp/mutation :unlink-file-from-library params)
+        (->> (rp/cmd! :unlink-file-from-library params)
              (rx/ignore))))))

frontend/src/app/main/data/workspace/persistence.cljs

@@ -8,6 +8,7 @@
   (:require
    [app.common.data :as d]
    [app.common.data.macros :as dm]
+   [app.common.files.features :as ffeat]
    [app.common.logging :as log]
    [app.common.pages :as cp]
    [app.common.pages.changes-spec :as pcs]
@@ -18,7 +19,6 @@
    [app.config :as cf]
    [app.main.data.dashboard :as dd]
    [app.main.data.fonts :as df]
-   [app.main.data.modal :as modal]
    [app.main.data.workspace.changes :as dch]
    [app.main.data.workspace.state-helpers :as wsh]
    [app.main.data.workspace.thumbnails :as dwt]
@@ -26,7 +26,6 @@
    [app.main.repo :as rp]
    [app.main.store :as st]
    [app.util.http :as http]
-   [app.util.i18n :as i18n :refer [tr]]
    [app.util.router :as rt]
    [app.util.time :as dt]
    [beicon.core :as rx]
@@ -138,7 +137,11 @@
   (ptk/reify ::persist-changes
     ptk/WatchEvent
     (watch [_ state _]
-      (let [features (cond-> #{}
+      (let [;; this features set does not includes the ffeat/enabled
+            ;; because they are already available on the backend and
+            ;; this request provides a set of features to enable in
+            ;; this request.
+            features (cond-> #{}
                        (features/active-feature? state :components-v2)
                        (conj "components/v2"))
             sid      (:session-id state)
@@ -150,7 +153,7 @@
                       :features features}]
 
         (when (= file-id (:id params))
-          (->> (rp/mutation :update-file params)
+          (->> (rp/cmd! :update-file params)
                (rx/mapcat (fn [lagged]
                             (log/debug :hint "changes persisted" :lagged (count lagged))
                             (let [lagged (cond->> lagged
@@ -285,14 +288,13 @@
     ptk/WatchEvent
     (watch [_ state _]
       (let [share-id (-> state :viewer-local :share-id)
-            features (cond-> #{}
+            features (cond-> ffeat/enabled
                        (features/active-feature? state :components-v2)
                        (conj "components/v2"))]
-
+        (->> (rx/zip (rp/cmd! :get-raw-file {:id file-id :features features})
-        (->> (rx/zip (rp/query! :file-raw {:id file-id :features features})
                      (rp/query! :team-users {:file-id file-id})
                      (rp/query! :project {:id project-id})
-                     (rp/query! :file-libraries {:file-id file-id})
+                     (rp/cmd! :get-file-libraries {:file-id file-id})
                      (rp/cmd! :get-profiles-for-file-comments {:file-id file-id :share-id share-id}))
              (rx/take 1)
              (rx/map (fn [[file-raw users project libraries file-comments-users]]
@@ -303,16 +305,7 @@
                         :file-comments-users file-comments-users}))
              (rx/mapcat (fn [{:keys [project] :as bundle}]
                           (rx/of (ptk/data-event ::bundle-fetched bundle)
-                                 (df/load-team-fonts (:team-id project)))))
+                                 (df/load-team-fonts (:team-id project))))))))))
-             (rx/catch (fn [err]
-                         (if (and (= (:type err) :restriction)
-                                  (= (:code err) :feature-disabled))
-                           (let [team-id (:current-team-id state)]
-                             (rx/of (modal/show
-                                      {:type :alert
-                                       :message (tr "errors.components-v2")
-                                       :on-accept #(st/emit! (rt/nav :dashboard-projects {:team-id team-id}))})))
-                           (rx/throw err)))))))))
 
 ;; --- Helpers
 

frontend/src/app/main/data/workspace/thumbnails.cljs

@@ -81,7 +81,7 @@
                      (rx/merge
                       ;; Update the local copy of the thumbnails so we don't need to request it again
                       (rx/of #(assoc-in % [:workspace-file :thumbnails object-id] data))
-                      (->> (rp/mutation! :upsert-file-object-thumbnail params)
+                      (->> (rp/cmd! :upsert-file-object-thumbnail params)
                            (rx/ignore))))
 
                    (rx/empty))))))))))

frontend/src/app/main/errors.cljs

@@ -14,6 +14,7 @@
    [app.common.spec :as us]
    [app.config :as cf]
    [app.main.data.messages :as msg]
+   [app.main.data.modal :as modal]
    [app.main.data.users :as du]
    [app.main.store :as st]
    [app.util.globals :as glob]
@@ -167,6 +168,26 @@
   (ts/schedule
    #(st/emit! (rt/assign-exception error))))
 
+(defmethod ptk/handle-error :restriction
+  [{:keys [code] :as error}]
+  (cond
+    (= :feature-mismatch code)
+    (let [message (tr "errors.feature-mismatch" (:feature error))]
+      (st/emit! (modal/show
+                 {:type :alert
+                  :message message
+                  :on-accept #(prn "kaka")})))
+
+    (= :features-not-supported code)
+    (let [message (tr "errors.feature-not-supported" (:feature error))]
+      (st/emit! (modal/show
+                 {:type :alert
+                  :message message
+                  :on-accept #(prn "kaka")})))
+
+    :else
+    (ptk/handle-error (assoc error :type :server-error))))
+
 ;; This happens when the backed server fails to process the
 ;; request. This can be caused by an internal assertion or any other
 ;; uncontrolled error.

frontend/src/app/main/features.cljs

@@ -16,7 +16,7 @@
 
 (log/set-level! :debug)
 
-(def features-list #{:auto-layout :components-v2})
+(def available-features #{:auto-layout :components-v2})
 
 (defn- toggle-feature
   [feature]
@@ -38,14 +38,14 @@
 
 (defn toggle-feature!
   [feature]
-  (assert (contains? features-list feature) "Not supported feature")
+  (assert (contains? available-features feature) "Not supported feature")
   (st/emit! (toggle-feature feature)))
 
 (defn active-feature?
   ([feature]
    (active-feature? @st/state feature))
   ([state feature]
-   (assert (contains? features-list feature) "Not supported feature")
+   (assert (contains? available-features feature) "Not supported feature")
    (contains? (get state :features) feature)))
 
 (def features
@@ -57,7 +57,7 @@
 
 (defn use-feature
   [feature]
-  (assert (contains? features-list feature) "Not supported feature")
+  (assert (contains? available-features feature) "Not supported feature")
   (let [active-feature-ref (mf/use-memo (mf/deps feature) #(active-feature feature))
         active-feature? (mf/deref active-feature-ref)]
     active-feature?))
@@ -69,6 +69,6 @@
   (when *assert*
     ;; By default, all features disabled, except in development
     ;; environment, that are enabled except components-v2
-    (doseq [f features-list]
+    (doseq [f available-features]
       (when (not= f :components-v2)
         (toggle-feature! f)))))

frontend/src/app/main/repo.cljs

@@ -12,6 +12,8 @@
    [app.util.http :as http]
    [beicon.core :as rx]))
 
+(derive :get-file ::query)
+
 (defn handle-response
   [{:keys [status body] :as response}]
   (cond
@@ -48,7 +50,6 @@
   query api."
   ([id params]
    (send-query! id params nil))
-
   ([id params {:keys [raw-transit?]}]
    (let [decode-transit (if raw-transit?
                           http/conditional-error-decode-transit
@@ -74,14 +75,24 @@
 (defn- send-command!
   "A simple helper for a common case of sending and receiving transit
   data to the penpot mutation api."
-  [id params {:keys [response-type form-data?]}]
+  [id params {:keys [response-type form-data? raw-transit?]}]
-  (->> (http/send! {:method :post
+  (let [decode-fn (if raw-transit?
+                    http/conditional-error-decode-transit
+                    http/conditional-decode-transit)
+        method    (if (isa? id ::query) :get :post)]
+
+    (->> (http/send! {:method method
                       :uri (u/join @cf/public-uri "api/rpc/command/" (name id))
                       :credentials "include"
-                    :body (if form-data? (http/form-data params) (http/transit-data params))
+                      :body (when (= method :post)
+                              (if form-data?
+                                (http/form-data params)
+                                (http/transit-data params)))
+                      :query (when (= method :get)
+                               params)
                       :response-type (or response-type :text)})
-       (rx/map http/conditional-decode-transit)
+         (rx/map decode-fn)
-       (rx/mapcat handle-response)))
+         (rx/mapcat handle-response))))
 
 (defn- dispatch [& args] (first args))
 
@@ -93,9 +104,9 @@
   [id params]
   (send-query! id params))
 
-(defmethod query :file-raw
+(defmethod command :get-raw-file
   [_id params]
-  (send-query! :file params {:raw-transit? true}))
+  (send-command! :get-file params {:raw-transit? true}))
 
 (defmethod mutation :default
   [id params]

frontend/src/app/main/ui/dashboard/grid.cljs

@@ -7,6 +7,7 @@
 (ns app.main.ui.dashboard.grid
   (:require
    [app.common.data.macros :as dm]
+   [app.common.files.features :as ffeat]
    [app.common.logging :as log]
    [app.main.data.dashboard :as dd]
    [app.main.data.messages :as msg]
@@ -34,18 +35,21 @@
    [cuerdas.core :as str]
    [rumext.v2 :as mf]))
 
-(log/set-level! :info)
+(log/set-level! :debug)
 
 ;; --- Grid Item Thumbnail
 
 (defn ask-for-thumbnail
   "Creates some hooks to handle the files thumbnails cache"
   [file]
+  (let [features (cond-> ffeat/enabled
+                   (features/active-feature? :components-v2)
+                   (conj "components/v2"))]
     (wrk/ask! {:cmd :thumbnails/generate
                :revn (:revn file)
                :file-id (:id file)
                :file-name (:name file)
-             :components-v2 (features/active-feature? :components-v2)}))
+               :features features})))
 
 (mf/defc grid-item-thumbnail
   {::mf/wrap [mf/memo]}
@@ -61,7 +65,7 @@
                (rx/subscribe-on :af)
                (rx/subs (fn [{:keys [data fonts] :as params}]
                           (run! fonts/ensure-loaded! fonts)
-                          (log/info :hint "loaded thumbnail"
+                          (log/debug :hint "loaded thumbnail"
                                      :file-id (dm/str (:id file))
                                      :file-name (:name file)
                                      :elapsed (str/ffmt "%ms" (tp)))

frontend/src/app/render.cljs

@@ -98,14 +98,15 @@
   [{:keys [page-id file-id object-id render-embed?]}]
   (let [components-v2 (features/use-feature :components-v2)
         fetch-state   (mf/use-fn
-                        (mf/deps file-id page-id object-id)
+                        (mf/deps file-id page-id object-id components-v2)
                         (fn []
+                          (let [features (cond-> #{} components-v2 (conj "components/v2"))]
                             (->> (rx/zip
                                   (repo/query! :font-variants {:file-id file-id})
-                                 (repo/query! :page {:file-id file-id
+                                  (repo/cmd! :page {:file-id file-id
                                                     :page-id page-id
                                                     :object-id object-id
-                                                     :components-v2 components-v2}))
+                                                    :features features}))
                                  (rx/tap (fn [[fonts]]
                                            (when (seq fonts)
                                              (st/emit! (df/fonts-fetched fonts)))))
@@ -113,7 +114,7 @@
                                  (rx/map (fn [objects]
                                            (let [objects (render/adapt-objects-for-shape objects object-id)]
                                              {:objects objects
-                                            :object (get objects object-id)}))))))
+                                              :object (get objects object-id)})))))))
 
         {:keys [objects object]} (use-resource fetch-state)]
 
@@ -137,17 +138,18 @@
   [{:keys [page-id file-id object-ids render-embed?]}]
   (let [components-v2 (features/use-feature :components-v2)
         fetch-state   (mf/use-fn
-                       (mf/deps file-id page-id)
+                       (mf/deps file-id page-id components-v2)
                        (fn []
+                         (let [features (cond-> #{} components-v2 (conj "components/v2"))]
                            (->> (rx/zip
                                  (repo/query! :font-variants {:file-id file-id})
-                               (repo/query! :page {:file-id file-id
+                                 (repo/cmd! :get-page {:file-id file-id
                                                        :page-id page-id
-                                                   :components-v2 components-v2}))
+                                                       :features features}))
                                 (rx/tap (fn [[fonts]]
                                           (when (seq fonts)
                                             (st/emit! (df/fonts-fetched fonts)))))
-                              (rx/map (comp :objects second)))))
+                                (rx/map (comp :objects second))))))
 
         objects (use-resource fetch-state)]
 
@@ -204,7 +206,7 @@
   [{:keys [file-id embed] :as props}]
   (let [fetch (mf/use-fn
                (mf/deps file-id)
-               (fn [] (repo/query! :file {:id file-id})))
+               (fn [] (repo/cmd! :get-file {:id file-id})))
 
         file  (use-resource fetch)
         state (mf/use-state nil)]

frontend/src/app/util/http.cljs

@@ -139,6 +139,7 @@
   [{:keys [body headers] :as response}]
   (let [contenttype (get headers "content-type")]
     (if (and (str/starts-with? contenttype "application/transit+json")
+             (string? body)
              (pos? (count body)))
       (assoc response :body (t/decode-str body))
       response)))

frontend/src/app/worker/export.cljs

@@ -155,14 +155,15 @@
   (->> (r/render-components (:data file) :deleted-components)
        (rx/map #(vector (str (:id file) "/deleted-components.svg") %))))
 
-(defn fetch-file-with-libraries [file-id components-v2]
+(defn fetch-file-with-libraries
-  (->> (rx/zip (rp/query :file {:id file-id :components-v2 components-v2})
+  [file-id components-v2]
-               (rp/query :file-libraries {:file-id file-id}))
+  (let [features (cond-> #{} components-v2 (conj "components/v2"))]
+    (->> (rx/zip (rp/cmd! :get-file {:id file-id :features features})
+                 (rp/cmd! :get-file-libraries {:file-id file-id}))
          (rx/map
           (fn [[file file-libraries]]
             (let [libraries-ids (->> file-libraries (map :id) (filterv #(not= (:id file) %)))]
-            (-> file
+              (assoc file :libraries libraries-ids)))))))
-                (assoc :libraries libraries-ids)))))))
 
 (defn get-component-ref-file
   [objects shape]

frontend/src/app/worker/impl.cljs

@@ -6,6 +6,7 @@
 
 (ns app.worker.impl
   (:require
+   [app.common.data.macros :as dm]
    [app.common.logging :as log]
    [app.common.pages.changes :as ch]
    [app.common.transit :as t]
@@ -36,18 +37,16 @@
   (let [data (-> (t/decode-str file-raw) :data)
         message (assoc message :data data)]
     (reset! state data)
-    (handler (-> message
+    (handler (assoc message :cmd :selection/initialize-index))
-                 (assoc :cmd :selection/initialize-index)))
+    (handler (assoc message :cmd :snaps/initialize-index))))
-    (handler (-> message
-                 (assoc :cmd :snaps/initialize-index)))))
 
 (defmethod handler :update-page-indices
   [{:keys [page-id changes] :as message}]
 
-  (let [old-page (get-in @state [:pages-index page-id])]
+  (let [old-page (dm/get-in @state [:pages-index page-id])]
     (swap! state ch/process-changes changes false)
 
-    (let [new-page (get-in @state [:pages-index page-id])
+    (let [new-page (dm/get-in @state [:pages-index page-id])
           message (assoc message
                          :old-page old-page
                          :new-page new-page)]

frontend/src/app/worker/import.cljs

@@ -15,7 +15,6 @@
    [app.common.logging :as log]
    [app.common.media :as cm]
    [app.common.text :as ct]
-   [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
    [app.main.repo :as rp]
    [app.util.http :as http]
@@ -128,16 +127,15 @@
 (defn create-file
   "Create a new file on the back-end"
   [context components-v2]
-  (let [resolve (:resolve context)
+  (let [resolve-fn (:resolve context)
-        file-id (resolve (:file-id context))]
+        file-id    (resolve-fn (:file-id context))
-    (rp/mutation :create-temp-file
+        features   (cond-> #{} components-v2 (conj "components/v2"))]
+    (rp/cmd! :create-temp-file
              {:id file-id
               :name (:name context)
               :is-shared (:shared context)
               :project-id (:project-id context)
-                  :data (-> ctf/empty-file-data
+              :features features})))
-                            (assoc :id file-id)
-                            (assoc-in [:options :components-v2] components-v2))})))
 
 (defn link-file-libraries
   "Create a new file on the back-end"
@@ -147,7 +145,7 @@
         libraries (->> context :libraries (mapv resolve))]
     (->> (rx/from libraries)
          (rx/map #(hash-map :file-id file-id :library-id %))
-         (rx/flat-map (partial rp/mutation :link-file-to-library)))))
+         (rx/flat-map (partial rp/cmd! :link-file-to-library)))))
 
 (defn send-changes
   "Creates batches of changes to be sent to the backend"
@@ -165,7 +163,7 @@
      (->> (rx/from (d/enumerate batches))
           (rx/merge-map
            (fn [[i change-batch]]
-             (->> (rp/mutation :update-temp-file
+             (->> (rp/cmd! :update-temp-file
                            {:id file-id
                             :session-id session-id
                             :revn i
@@ -175,7 +173,7 @@
           (rx/map first)
           (rx/ignore))
 
-     (->> (rp/mutation :persist-temp-file {:id file-id})
+     (->> (rp/cmd! :persist-temp-file {:id file-id})
           ;; We use merge to keep some information not stored in back-end
           (rx/map #(merge file %))))))
 

frontend/src/app/worker/thumbnails.cljs

@@ -7,6 +7,7 @@
 (ns app.worker.thumbnails
   (:require
    ["react-dom/server" :as rds]
+   [app.common.logging :as log]
    [app.common.uri :as u]
    [app.config :as cf]
    [app.main.fonts :as fonts]
@@ -17,6 +18,9 @@
    [debug :refer [debug?]]
    [rumext.v2 :as mf]))
 
+(log/set-level! :trace)
+
+
 (defn- handle-response
   [{:keys [body status] :as response}]
   (cond
@@ -48,12 +52,12 @@
        (= :request-body-too-large code)))
 
 (defn- request-data-for-thumbnail
-  [file-id revn components-v2]
+  [file-id revn features]
-  (let [path    "api/rpc/query/file-data-for-thumbnail"
+  (let [path    "api/rpc/command/get-file-data-for-thumbnail"
         params   {:file-id file-id
                   :revn revn
                   :strip-frames-with-thumbnails true
-                 :components-v2 components-v2}
+                  :features features}
         request  {:method :get
                   :uri (u/join @cf/public-uri path)
                   :credentials "include"
@@ -64,14 +68,13 @@
 
 (defn- request-thumbnail
   [file-id revn]
-  (let [path    "api/rpc/query/file-thumbnail"
+  (let [path    "api/rpc/command/get-file-thumbnail"
         params  {:file-id file-id
                  :revn revn}
         request {:method :get
                  :uri (u/join @cf/public-uri path)
                  :credentials "include"
                  :query params}]
-
     (->> (http/send! request)
          (rx/map http/conditional-decode-transit)
          (rx/mapcat handle-response))))
@@ -91,7 +94,7 @@
 
 (defn- persist-thumbnail
   [{:keys [file-id data revn fonts]}]
-  (let [path    "api/rpc/mutation/upsert-file-thumbnail"
+  (let [path    "api/rpc/command/upsert-file-thumbnail"
         params  {:file-id file-id
                  :revn revn
                  :props {:fonts fonts}
@@ -108,19 +111,22 @@
          (rx/map (constantly params)))))
 
 (defmethod impl/handler :thumbnails/generate
-  [{:keys [file-id revn components-v2] :as message}]
+  [{:keys [file-id revn features] :as message}]
   (letfn [(on-result [{:keys [data props]}]
             {:data data
              :fonts (:fonts props)})
 
           (on-cache-miss [_]
-            (->> (request-data-for-thumbnail file-id revn components-v2)
+            (log/debug :hint "request-thumbnail" :file-id file-id :revn revn :cache "miss")
+            (->> (request-data-for-thumbnail file-id revn features)
                  (rx/map render-thumbnail)
                  (rx/mapcat persist-thumbnail)))]
 
     (if (debug? :disable-thumbnail-cache)
-      (->> (request-data-for-thumbnail file-id revn components-v2)
+      (->> (request-data-for-thumbnail file-id revn features)
            (rx/map render-thumbnail))
       (->> (request-thumbnail file-id revn)
+           (rx/tap (fn [_]
+                     (log/debug :hint "request-thumbnail" :file-id file-id :revn revn :cache "hit")))
            (rx/catch not-found? on-cache-miss)
            (rx/map on-result)))))

frontend/translations/en.po

@@ -710,9 +710,13 @@ msgstr "The fonts %s could not be loaded"
 msgid "errors.clipboard-not-implemented"
 msgstr "Your browser cannot do this operation"
 
-#: src/app/main/data/workspace/persistence.cljs
+#: src/app/main/errors.cljs
-msgid "errors.components-v2"
+msgid "errors.feature-not-supported"
-msgstr "This file has already used with Components V2 enabled."
+msgstr "Feature '%s' is not supported."
+
+#: src/app/main/errors.cljs
+msgid "errors.feature-mismatch"
+msgstr "Looks like you are opening a file that has the feature '%s' enabled bug your penpot frontend does not supports it or has it disabled."
 
 #: src/app/main/ui/auth/verify_token.cljs, src/app/main/ui/settings/change_email.cljs
 msgid "errors.email-already-exists"