diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index 9ea2129f7..ac1ea0b78 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -205,6 +205,9 @@ {:name "0065-add-trivial-spelling-fixes" :fn (mg/resource "app/migrations/sql/0065-add-trivial-spelling-fixes.sql")} + + {:name "0066-add-frame-thumbnail-table" + :fn (mg/resource "app/migrations/sql/0066-add-frame-thumbnail-table.sql")} ]) diff --git a/backend/src/app/migrations/sql/0066-add-frame-thumbnail-table.sql b/backend/src/app/migrations/sql/0066-add-frame-thumbnail-table.sql new file mode 100644 index 000000000..3134cbe21 --- /dev/null +++ b/backend/src/app/migrations/sql/0066-add-frame-thumbnail-table.sql @@ -0,0 +1,10 @@ +CREATE TABLE file_frame_thumbnail ( + file_id uuid NOT NULL REFERENCES file(id) ON DELETE CASCADE, + frame_id uuid NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + + data text NULL, + + PRIMARY KEY(file_id, frame_id) +); diff --git a/backend/src/app/rpc/mutations/files.clj b/backend/src/app/rpc/mutations/files.clj index c145e2ceb..ce8d98ef9 100644 --- a/backend/src/app/rpc/mutations/files.clj +++ b/backend/src/app/rpc/mutations/files.clj @@ -27,6 +27,8 @@ ;; --- Helpers & Specs +(s/def ::frame-id ::us/uuid) +(s/def ::file-id ::us/uuid) (s/def ::id ::us/uuid) (s/def ::name ::us/string) (s/def ::profile-id ::us/uuid) @@ -472,3 +474,25 @@ {:id id}))) nil))) + + +;; --- Mutation: Upsert frame thumbnail + +(def sql:upsert-frame-thumbnail + "insert into file_frame_thumbnail(file_id, frame_id, data) + values (?, ?, ?) + on conflict(file_id, frame_id) do + update set data = ?;") + +(s/def ::data ::us/string) +(s/def ::upsert-frame-thumbnail + (s/keys :req-un [::profile-id ::file-id ::frame-id ::data])) + +(sv/defmethod ::upsert-frame-thumbnail + [{:keys [pool] :as cfg} {:keys [profile-id file-id frame-id data]}] + (db/with-atomic [conn pool] + (files/check-edition-permissions! conn profile-id file-id) + (db/exec-one! conn [sql:upsert-frame-thumbnail file-id frame-id data data]) + nil)) + + diff --git a/backend/src/app/rpc/queries/files.clj b/backend/src/app/rpc/queries/files.clj index 91e0c4023..50ed64837 100644 --- a/backend/src/app/rpc/queries/files.clj +++ b/backend/src/app/rpc/queries/files.clj @@ -26,6 +26,7 @@ ;; --- Helpers & Specs +(s/def ::frame-id ::us/uuid) (s/def ::id ::us/uuid) (s/def ::name ::us/string) (s/def ::project-id ::us/uuid) @@ -392,6 +393,7 @@ ) select * from recent_files where row_num <= 10;") + (s/def ::team-recent-files (s/keys :req-un [::profile-id ::team-id])) @@ -401,6 +403,25 @@ (teams/check-read-permissions! conn profile-id team-id) (db/exec! conn [sql:team-recent-files team-id]))) + +;; --- QUERY: get the thumbnail for an frame + +(def ^:private sql:file-frame-thumbnail + "select data + from file_frame_thumbnail + where file_id = ? + and frame_id = ?") + +(s/def ::file-frame-thumbnail + (s/keys :req-un [::profile-id ::file-id ::frame-id])) + +(sv/defmethod ::file-frame-thumbnail + [{:keys [pool]} {:keys [profile-id file-id frame-id]}] + (with-open [conn (db/open pool)] + (check-read-permissions! conn profile-id file-id) + (db/exec-one! conn [sql:file-frame-thumbnail file-id frame-id]))) + + ;; --- Helpers (defn decode-row diff --git a/backend/src/app/tasks/file_media_gc.clj b/backend/src/app/tasks/file_media_gc.clj index 39d194f68..4a04f4fc6 100644 --- a/backend/src/app/tasks/file_media_gc.clj +++ b/backend/src/app/tasks/file_media_gc.clj @@ -10,6 +10,7 @@ after some period of inactivity (the default threshold is 72h)." (:require [app.common.logging :as l] + [app.common.pages.helpers :as cph] [app.common.pages.migrations :as pmg] [app.db :as db] [app.util.blob :as blob] @@ -52,6 +53,7 @@ limit 10 for update skip locked") + (defn- retrieve-candidates [{:keys [conn max-age] :as cfg}] (let [interval (db/interval max-age)] @@ -64,12 +66,11 @@ (comp (map :objects) (mapcat vals) - (map (fn [{:keys [type] :as obj}] - (case type - :path (get-in obj [:fill-image :id]) - :image (get-in obj [:metadata :id]) - nil))) - (filter uuid?))) + (keep (fn [{:keys [type] :as obj}] + (case type + :path (get-in obj [:fill-image :id]) + :image (get-in obj [:metadata :id]) + nil))))) (defn- collect-used-media [data] @@ -80,37 +81,59 @@ (into collect-media-xf pages) (into (keys (:media data)))))) +(def ^:private + collect-frames-xf + (comp + (map :objects) + (mapcat vals) + (filter cph/frame-shape?) + (keep :id))) + +(defn- collect-frames + [data] + (let [pages (concat + (vals (:pages-index data)) + (vals (:components data)))] + (into #{} collect-frames-xf pages))) + (defn- process-file [{:keys [conn] :as cfg} {:keys [id data age] :as file}] - (let [data (-> (blob/decode data) - (assoc :id id) - (pmg/migrate-data)) + (let [data (-> (blob/decode data) + (assoc :id id) + (pmg/migrate-data))] - used (collect-used-media data) - unused (->> (db/query conn :file-media-object {:file-id id}) - (remove #(contains? used (:id %))))] + (let [used (collect-used-media data) + unused (->> (db/query conn :file-media-object {:file-id id}) + (remove #(contains? used (:id %))))] - (l/debug :action "processing file" - :id id - :age age - :to-delete (count unused)) + (l/debug :action "processing file" + :id id + :age age + :to-delete (count unused)) - ;; Mark file as trimmed - (db/update! conn :file - {:has-media-trimmed true} - {:id id}) + ;; Mark file as trimmed + (db/update! conn :file + {:has-media-trimmed true} + {:id id}) - (doseq [mobj unused] - (l/debug :action "deleting media object" - :id (:id mobj) - :media-id (:media-id mobj) - :thumbnail-id (:thumbnail-id mobj)) + (doseq [mobj unused] + (l/debug :action "deleting media object" + :id (:id mobj) + :media-id (:media-id mobj) + :thumbnail-id (:thumbnail-id mobj)) - ;; NOTE: deleting the file-media-object in the database - ;; automatically marks as touched the referenced storage - ;; objects. The touch mechanism is needed because many files can - ;; point to the same storage objects and we can't just delete - ;; them. - (db/delete! conn :file-media-object {:id (:id mobj)})) + ;; NOTE: deleting the file-media-object in the database + ;; automatically marks as touched the referenced storage + ;; objects. The touch mechanism is needed because many files can + ;; point to the same storage objects and we can't just delete + ;; them. + (db/delete! conn :file-media-object {:id (:id mobj)}))) + + (let [sql (str "delete from file_frame_thumbnail " + " where file_id = ? and not (frame_id = ANY(?))") + ids (->> (collect-frames data) + (db/create-array conn "uuid"))] + ;; delete the unused frame thumbnails + (db/exec! conn [sql (:id file) ids])) nil)) diff --git a/backend/test/app/services_files_test.clj b/backend/test/app/services_files_test.clj index c0dc39e32..4c41ee557 100644 --- a/backend/test/app/services_files_test.clj +++ b/backend/test/app/services_files_test.clj @@ -389,3 +389,73 @@ (t/is (th/ex-info? error)) (t/is (= (:type error-data) :not-found)))) )) + +(t/deftest query-frame-thumbnails + (let [prof (th/create-profile* 1 {:is-active true}) + file (th/create-file* 1 {:profile-id (:id prof) + :project-id (:default-project-id prof) + :is-shared false}) + data {::th/type :file-frame-thumbnail + :profile-id (:id prof) + :file-id (:id file) + :frame-id (uuid/next)}] + + ;;insert an entry on the database with a test value for the thumbnail of this frame + (db/exec-one! th/*pool* + ["insert into file_frame_thumbnail(file_id, frame_id, data) values (?, ?, ?)" + (:file-id data) (:frame-id data) "testvalue"]) + + (let [out (th/query! data)] + (t/is (nil? (:error out))) + (let [result (:result out)] + (t/is (= 1 (count result))) + (t/is (= "testvalue" (:data result))))))) + +(t/deftest insert-frame-thumbnails + (let [prof (th/create-profile* 1 {:is-active true}) + file (th/create-file* 1 {:profile-id (:id prof) + :project-id (:default-project-id prof) + :is-shared false}) + data {::th/type :upsert-frame-thumbnail + :profile-id (:id prof) + :file-id (:id file) + :frame-id (uuid/next) + :data "test insert new value"} + out (th/mutation! data)] + + (t/is (nil? (:error out))) + (t/is (nil? (:result out))) + + ;;retrieve the value from the database and check its content + (let [result (db/exec-one! + th/*pool* + ["select data from file_frame_thumbnail where file_id = ? and frame_id = ?" + (:file-id data) (:frame-id data)])] + (t/is (= "test insert new value" (:data result)))))) + +(t/deftest frame-thumbnails + (let [prof (th/create-profile* 1 {:is-active true}) + file (th/create-file* 1 {:profile-id (:id prof) + :project-id (:default-project-id prof) + :is-shared false}) + data {::th/type :upsert-frame-thumbnail + :profile-id (:id prof) + :file-id (:id file) + :frame-id (uuid/next) + :data "updated value"}] + + ;;insert an entry on the database with and old value for the thumbnail of this frame + (db/exec-one! th/*pool* + ["insert into file_frame_thumbnail(file_id, frame_id, data) values (?, ?, ?)" + (:file-id data) (:frame-id data) "old value"]) + + (let [out (th/mutation! data)] + (t/is (nil? (:error out))) + (t/is (nil? (:result out))) + + ;;retrieve the value from the database and check its content + (let [result (db/exec-one! + th/*pool* + ["select data from file_frame_thumbnail where file_id = ? and frame_id = ?" + (:file-id data) (:frame-id data)])] + (t/is (= "updated value" (:data result)))))))