♻️ Make the media cleaning mechanism on file-gc task more efficient

Replaces the use of db/cursor with db/plan, that teorethically allows
processing large results without consuming all result set in memory

backend/src/app/tasks/file_gc.clj

@@ -53,16 +53,27 @@
    RETURNING id")
 
 (def ^:private xf:collect-used-media
-  (comp (map :data) (mapcat bfc/collect-used-media)))
+  (comp
+   (map :data)
+   (mapcat bfc/collect-used-media)))
+
+(def ^:private plan-opts
+  {:fetch-size 1
+   :concurrency :read-only
+   :cursors :close
+   :result-type :forward-only})
 
 (defn- clean-file-media!
   "Performs the garbage collection of file media objects."
   [{:keys [::db/conn] :as cfg} {:keys [id] :as file}]
-  (let [used   (into #{}
+  (let [xform  (comp
-                     xf:collect-used-media
+                (map (partial decode-file cfg))
-                     (cons file
+                xf:collect-used-media)
-                           (->> (db/cursor conn [sql:get-snapshots id])
+
-                                (map (partial decode-file cfg)))))
+        used   (->> (db/plan conn [sql:get-snapshots id] plan-opts)
+                    (transduce xform conj #{}))
+        used   (into used xf:collect-used-media [file])
+
         ids    (db/create-array conn "uuid" used)
         unused (->> (db/exec! conn [sql:mark-file-media-object-deleted id ids])
                     (into #{} (map :id)))]

backend/test/backend_tests/helpers.clj

@@ -62,7 +62,7 @@
 (def default
   {:database-uri "postgresql://postgres/penpot_test"
    :redis-uri "redis://redis/1"
-   :file-snapshot-every 1})
+   :auto-file-snapshot-every 1})
 
 (def config
   (cf/read-config :prefix "penpot-test"

backend/test/backend_tests/rpc_file_test.clj

@@ -383,8 +383,19 @@
       ;; as deleted.
       (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
 
+      ;; This only clears fragments, the file media objects still referenced because
+      ;; snapshots are preserved
       (let [res (th/run-task! :objects-gc {:min-age 0})]
-        (t/is (= 3 (:processed res))))
+        (t/is (= 2 (:processed res))))
+
+      ;; Mark all snapshots to be a non-snapshot file change
+      (th/db-exec! ["update file_change set data = null where file_id = ?" (:id file)])
+      (th/db-exec! ["update file set has_media_trimmed = false where id = ?" (:id file)])
+
+      ;; Rerun the file-gc and objects-gc
+      (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
+      (let [res (th/run-task! :objects-gc {:min-age 0})]
+        (t/is (= 2 (:processed res))))
 
       ;; Now that file-gc have deleted the file-media-object usage,
       ;; lets execute the touched-gc task, we should see that two of
@@ -417,20 +428,6 @@
 
               ;; (th/print-result! out)
               (t/is (nil? (:error out)))
-              (:result out)))
-
-          (update-file! [& {:keys [profile-id file-id changes revn] :or {revn 0}}]
-            (let [params {::th/type :update-file
-                          ::rpc/profile-id profile-id
-                          :id file-id
-                          :session-id (uuid/random)
-                          :revn revn
-                          :vern 0
-                          :components-v2 true
-                          :changes changes}
-                  out    (th/command! params)]
-              ;; (th/print-result! out)
-              (t/is (nil? (:error out)))
               (:result out)))]
 
     (let [storage (:app.storage/storage th/*system*)
@@ -550,8 +547,20 @@
       ;; as deleted.
       (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
 
+      ;; This only removes unused fragments, file media are still
+      ;; referenced on snapshots.
       (let [res (th/run-task! :objects-gc {:min-age 0})]
-        (t/is (= 7 (:processed res))))
+        (t/is (= 2 (:processed res))))
+
+      ;; Mark all snapshots to be a non-snapshot file change
+      (th/db-exec! ["update file set has_media_trimmed = false where id = ?" (:id file)])
+      (th/db-exec! ["update file_change set data = null where file_id = ?" (:id file)])
+
+      ;; Rerun file-gc and objects-gc task for the same file once all snapshots are
+      ;; "expired/deleted"
+      (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
+      (let [res (th/run-task! :objects-gc {:min-age 0})]
+        (t/is (= 6 (:processed res))))
 
       (let [rows (th/db-query :file-data-fragment {:file-id (:id file)
                                                    :deleted-at nil})]
@@ -589,19 +598,6 @@
                              :media mfile}
                   out       (th/command! params)]
 
-              ;; (th/print-result! out)
-              (t/is (nil? (:error out)))
-              (:result out)))
-
-          #_(update-file! [& {:keys [profile-id file-id changes revn] :or {revn 0}}]
-                          (let [params {::th/type :update-file
-                                        ::rpc/profile-id profile-id
-                                        :id file-id
-                                        :session-id (uuid/random)
-                                        :revn revn
-                                        :features cfeat/supported-features
-                                        :changes changes}
-                                out    (th/command! params)]
               ;; (th/print-result! out)
               (t/is (nil? (:error out)))
               (:result out)))]