mirror of
https://github.com/penpot/penpot.git
synced 2025-07-24 04:47:12 +02:00
🎉 Add file offloading to external storage mechanism.
This commit is contained in:
parent
0c49ed1fec
commit
0c97a44a2a
18 changed files with 334 additions and 125 deletions
64
backend/src/app/tasks/file_offload.clj
Normal file
64
backend/src/app/tasks/file_offload.clj
Normal file
|
@ -0,0 +1,64 @@
|
|||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) UXBOX Labs SL
|
||||
|
||||
(ns app.tasks.file-offload
|
||||
"A maintenance task that offloads file data to an external storage (S3)."
|
||||
(:require
|
||||
[app.common.spec :as us]
|
||||
[app.db :as db]
|
||||
[app.storage :as sto]
|
||||
[app.storage.impl :as simpl]
|
||||
[app.util.logging :as l]
|
||||
[app.util.time :as dt]
|
||||
[clojure.spec.alpha :as s]
|
||||
[integrant.core :as ig]))
|
||||
|
||||
(def sql:offload-candidates-chunk
|
||||
"select f.id, f.data from file as f
|
||||
where f.data is not null
|
||||
and f.modified_at < now() - ?::interval
|
||||
order by f.modified_at
|
||||
limit 10")
|
||||
|
||||
(defn- retrieve-candidates
|
||||
[{:keys [conn max-age]}]
|
||||
(db/exec! conn [sql:offload-candidates-chunk max-age]))
|
||||
|
||||
(defn- offload-candidate
|
||||
[{:keys [storage conn backend] :as cfg} {:keys [id data] :as file}]
|
||||
(l/debug :action "offload file data" :id id)
|
||||
(let [backend (simpl/resolve-backend storage backend)]
|
||||
(->> (simpl/content data)
|
||||
(simpl/put-object backend file))
|
||||
(db/update! conn :file
|
||||
{:data nil
|
||||
:data-backend (name (:id backend))}
|
||||
{:id id})))
|
||||
|
||||
;; ---- STATE INIT
|
||||
|
||||
(s/def ::max-age ::dt/duration)
|
||||
(s/def ::backend ::us/keyword)
|
||||
|
||||
|
||||
(defmethod ig/pre-init-spec ::handler [_]
|
||||
(s/keys :req-un [::db/pool ::max-age ::sto/storage ::backend]))
|
||||
|
||||
(defmethod ig/init-key ::handler
|
||||
[_ {:keys [pool max-age] :as cfg}]
|
||||
(fn [_]
|
||||
(db/with-atomic [conn pool]
|
||||
(let [max-age (db/interval max-age)
|
||||
cfg (-> cfg
|
||||
(assoc :conn conn)
|
||||
(assoc :max-age max-age))]
|
||||
(loop [n 0]
|
||||
(let [candidates (retrieve-candidates cfg)]
|
||||
(if (seq candidates)
|
||||
(do
|
||||
(run! (partial offload-candidate cfg) candidates)
|
||||
(recur (+ n (count candidates))))
|
||||
(l/debug :hint "offload summary" :count n))))))))
|
Loading…
Add table
Add a link
Reference in a new issue