diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index ac76d9de3..742b20727 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -5,7 +5,7 @@ ;; This Source Code Form is "Incompatible With Secondary Licenses", as ;; defined by the Mozilla Public License, v. 2.0. ;; -;; Copyright (c) 2020 UXBOX Labs SL +;; Copyright (c) 2020-2021 UXBOX Labs SL (ns app.config "A configuration management." @@ -80,92 +80,78 @@ ;; :initial-data-project-name "Penpot Oboarding" }) -(s/def ::http-server-port ::us/integer) - -(s/def ::host ::us/string) -(s/def ::tenant ::us/string) - -(s/def ::database-username (s/nilable ::us/string)) +(s/def ::allow-demo-users ::us/boolean) +(s/def ::asserts-enabled ::us/boolean) +(s/def ::assets-path ::us/string) (s/def ::database-password (s/nilable ::us/string)) (s/def ::database-uri ::us/string) -(s/def ::redis-uri ::us/string) - -(s/def ::loggers-loki-uri ::us/string) -(s/def ::loggers-zmq-uri ::us/string) - -(s/def ::storage-backend ::us/keyword) -(s/def ::storage-fs-directory ::us/string) -(s/def ::assets-path ::us/string) -(s/def ::storage-s3-region ::us/keyword) -(s/def ::storage-s3-bucket ::us/string) - -(s/def ::media-uri ::us/string) -(s/def ::media-directory ::us/string) -(s/def ::asserts-enabled ::us/boolean) - -(s/def ::feedback-enabled ::us/boolean) -(s/def ::feedback-destination ::us/string) - -(s/def ::profile-complaint-max-age ::dt/duration) -(s/def ::profile-complaint-threshold ::us/integer) -(s/def ::profile-bounce-max-age ::dt/duration) -(s/def ::profile-bounce-threshold ::us/integer) - +(s/def ::database-username (s/nilable ::us/string)) +(s/def ::default-blob-version ::us/integer) (s/def ::error-report-webhook ::us/string) - -(s/def ::smtp-enabled ::us/boolean) -(s/def ::smtp-default-reply-to ::us/string) -(s/def ::smtp-default-from ::us/string) -(s/def ::smtp-host ::us/string) -(s/def ::smtp-port ::us/integer) -(s/def ::smtp-username (s/nilable ::us/string)) -(s/def ::smtp-password (s/nilable ::us/string)) -(s/def ::smtp-tls ::us/boolean) -(s/def ::smtp-ssl ::us/boolean) -(s/def ::allow-demo-users ::us/boolean) -(s/def ::registration-enabled ::us/boolean) -(s/def ::registration-domain-whitelist ::us/string) -(s/def ::public-uri ::us/string) - -(s/def ::srepl-host ::us/string) -(s/def ::srepl-port ::us/integer) - -(s/def ::rlimits-password ::us/integer) -(s/def ::rlimits-image ::us/integer) - -(s/def ::google-client-id ::us/string) -(s/def ::google-client-secret ::us/string) - -(s/def ::gitlab-client-id ::us/string) -(s/def ::gitlab-client-secret ::us/string) -(s/def ::gitlab-base-uri ::us/string) - +(s/def ::feedback-destination ::us/string) +(s/def ::feedback-enabled ::us/boolean) (s/def ::github-client-id ::us/string) (s/def ::github-client-secret ::us/string) - -(s/def ::ldap-host ::us/string) -(s/def ::ldap-port ::us/integer) -(s/def ::ldap-bind-dn ::us/string) -(s/def ::ldap-bind-password ::us/string) -(s/def ::ldap-ssl ::us/boolean) -(s/def ::ldap-starttls ::us/boolean) -(s/def ::ldap-base-dn ::us/string) -(s/def ::ldap-user-query ::us/string) -(s/def ::ldap-attrs-username ::us/string) +(s/def ::gitlab-base-uri ::us/string) +(s/def ::gitlab-client-id ::us/string) +(s/def ::gitlab-client-secret ::us/string) +(s/def ::google-client-id ::us/string) +(s/def ::google-client-secret ::us/string) +(s/def ::host ::us/string) +(s/def ::http-server-port ::us/integer) +(s/def ::http-session-cookie-name ::us/string) +(s/def ::http-session-idle-max-age ::dt/duration) +(s/def ::http-session-updater-batch-max-age ::dt/duration) +(s/def ::http-session-updater-batch-max-size ::us/integer) +(s/def ::initial-data-file ::us/string) +(s/def ::initial-data-project-name ::us/string) (s/def ::ldap-attrs-email ::us/string) (s/def ::ldap-attrs-fullname ::us/string) (s/def ::ldap-attrs-photo ::us/string) - +(s/def ::ldap-attrs-username ::us/string) +(s/def ::ldap-base-dn ::us/string) +(s/def ::ldap-bind-dn ::us/string) +(s/def ::ldap-bind-password ::us/string) +(s/def ::ldap-host ::us/string) +(s/def ::ldap-port ::us/integer) +(s/def ::ldap-ssl ::us/boolean) +(s/def ::ldap-starttls ::us/boolean) +(s/def ::ldap-user-query ::us/string) +(s/def ::loggers-loki-uri ::us/string) +(s/def ::loggers-zmq-uri ::us/string) +(s/def ::media-directory ::us/string) +(s/def ::media-uri ::us/string) +(s/def ::profile-bounce-max-age ::dt/duration) +(s/def ::profile-bounce-threshold ::us/integer) +(s/def ::profile-complaint-max-age ::dt/duration) +(s/def ::profile-complaint-threshold ::us/integer) +(s/def ::public-uri ::us/string) +(s/def ::redis-uri ::us/string) +(s/def ::registration-domain-whitelist ::us/string) +(s/def ::registration-enabled ::us/boolean) +(s/def ::rlimits-image ::us/integer) +(s/def ::rlimits-password ::us/integer) +(s/def ::smtp-default-from ::us/string) +(s/def ::smtp-default-reply-to ::us/string) +(s/def ::smtp-enabled ::us/boolean) +(s/def ::smtp-host ::us/string) +(s/def ::smtp-password (s/nilable ::us/string)) +(s/def ::smtp-port ::us/integer) +(s/def ::smtp-ssl ::us/boolean) +(s/def ::smtp-tls ::us/boolean) +(s/def ::smtp-username (s/nilable ::us/string)) +(s/def ::srepl-host ::us/string) +(s/def ::srepl-port ::us/integer) +(s/def ::storage-backend ::us/keyword) +(s/def ::storage-fs-directory ::us/string) +(s/def ::storage-s3-bucket ::us/string) +(s/def ::storage-s3-region ::us/keyword) (s/def ::telemetry-enabled ::us/boolean) -(s/def ::telemetry-with-taiga ::us/boolean) -(s/def ::telemetry-uri ::us/string) (s/def ::telemetry-server-enabled ::us/boolean) (s/def ::telemetry-server-port ::us/integer) - -(s/def ::initial-data-file ::us/string) -(s/def ::initial-data-project-name ::us/string) - -(s/def ::default-blob-version ::us/integer) +(s/def ::telemetry-uri ::us/string) +(s/def ::telemetry-with-taiga ::us/boolean) +(s/def ::tenant ::us/string) (s/def ::config (s/keys :opt-un [::allow-demo-users @@ -185,6 +171,9 @@ ::google-client-id ::google-client-secret ::http-server-port + ::http-session-updater-batch-max-age + ::http-session-updater-batch-max-size + ::http-session-idle-max-age ::host ::ldap-attrs-username ::ldap-attrs-email diff --git a/backend/src/app/http/session.clj b/backend/src/app/http/session.clj index 45e25699f..bb5afecc5 100644 --- a/backend/src/app/http/session.clj +++ b/backend/src/app/http/session.clj @@ -9,11 +9,20 @@ (ns app.http.session (:require + [app.common.data :as d] + [app.common.exceptions :as ex] + [app.config :as cfg] [app.db :as db] + [app.metrics :as mtx] + [app.util.async :as aa] [app.util.log4j :refer [update-thread-context!]] + [app.util.time :as dt] + [app.worker :as wrk] [buddy.core.codecs :as bc] [buddy.core.nonce :as bn] + [clojure.core.async :as a] [clojure.spec.alpha :as s] + [clojure.tools.logging :as log] [integrant.core :as ig])) ;; --- IMPL @@ -42,8 +51,7 @@ (defn- retrieve [{:keys [conn] :as cfg} token] (when token - (-> (db/exec-one! conn ["select profile_id from http_session where id = ?" token]) - (:profile-id)))) + (db/exec-one! conn ["select id, profile_id from http_session where id = ?" token]))) (defn- retrieve-from-request [{:keys [cookie-name] :as cfg} {:keys [cookies] :as request}] @@ -57,24 +65,33 @@ (defn- middleware [cfg handler] (fn [request] - (if-let [profile-id (retrieve-from-request cfg request)] - (do + (if-let [{:keys [id profile-id] :as session} (retrieve-from-request cfg request)] + (let [ech (::events-ch cfg)] + (a/>!! ech id) (update-thread-context! {:profile-id profile-id}) (handler (assoc request :profile-id profile-id))) (handler request)))) -;; --- STATE INIT +;; --- STATE INIT: SESSION + +(s/def ::cookie-name ::cfg/http-session-cookie-name) (defmethod ig/pre-init-spec ::session [_] - (s/keys :req-un [::db/pool])) + (s/keys :req-un [::db/pool] + :opt-un [::cookie-name])) (defmethod ig/prep-key ::session [_ cfg] - (merge {:cookie-name "auth-token"} cfg)) + (merge {:cookie-name "auth-token" + :buffer-size 64} + (d/without-nils cfg))) (defmethod ig/init-key ::session [_ {:keys [pool] :as cfg}] - (let [cfg (assoc cfg :conn pool)] + (let [events (a/chan (a/dropping-buffer (:buffer-size cfg))) + cfg (assoc cfg + :conn pool + ::events-ch events)] (-> cfg (assoc :middleware #(middleware cfg %)) (assoc :create (fn [profile-id] @@ -89,3 +106,113 @@ :body "" :cookies (cookies cfg {:value "" :max-age -1}))))))) +(defmethod ig/halt-key! ::session + [_ data] + (a/close! (::events-ch data))) + +;; --- STATE INIT: SESSION UPDATER + +(declare batch-events) +(declare update-sessions) + +(s/def ::session map?) +(s/def ::max-batch-age ::cfg/http-session-updater-batch-max-age) +(s/def ::max-batch-size ::cfg/http-session-updater-batch-max-size) + +(defmethod ig/pre-init-spec ::updater [_] + (s/keys :req-un [::db/pool ::wrk/executor ::mtx/metrics ::session] + :opt-un [::max-batch-age + ::max-batch-size])) + +(defmethod ig/prep-key ::updater + [_ cfg] + (merge {:max-batch-age (dt/duration {:minutes 5}) + :max-batch-size 200} + (d/without-nils cfg))) + +(defmethod ig/init-key ::updater + [_ {:keys [session metrics] :as cfg}] + (log/infof "initialize session updater (max-batch-age=%s, max-batch-size=%s)" + (str (:max-batch-age cfg)) + (str (:max-batch-size cfg))) + (let [input (batch-events cfg (::events-ch session)) + mcnt (mtx/create + {:name "http_session_updater_count" + :help "A counter of session update batch events." + :registry (:registry metrics) + :type :counter})] + (a/go-loop [] + (when-let [[reason batch] (a/! out [:timeout buf]) + (recur (timeout-chan cfg) #{}))) + + (nil? val) + (a/close! out) + + (identical? port in) + (let [buf (conj buf val)] + (if (>= (count buf) (:max-batch-size cfg)) + (do + (a/>! out [:size buf]) + (recur (timeout-chan cfg) #{})) + (recur tch buf)))))) + out)) + +(defn- update-sessions + [{:keys [pool executor]} ids] + (aa/with-thread executor + (db/exec-one! pool ["update http_session set updated_at=now() where id = ANY(?)" + (into-array String ids)]) + (count ids))) + +;; --- STATE INIT: SESSION GC + +(declare sql:delete-expired) + +(s/def ::max-age ::dt/duration) + +(defmethod ig/pre-init-spec ::gc-task [_] + (s/keys :req-un [::db/pool] + :opt-un [::max-age])) + +(defmethod ig/prep-key ::gc-task + [_ cfg] + (merge {:max-age (dt/duration {:days 2})} + (d/without-nils cfg))) + +(defmethod ig/init-key ::gc-task + [_ {:keys [pool max-age] :as cfg}] + (fn [_] + (db/with-atomic [conn pool] + (let [interval (db/interval max-age) + result (db/exec-one! conn [sql:delete-expired interval]) + result (:next.jdbc/update-count result)] + (log/debugf "gc-task: removed %s rows from http-session table" result) + result)))) + +(def ^:private + sql:delete-expired + "delete from http_session + where updated_at < now() - ?::interval") diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index 03483c38e..81765c89a 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -69,7 +69,19 @@ :app.http.session/session {:pool (ig/ref :app.db/pool) - :cookie-name "auth-token"} + :cookie-name (:http-session-cookie-name config)} + + :app.http.session/gc-task + {:pool (ig/ref :app.db/pool) + :max-age (:http-session-idle-max-age config)} + + :app.http.session/updater + {:pool (ig/ref :app.db/pool) + :metrics (ig/ref :app.metrics/metrics) + :executor (ig/ref :app.worker/executor) + :session (ig/ref :app.http.session/session) + :max-batch-age (:http-session-updater-batch-max-age config) + :max-batch-size (:http-session-updater-batch-max-size config)} :app.http.awsns/handler {:tokens (ig/ref :app.tokens/tokens) @@ -197,6 +209,10 @@ :cron #app/cron "0 0 2 */1 * ?" ;; daily (2 hour shift) :task :storage-touched-gc} + {:id "session-gc" + :cron #app/cron "0 0 3 */1 * ?" ;; daily (3 hour shift) + :task :session-gc} + {:id "storage-recheck" :cron #app/cron "0 0 */1 * * ?" ;; hourly :task :storage-recheck} @@ -223,7 +239,8 @@ :storage-touched-gc (ig/ref :app.storage/gc-touched-task) :storage-recheck (ig/ref :app.storage/recheck-task) :tasks-gc (ig/ref :app.tasks.tasks-gc/handler) - :telemetry (ig/ref :app.tasks.telemetry/handler)}} + :telemetry (ig/ref :app.tasks.telemetry/handler) + :session-gc (ig/ref :app.http.session/gc-task)}} :app.tasks.sendmail/handler {:host (:smtp-host config) diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index f124bebb9..9afa129bf 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -158,6 +158,8 @@ {:name "0048-mod-storage-tables" :fn (mg/resource "app/migrations/sql/0048-mod-storage-tables.sql")} + {:name "0049-mod-http-session-table" + :fn (mg/resource "app/migrations/sql/0049-mod-http-session-table.sql")} ]) diff --git a/backend/src/app/migrations/sql/0049-mod-http-session-table.sql b/backend/src/app/migrations/sql/0049-mod-http-session-table.sql new file mode 100644 index 000000000..4ee4657ab --- /dev/null +++ b/backend/src/app/migrations/sql/0049-mod-http-session-table.sql @@ -0,0 +1,6 @@ +ALTER TABLE http_session + ADD COLUMN updated_at timestamptz NULL; + +CREATE INDEX http_session__updated_at__idx + ON http_session (updated_at) + WHERE updated_at IS NOT NULL;