diff --git a/backend/src/app/cli/manage.clj b/backend/src/app/cli/manage.clj index d20758196..dad68fc6b 100644 --- a/backend/src/app/cli/manage.clj +++ b/backend/src/app/cli/manage.clj @@ -140,7 +140,6 @@ indicating the action the program should take and the options provided." [args] (let [{:keys [options arguments errors summary] :as opts} (parse-opts args cli-options)] - ;; (pp/pprint opts) (cond (:help options) ; help => exit OK with usage summary {:exit-message (usage summary) :ok? true} diff --git a/backend/src/app/db.clj b/backend/src/app/db.clj index d6eb7fa72..479a8fdbc 100644 --- a/backend/src/app/db.clj +++ b/backend/src/app/db.clj @@ -50,16 +50,26 @@ (declare instrument-jdbc!) (declare apply-migrations!) -(s/def ::name keyword?) -(s/def ::uri ::us/not-empty-string) -(s/def ::min-pool-size ::us/integer) +(s/def ::connection-timeout ::us/integer) (s/def ::max-pool-size ::us/integer) (s/def ::migrations map?) +(s/def ::min-pool-size ::us/integer) +(s/def ::name keyword?) +(s/def ::password ::us/string) (s/def ::read-only ::us/boolean) +(s/def ::uri ::us/not-empty-string) +(s/def ::username ::us/string) +(s/def ::validation-timeout ::us/integer) (defmethod ig/pre-init-spec ::pool [_] - (s/keys :req-un [::uri ::name ::min-pool-size ::max-pool-size] - :opt-un [::migrations ::mtx/metrics ::read-only])) + (s/keys :req-un [::uri ::name ::username ::password] + :opt-un [::min-pool-size + ::max-pool-size + ::connection-timeout + ::validation-timeout + ::migrations + ::mtx/metrics + ::read-only])) (defmethod ig/init-key ::pool [_ {:keys [migrations metrics name read-only] :as cfg}] @@ -111,11 +121,11 @@ (.setPoolName (d/name (:name cfg))) (.setAutoCommit true) (.setReadOnly read-only) - (.setConnectionTimeout 10000) ;; 10seg - (.setValidationTimeout 10000) ;; 10seg + (.setConnectionTimeout (:connection-timeout cfg 10000)) ;; 10seg + (.setValidationTimeout (:validation-timeout cfg 10000)) ;; 10seg (.setIdleTimeout 120000) ;; 2min (.setMaxLifetime 1800000) ;; 30min - (.setMinimumIdle (:min-pool-size cfg 0)) + (.setMinimumIdle (:min-pool-size cfg 0)) (.setMaximumPoolSize (:max-pool-size cfg 50)) (.setConnectionInitSql initsql) (.setInitializationFailTimeout -1)) diff --git a/backend/src/app/http.clj b/backend/src/app/http.clj index d55028199..594916a06 100644 --- a/backend/src/app/http.clj +++ b/backend/src/app/http.clj @@ -76,11 +76,11 @@ (try (handler request) (catch Throwable e - (l/with-context (errors/get-error-context request e) - (l/error :hint "unexpected error processing request" - :query-string (:query-string request) - :cause e) - {:status 500 :body "internal server error"})))))) + (l/error :hint "unexpected error processing request" + ::l/context (errors/get-error-context request e) + :query-string (:query-string request) + :cause e) + {:status 500 :body "internal server error"}))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Http Router @@ -150,8 +150,8 @@ [middleware/errors errors/handle] [middleware/cookies]]} + ["/health" {:get (:health-check debug)}] ["/_doc" {:get (doc/handler rpc)}] - ["/feedback" {:middleware [(:middleware session)] :post feedback}] ["/auth/oauth/:provider" {:post (:handler oauth)}] diff --git a/backend/src/app/http/debug.clj b/backend/src/app/http/debug.clj index 2098cd084..e31040618 100644 --- a/backend/src/app/http/debug.clj +++ b/backend/src/app/http/debug.clj @@ -18,9 +18,9 @@ [app.util.template :as tmpl] [app.util.time :as dt] [clojure.java.io :as io] - [clojure.pprint :as ppr] [cuerdas.core :as str] [datoteka.core :as fs] + [fipp.edn :as fpp] [integrant.core :as ig])) ;; (selmer.parser/cache-off!) @@ -147,21 +147,20 @@ (some-> (db/get-by-id pool :server-error-report id) :content db/decode-transit-pgobject))) (render-template [report] - (binding [ppr/*print-right-margin* 300] - (let [context (dissoc report - :trace :cause :params :data :spec-problems - :spec-explain :spec-value :error :explain :hint) - params {:context (with-out-str (ppr/pprint context)) - :hint (:hint report) - :spec-explain (:spec-explain report) - :spec-problems (:spec-problems report) - :spec-value (:spec-value report) - :data (:data report) - :trace (or (:trace report) - (some-> report :error :trace)) - :params (:params report)}] - (-> (io/resource "templates/error-report.tmpl") - (tmpl/render params))))) + (let [context (dissoc report + :trace :cause :params :data :spec-problems + :spec-explain :spec-value :error :explain :hint) + params {:context (with-out-str (fpp/pprint context {:width 300})) + :hint (:hint report) + :spec-explain (:spec-explain report) + :spec-problems (:spec-problems report) + :spec-value (:spec-value report) + :data (:data report) + :trace (or (:trace report) + (some-> report :error :trace)) + :params (:params report)}] + (-> (io/resource "templates/error-report.tmpl") + (tmpl/render params)))) ] (when-not (authorized? pool request) @@ -195,9 +194,17 @@ :body (-> (io/resource "templates/error-list.tmpl") (tmpl/render {:items items}))})) +(defn health-check + "Mainly a task that performs a health check." + [{:keys [pool]} _] + (db/with-atomic [conn pool] + (db/exec-one! conn ["select count(*) as count from server_prop;"]) + {:status 200 :body "Ok"})) + (defmethod ig/init-key ::handlers [_ cfg] {:index (partial index cfg) + :health-check (partial health-check cfg) :retrieve-file-data (partial retrieve-file-data cfg) :retrieve-file-changes (partial retrieve-file-changes cfg) :retrieve-error (partial retrieve-error cfg) diff --git a/backend/src/app/http/errors.clj b/backend/src/app/http/errors.clj index 052764b79..a53637849 100644 --- a/backend/src/app/http/errors.clj +++ b/backend/src/app/http/errors.clj @@ -11,7 +11,6 @@ [app.common.logging :as l] [app.common.spec :as us] [app.common.uuid :as uuid] - [clojure.pprint] [clojure.spec.alpha :as s] [cuerdas.core :as str])) @@ -36,6 +35,7 @@ :data (some-> data (dissoc ::s/problems ::s/value ::s/spec)) :ip-addr (parse-client-ip request) :profile-id (:profile-id request)} + (let [headers (:headers request)] {:user-agent (get headers "user-agent") :frontend-version (get headers "x-frontend-version" "unknown")}) @@ -70,8 +70,10 @@ (defmethod handle-exception :assertion [error request] (let [edata (ex-data error)] - (l/with-context (get-error-context request error) - (l/error ::l/raw (ex-message error) :cause error)) + (l/error ::l/raw (ex-message error) + ::l/context (get-error-context request error) + :cause error) + {:status 500 :body {:type :server-error :code :assertion @@ -93,9 +95,9 @@ (ex/exception? (:handling edata))) (handle-exception (:handling edata) request) (do - (l/with-context (get-error-context request error) - (l/error ::l/raw (ex-message error) :cause error)) - + (l/error ::l/raw (ex-message error) + ::l/context (get-error-context request error) + :cause error) {:status 500 :body {:type :server-error :code :unexpected @@ -105,10 +107,9 @@ (defmethod handle-exception org.postgresql.util.PSQLException [error request] (let [state (.getSQLState ^java.sql.SQLException error)] - - (l/with-context (get-error-context request error) - (l/error ::l/raw (ex-message error) :cause error)) - + (l/error ::l/raw (ex-message error) + ::l/context (get-error-context request error) + :cause error) (cond (= state "57014") {:status 504 diff --git a/backend/src/app/loggers/audit.clj b/backend/src/app/loggers/audit.clj index b212b24c4..7432aa9f7 100644 --- a/backend/src/app/loggers/audit.clj +++ b/backend/src/app/loggers/audit.clj @@ -193,20 +193,21 @@ (defn- persist-events [{:keys [pool executor] :as cfg} events] (letfn [(event->row [event] - [(uuid/next) - (:name event) - (:type event) - (:profile-id event) - (:tracked-at event) - (some-> (:ip-addr event) db/inet) - (db/tjson (:props event)) - "backend"])] + (when (:profile-id event) + [(uuid/next) + (:name event) + (:type event) + (:profile-id event) + (:tracked-at event) + (some-> (:ip-addr event) db/inet) + (db/tjson (:props event)) + "backend"]))] (aa/with-thread executor (when (seq events) (db/with-atomic [conn pool] (db/insert-multi! conn :audit-log [:id :name :type :profile-id :tracked-at :ip-addr :props :source] - (sequence (map event->row) events))))))) + (sequence (keep event->row) events))))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Archive Task diff --git a/backend/src/app/loggers/database.clj b/backend/src/app/loggers/database.clj index 27a4764de..22389d3a2 100644 --- a/backend/src/app/loggers/database.clj +++ b/backend/src/app/loggers/database.clj @@ -70,12 +70,15 @@ (defmethod ig/pre-init-spec ::reporter [_] (s/keys :req-un [::wrk/executor ::db/pool ::receiver])) +(defn error-event? + [event] + (= "error" (:logger/level event))) + (defmethod ig/init-key ::reporter [_ {:keys [receiver] :as cfg}] (l/info :msg "initializing database error persistence") - (let [output (a/chan (a/sliding-buffer 128) - (filter (fn [event] - (= (:logger/level event) "error"))))] + (let [output (a/chan (a/sliding-buffer 5) + (filter error-event?))] (receiver :sub output) (a/go-loop [] (let [msg (a/> data ::s/problems (take 10) seq vec) + :spec-value (some->> data ::s/value) + :data (some-> data (dissoc ::s/problems ::s/value ::s/spec)) + :params item} + (when (and data (::s/problems data)) + {:spec-explain (us/pretty-explain data)})))) (defn- handle-exception [error item] @@ -277,8 +283,10 @@ (= ::noop (:strategy edata)) (assoc :inc-by 0)) - (l/with-context (get-error-context error item) - (l/error :cause error :hint "unhandled exception on task") + (do + (l/error :hint "unhandled exception on task" + ::l/context (get-error-context error item) + :cause error) (if (>= (:retry-num item) (:max-retries item)) {:status :failed :task item :error error} {:status :retry :task item :error error}))))) diff --git a/common/src/app/common/logging.cljc b/common/src/app/common/logging.cljc index aea40b935..8cead52ed 100644 --- a/common/src/app/common/logging.cljc +++ b/common/src/app/common/logging.cljc @@ -9,6 +9,7 @@ [app.common.exceptions :as ex] [clojure.pprint :refer [pprint]] [cuerdas.core :as str] + [fipp.edn :as fpp] #?(:clj [io.aviso.exception :as ie]) #?(:cljs [goog.log :as glog])) #?(:cljs (:require-macros [app.common.logging]) @@ -52,22 +53,16 @@ (defn stringify-data [val] (cond - (instance? clojure.lang.Named val) - (name val) - - (instance? Throwable val) - (binding [ie/*app-frame-names* [#"app.*"] - ie/*fonts* nil - ie/*traditional* true] - (ie/format-exception val nil)) - (string? val) val + (instance? clojure.lang.Named val) + (name val) + (coll? val) - (binding [clojure.pprint/*print-right-margin* 200] - (-> (with-out-str (pprint val)) - (simple-prune (* 1024 1024 3)))) + (binding [*print-level* 5 + *print-length* 20] + (with-out-str (fpp/pprint val {:width 200}))) :else (str val)))) @@ -163,13 +158,13 @@ (.isEnabled ^Logger logger ^Level level))) (defmacro log - [& {:keys [level cause ::logger ::async ::raw] :or {async true} :as props}] + [& {:keys [level cause ::logger ::async ::raw ::context] :or {async true} :as props}] (if (:ns &env) ; CLJS `(write-log! ~(or logger (str *ns*)) ~level ~cause - (or ~raw ~(dissoc props :level :cause ::logger ::raw))) - (let [props (dissoc props :level :cause ::logger ::async ::raw) + (or ~raw ~(dissoc props :level :cause ::logger ::raw ::context))) + (let [props (dissoc props :level :cause ::logger ::async ::raw ::context) logger (or logger (str *ns*)) logger-sym (gensym "log") level-sym (gensym "log")] @@ -180,7 +175,7 @@ `(->> (ThreadContext/getImmutableContext) (send-off logging-agent (fn [_# cdata#] - (with-context (into {} cdata#) + (with-context (-> {} (into cdata#) (into ~context)) (->> (or ~raw (build-map-message ~props)) (write-log! ~logger-sym ~level-sym ~cause)))))) diff --git a/frontend/src/app/worker/import.cljs b/frontend/src/app/worker/import.cljs index 1dc38cc52..ad09e48bb 100644 --- a/frontend/src/app/worker/import.cljs +++ b/frontend/src/app/worker/import.cljs @@ -340,7 +340,10 @@ pre-process-images (->> (rx/from nodes) (rx/filter media-node?) - (rx/merge-map + ;; TODO: this should be merge-map, but we disable the + ;; parallel upload until we resolve resource usage issues + ;; on backend. + (rx/mapcat (fn [node] (->> (resolve-media context file-id node) (rx/map (fn [result] [node result])))))