From e768600df36335f0c0e24f4a6281fdc229d109ba Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 25 Aug 2021 10:38:15 +0200 Subject: [PATCH] :recycle: Enable receiving frontend audit log on backend. --- backend/src/app/db.clj | 1 - backend/src/app/http.clj | 16 ++- backend/src/app/loggers/audit.clj | 119 ++++++++++++++---- backend/src/app/main.clj | 6 + backend/src/app/migrations.clj | 3 + .../sql/0064-mod-audit-log-table.sql | 13 ++ backend/src/app/rpc.clj | 4 +- frontend/src/app/main/data/events.cljs | 2 +- 8 files changed, 130 insertions(+), 34 deletions(-) create mode 100644 backend/src/app/migrations/sql/0064-mod-audit-log-table.sql diff --git a/backend/src/app/db.clj b/backend/src/app/db.clj index dd23992f7..6f1b940af 100644 --- a/backend/src/app/db.clj +++ b/backend/src/app/db.clj @@ -300,7 +300,6 @@ (.createArrayOf conn ^String type (into-array Object objects)) (.createArrayOf conn ^String type objects)))) - (defn decode-pgpoint [^PGpoint v] (gpt/point (.-x v) (.-y v))) diff --git a/backend/src/app/http.clj b/backend/src/app/http.clj index 248a125dd..398ee1c97 100644 --- a/backend/src/app/http.clj +++ b/backend/src/app/http.clj @@ -114,9 +114,14 @@ (s/def ::storage map?) (s/def ::assets map?) (s/def ::feedback fn?) +(s/def ::error-report-handler fn?) +(s/def ::audit-http-handler fn?) (defmethod ig/pre-init-spec ::router [_] - (s/keys :req-un [::rpc ::session ::mtx/metrics ::oauth ::storage ::assets ::feedback])) + (s/keys :req-un [::rpc ::session ::mtx/metrics + ::oauth ::storage ::assets ::feedback + ::error-report-handler + ::audit-http-handler])) (defmethod ig/init-key ::router [_ {:keys [session rpc oauth metrics assets feedback] :as cfg}] @@ -136,10 +141,7 @@ ["/webhooks" ["/sns" {:post (:sns-webhook cfg)}]] - ["/api" {:middleware [ - ;; Temporary disabled - #_[middleware/etag] - [middleware/format-response-body] + ["/api" {:middleware [[middleware/format-response-body] [middleware/params] [middleware/multipart-params] [middleware/keyword-params] @@ -149,10 +151,12 @@ ["/feedback" {:middleware [(:middleware session)] :post feedback}] - ["/auth/oauth/:provider" {:post (:handler oauth)}] ["/auth/oauth/:provider/callback" {:get (:callback-handler oauth)}] + ["/audit/events" {:middleware [(:middleware session)] + :post (:audit-http-handler cfg)}] + ["/rpc" {:middleware [(:middleware session)]} ["/query/:type" {:get (:query-handler rpc) :post (:query-handler rpc)}] diff --git a/backend/src/app/loggers/audit.clj b/backend/src/app/loggers/audit.clj index dbdff8920..5fe39086f 100644 --- a/backend/src/app/loggers/audit.clj +++ b/backend/src/app/loggers/audit.clj @@ -23,7 +23,8 @@ [clojure.spec.alpha :as s] [cuerdas.core :as str] [integrant.core :as ig] - [lambdaisland.uri :as u])) + [lambdaisland.uri :as u] + [promesa.exec :as px])) (defn parse-client-ip [{:keys [headers] :as request}] @@ -67,6 +68,65 @@ (update event :props #(-> % clean-common clean-profile-id clean-complex-data)))) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; HTTP Handler +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(declare persist-http-events) + +(s/def ::profile-id ::us/uuid) +(s/def ::name ::us/string) +(s/def ::type ::us/string) +(s/def ::props (s/map-of ::us/keyword any?)) +(s/def ::timestamp dt/instant?) +(s/def ::context (s/map-of ::us/keyword any?)) + +(s/def ::event + (s/keys :req-un [::type ::name ::props ::timestamp ::profile-id] + :opt-un [::context])) + +(s/def ::events (s/every ::event)) + +(defmethod ig/init-key ::http-handler + [_ {:keys [executor enabled] :as cfg}] + (fn [{:keys [params headers cookies profile-id] :as request}] + (when enabled + (let [events (->> (:events params) + (remove #(not= profile-id (:profile-id %))) + (us/conform ::events)) + ip-addr (parse-client-ip request) + cfg (-> cfg + (assoc :source "frontend") + (assoc :events events) + (assoc :ip-addr ip-addr))] + (px/run! executor #(persist-http-events cfg)))) + {:status 204 :body ""})) + +(defn- persist-http-events + [{:keys [pool events ip-addr source] :as cfg}] + (try + (let [columns [:id :name :source :type :tracked-at :profile-id :ip-addr :props :context] + prepare-xf (map (fn [event] + [(uuid/next) + (:name event) + source + (:type event) + (:timestamp event) + (:profile-id event) + (db/inet ip-addr) + (db/tjson (:props event)) + (db/tjson (d/without-nils (:context event)))])) + events (us/conform ::events events) + rows (into [] prepare-xf events)] + (db/insert-multi! pool :audit-log columns rows)) + (catch Throwable e + (let [xdata (ex-data e)] + (if (= :spec-validation (:code xdata)) + (l/error ::l/raw (str "spec validation on persist-events:\n" + (:explain xdata))) + (l/error :hint "error on persist-events" + :cause e)))))) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Collector ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -103,7 +163,9 @@ (recur))) (fn [& {:keys [cmd] :as params}] - (let [params (dissoc params :cmd)] + (let [params (-> params + (dissoc :cmd) + (assoc :tracked-at (dt/now)))] (case cmd :stop (a/close! input) :submit (when-not (a/offer! input params) @@ -117,13 +179,14 @@ (:name event) (:type event) (:profile-id event) + (:tracked-at event) (some-> (:ip-addr event) db/inet) - (db/tjson (:props event))])] - + (db/tjson (:props event)) + "backend"])] (aa/with-thread executor (db/with-atomic [conn pool] (db/insert-multi! conn :audit-log - [:id :name :type :profile-id :ip-addr :props] + [:id :name :type :profile-id :tracked-at :ip-addr :props :source] (sequence (map event->row) events)))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -144,16 +207,19 @@ (defmethod ig/init-key ::archive-task [_ {:keys [uri enabled] :as cfg}] - (fn [_] - (when (and enabled (not uri)) - (ex/raise :type :internal - :code :task-not-configured - :hint "archive task not configured, missing uri")) - (loop [] - (let [res (archive-events cfg)] - (when (= res :continue) - (aa/thread-sleep 200) - (recur)))))) + (fn [props] + (let [enabled (or enabled (:enabled props false)) + uri (or uri (:uri props))] + (when (and enabled (not uri)) + (ex/raise :type :internal + :code :task-not-configured + :hint "archive task not configured, missing uri")) + (when enabled + (loop [] + (let [res (archive-events cfg)] + (when (= res :continue) + (aa/thread-sleep 200) + (recur)))))))) (def sql:retrieve-batch-of-audit-log "select * from audit_log @@ -164,22 +230,27 @@ (defn archive-events [{:keys [pool uri tokens] :as cfg}] - (letfn [(decode-row [{:keys [props ip-addr] :as row}] + (letfn [(decode-row [{:keys [props ip-addr context] :as row}] (cond-> row (db/pgobject? props) (assoc :props (db/decode-transit-pgobject props)) + (db/pgobject? context) + (assoc :context (db/decode-transit-pgobject context)) + (db/pgobject? ip-addr "inet") (assoc :ip-addr (db/decode-inet ip-addr)))) - (row->event [{:keys [name type created-at profile-id props ip-addr]}] - (cond-> {:type type - :name name - :timestamp created-at - :profile-id profile-id - :props props} - (some? ip-addr) - (update :context assoc :source-ip ip-addr))) + (row->event [row] + (select-keys row [:type + :name + :source + :created-at + :tracked-at + :profile-id + :ip-addr + :props + :context])) (send [events] (let [token (tokens :generate {:iss "authentication" diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index e882f7165..8baefbbb9 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -95,6 +95,7 @@ :storage (ig/ref :app.storage/storage) :sns-webhook (ig/ref :app.http.awsns/handler) :feedback (ig/ref :app.http.feedback/handler) + :audit-http-handler (ig/ref :app.loggers.audit/http-handler) :error-report-handler (ig/ref :app.loggers.mattermost/handler)} :app.http.assets/handlers @@ -289,6 +290,11 @@ :app.loggers.zmq/receiver {:endpoint (cf/get :loggers-zmq-uri)} + :app.loggers.audit/http-handler + {:enabled (cf/get :audit-enabled false) + :pool (ig/ref :app.db/pool) + :executor (ig/ref :app.worker/executor)} + :app.loggers.audit/collector {:enabled (cf/get :audit-enabled false) :pool (ig/ref :app.db/pool) diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index 831213383..d1afda81e 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -199,6 +199,9 @@ {:name "0063-add-share-link-table" :fn (mg/resource "app/migrations/sql/0063-add-share-link-table.sql")} + + {:name "0064-mod-audit-log-table" + :fn (mg/resource "app/migrations/sql/0064-mod-audit-log-table.sql")} ]) diff --git a/backend/src/app/migrations/sql/0064-mod-audit-log-table.sql b/backend/src/app/migrations/sql/0064-mod-audit-log-table.sql new file mode 100644 index 000000000..5d85898b9 --- /dev/null +++ b/backend/src/app/migrations/sql/0064-mod-audit-log-table.sql @@ -0,0 +1,13 @@ +ALTER TABLE audit_log + ADD COLUMN tracked_at timestamptz NULL DEFAULT clock_timestamp(), + ADD COLUMN source text NULL, + ADD COLUMN context jsonb NULL; + +ALTER TABLE audit_log + ALTER COLUMN source SET STORAGE external, + ALTER COLUMN context SET STORAGE external; + +UPDATE audit_log SET source = 'backend', tracked_at=created_at; + +-- ALTER TABLE audit_log ALTER COLUMN source SET NOT NULL; +-- ALTER TABLE audit_log ALTER COLUMN tracked_at SET NOT NULL; diff --git a/backend/src/app/rpc.clj b/backend/src/app/rpc.clj index eae7ef33c..d77f83e3e 100644 --- a/backend/src/app/rpc.clj +++ b/backend/src/app/rpc.clj @@ -117,14 +117,14 @@ profile-id (or (:profile-id params') (:profile-id result) (::audit/profile-id resultm)) - props (d/merge params (::audit/props resultm))] + props (d/merge params' (::audit/props resultm))] (audit :cmd :submit :type (::type cfg) :name (or (::audit/name resultm) (::sv/name mdata)) :profile-id profile-id :ip-addr (audit/parse-client-ip request) - :props (audit/profile->props props)))) + :props props))) result)))) diff --git a/frontend/src/app/main/data/events.cljs b/frontend/src/app/main/data/events.cljs index 83c2a835f..e1fe56dd5 100644 --- a/frontend/src/app/main/data/events.cljs +++ b/frontend/src/app/main/data/events.cljs @@ -164,7 +164,7 @@ (defn- persist-events [events] (if (seq events) - (let [uri (u/join cf/public-uri "events") + (let [uri (u/join cf/public-uri "api/audit/events") params {:events events}] (->> (http/send! {:uri uri :method :post