From 334ac26f0d4ee1e2e8913cee420987b0ebf1fab9 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Sun, 9 May 2021 14:06:27 +0200 Subject: [PATCH] :sparkles: Add improved activity logging. --- backend/src/app/config.clj | 14 ++- backend/src/app/http/oauth.clj | 23 +++- backend/src/app/loggers/activity.clj | 127 ++++++++++++++++++++++ backend/src/app/loggers/loki.clj | 8 +- backend/src/app/main.clj | 16 ++- backend/src/app/rpc.clj | 30 +++-- backend/src/app/rpc/mutations/profile.clj | 14 ++- backend/src/app/rpc/queries/files.clj | 3 +- frontend/src/app/main/data/events.cljs | 1 - frontend/src/app/main/ui/auth.cljs | 3 +- 10 files changed, 204 insertions(+), 35 deletions(-) create mode 100644 backend/src/app/loggers/activity.clj diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index e50fd7cc9..fde08020d 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -8,6 +8,7 @@ "A configuration management." (:refer-clojure :exclude [get]) (:require + [app.common.data :as d] [app.common.spec :as us] [app.common.version :as v] [app.util.time :as dt] @@ -16,7 +17,8 @@ [clojure.pprint :as pprint] [clojure.spec.alpha :as s] [cuerdas.core :as str] - [environ.core :refer [env]])) + [environ.core :refer [env]] + [integrant.core :as ig])) (prefer-method print-method clojure.lang.IRecord @@ -26,6 +28,16 @@ clojure.lang.IPersistentMap clojure.lang.IDeref) +(defmethod ig/init-key :default + [_ data] + (d/without-nils data)) + +(defmethod ig/prep-key :default + [_ data] + (if (map? data) + (d/without-nils data) + data)) + (def defaults {:http-server-port 6060 :host "devenv" diff --git a/backend/src/app/http/oauth.clj b/backend/src/app/http/oauth.clj index 60e0a90c8..55c7a8a50 100644 --- a/backend/src/app/http/oauth.clj +++ b/backend/src/app/http/oauth.clj @@ -6,6 +6,7 @@ (ns app.http.oauth (:require + [app.common.data :as d] [app.common.exceptions :as ex] [app.common.spec :as us] [app.common.uri :as u] @@ -138,16 +139,32 @@ (cond-> info (some? (:invitation-token state)) - (assoc :invitation-token (:invitation-token state))))) + (assoc :invitation-token (:invitation-token state)) + + (map? (:props state)) + (d/merge (:props state))))) ;; --- HTTP HANDLERS +(defn extract-props + [params] + (reduce-kv (fn [params k v] + (let [sk (name k)] + (cond-> params + (or (str/starts-with? sk "pm_") + (str/starts-with? sk "pm-")) + (assoc (-> sk str/kebab keyword) v)))) + {} + params)) + (defn- auth-handler - [{:keys [tokens] :as cfg} request] - (let [invitation (get-in request [:params :invitation-token]) + [{:keys [tokens] :as cfg} {:keys [params] :as request}] + (let [invitation (:invitation-token params) + props (extract-props params) state (tokens :generate {:iss :oauth :invitation-token invitation + :props props :exp (dt/in-future "15m")}) uri (build-auth-uri cfg state)] {:status 200 diff --git a/backend/src/app/loggers/activity.clj b/backend/src/app/loggers/activity.clj new file mode 100644 index 000000000..4f02fb701 --- /dev/null +++ b/backend/src/app/loggers/activity.clj @@ -0,0 +1,127 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) UXBOX Labs SL + +(ns app.loggers.activity + "Activity registry logger consumer." + (:require + [app.common.data :as d] + [app.common.spec :as us] + [app.config :as cf] + [app.util.async :as aa] + [app.util.http :as http] + [app.util.logging :as l] + [app.util.time :as dt] + [app.util.transit :as t] + [app.worker :as wrk] + [clojure.core.async :as a] + [clojure.spec.alpha :as s] + [integrant.core :as ig] + [lambdaisland.uri :as u])) + +(declare process-event) +(declare handle-event) + +(s/def ::uri ::us/string) + +(defmethod ig/pre-init-spec ::reporter [_] + (s/keys :req-un [::wrk/executor] + :opt-un [::uri])) + +(defmethod ig/init-key ::reporter + [_ {:keys [uri] :as cfg}] + (if (string? uri) + (do + (l/info :msg "intializing activity reporter" :uri uri) + (let [xform (comp (map process-event) + (filter map?)) + input (a/chan (a/sliding-buffer 1024) xform)] + (a/go-loop [] + (when-let [event (a/ params + (or (string? v) + (uuid? v) + (number? v)) + (assoc k v))) + {} + params))) + +(defn- process-event + [{:keys [type name params result] :as event}] + (let [profile-id (:profile-id params)] + (if (uuid? profile-id) + {:type (str "backend:" (d/name type)) + :name name + :timestamp (dt/now) + :profile-id profile-id + :props (clean-params params)} + (cond + (= "register-profile" name) + {:type (str "backend:" (d/name type)) + :name name + :timestamp (dt/now) + :profile-id (:id result) + :props (clean-params (:props result))} + + :else nil)))) + +(defn- send-activity + [{:keys [uri tokens]} event i] + (try + (let [token (tokens :generate {:iss "authentication" + :iat (dt/now) + :uid (:profile-id event)}) + body (t/encode {:events [event]}) + headers {"content-type" "application/transit+json" + "origin" (cf/get :public-uri) + "cookie" (u/map->query-string {:auth-token token})} + params {:uri uri + :timeout 6000 + :method :post + :headers headers + :body body} + response (http/send! params)] + (if (= (:status response) 204) + true + (do + (l/error :hint "error on sending activity" + :try i + :rsp (pr-str response)) + false))) + (catch Exception e + (l/error :hint "error on sending message to loki" + :cause e + :try i) + false))) + +(defn- handle-event + [{:keys [executor] :as cfg} event] + (aa/with-thread executor + (loop [i 1] + (when (and (not (send-activity cfg event i)) (< i 20)) + (Thread/sleep (* i 2000)) + (recur (inc i)))))) + diff --git a/backend/src/app/loggers/loki.clj b/backend/src/app/loggers/loki.clj index 607f06e3b..1bafb7c4f 100644 --- a/backend/src/app/loggers/loki.clj +++ b/backend/src/app/loggers/loki.clj @@ -31,16 +31,16 @@ [_ {:keys [receiver uri] :as cfg}] (when uri (l/info :msg "intializing loki reporter" :uri uri) - (let [output (a/chan (a/sliding-buffer 1024))] - (receiver :sub output) + (let [input (a/chan (a/sliding-buffer 1024))] + (receiver :sub input) (a/go-loop [] - (let [msg (a/> (sv/scan-ns 'app.rpc.mutations.demo 'app.rpc.mutations.media 'app.rpc.mutations.profile @@ -152,9 +164,11 @@ (s/def ::storage some?) (s/def ::session map?) (s/def ::tokens fn?) +(s/def ::activity some?) (defmethod ig/pre-init-spec ::rpc [_] - (s/keys :req-un [::db/pool ::storage ::session ::tokens ::mtx/metrics ::rlm/rlimits])) + (s/keys :req-un [::storage ::session ::tokens ::activity + ::mtx/metrics ::rlm/rlimits ::db/pool])) (defmethod ig/init-key ::rpc [_ cfg] diff --git a/backend/src/app/rpc/mutations/profile.clj b/backend/src/app/rpc/mutations/profile.clj index 01fdb5243..6e75017a5 100644 --- a/backend/src/app/rpc/mutations/profile.clj +++ b/backend/src/app/rpc/mutations/profile.clj @@ -13,6 +13,7 @@ [app.config :as cfg] [app.db :as db] [app.emails :as eml] + [app.http.oauth :refer [extract-props]] [app.media :as media] [app.rpc.mutations.projects :as projects] [app.rpc.mutations.teams :as teams] @@ -101,7 +102,8 @@ resp {:invitation-token token}] (with-meta resp {:transform-response ((:create session) (:id profile)) - :before-complete (annotate-profile-register metrics profile)})) + :before-complete (annotate-profile-register metrics profile) + :result profile})) ;; If no token is provided, send a verification email (let [vtoken (tokens :generate @@ -129,7 +131,8 @@ :extra-data ptoken}) (with-meta profile - {:before-complete (annotate-profile-register metrics profile)}))))) + {:before-complete (annotate-profile-register metrics profile) + :result profile}))))) (defn email-domain-in-whitelist? "Returns true if email's domain is in the given whitelist or if given @@ -174,11 +177,12 @@ (defn create-profile "Create the profile entry on the database with limited input filling all the other fields with defaults." - [conn {:keys [id fullname email password props is-active is-muted is-demo opts] - :or {is-active false is-muted false is-demo false}}] + [conn {:keys [id fullname email password is-active is-muted is-demo opts] + :or {is-active false is-muted false is-demo false} + :as params}] (let [id (or id (uuid/next)) is-active (if is-demo true is-active) - props (db/tjson (or props {})) + props (-> params extract-props db/tjson) password (derive-password password) params {:id id :fullname fullname diff --git a/backend/src/app/rpc/queries/files.clj b/backend/src/app/rpc/queries/files.clj index 066ee7e10..b781c783a 100644 --- a/backend/src/app/rpc/queries/files.clj +++ b/backend/src/app/rpc/queries/files.clj @@ -6,7 +6,6 @@ (ns app.rpc.queries.files (:require - [app.common.exceptions :as ex] [app.common.pages.migrations :as pmg] [app.common.spec :as us] [app.db :as db] @@ -254,7 +253,7 @@ (defn retrieve-file-libraries [conn is-indirect file-id] (let [libraries (->> (db/exec! conn [sql:file-libraries file-id]) - (map #(assoc :is-indirect is-indirect)) + (map #(assoc % :is-indirect is-indirect)) (into #{} decode-row-xf))] (reduce #(into %1 (retrieve-file-libraries conn true %2)) libraries diff --git a/frontend/src/app/main/data/events.cljs b/frontend/src/app/main/data/events.cljs index 1e75b1d4a..ca4b99a61 100644 --- a/frontend/src/app/main/data/events.cljs +++ b/frontend/src/app/main/data/events.cljs @@ -179,7 +179,6 @@ (effect [_ state stream] (let [profile-id (:profile-id state) events (into [] (take max-buffer-size) @buffer)] - (prn ::persistence (count events) profile-id) (when (seq events) (->> events (filterv #(= profile-id (:profile-id %))) diff --git a/frontend/src/app/main/ui/auth.cljs b/frontend/src/app/main/ui/auth.cljs index c5f9cf846..06b56dca4 100644 --- a/frontend/src/app/main/ui/auth.cljs +++ b/frontend/src/app/main/ui/auth.cljs @@ -54,8 +54,7 @@ [:& recovery-request-page {:locale locale}] :auth-recovery - [:& recovery-page {:locale locale - :params (:query-params route)}]) + [:& recovery-page {:locale locale :params params}]) [:div.terms-login [:a {:href "https://penpot.app/terms.html" :target "_blank"} "Terms of service"] [:span "and"]