Add metrics implementation* Covers proxy service on server side* Update documentation

This commit is contained in:
Travis Groth 2019-06-10 21:17:32 -04:00
parent fb3ed64fa1
commit ff528e8c7b
10 changed files with 310 additions and 1 deletions

View file

@ -5,6 +5,7 @@
### NEW
- Add programmatic authentication support. [GH-177]
- Add Prometheus format metrics endpoint. [GH-35]
### CHANGED

View file

@ -8,6 +8,8 @@ import (
"os"
"time"
"github.com/pomerium/pomerium/internal/metrics"
"github.com/fsnotify/fsnotify"
"github.com/pomerium/pomerium/authenticate"
"github.com/pomerium/pomerium/authorize"
@ -83,6 +85,10 @@ func main() {
IdleTimeout: opt.IdleTimeout,
}
if opt.MetricsAddr != "" {
go newPromListener(opt.MetricsAddr)
}
if srv, err := startRedirectServer(opt.HTTPRedirectAddr); err != nil {
log.Debug().Err(err).Msg("cmd/pomerium: http redirect server not started")
} else {
@ -151,8 +157,14 @@ func newProxyService(opt config.Options, mux *http.ServeMux) (*proxy.Proxy, erro
return service, nil
}
func newPromListener(addr string) {
log.Info().Str("MetricsAddr", addr).Msg("cmd/pomerium: starting prometheus endpoint")
log.Error().Err(metrics.NewPromHTTPListener(addr)).Str("MetricsAddr", addr).Msg("cmd/pomerium: could not start metrics exporter")
}
func wrapMiddleware(o config.Options, mux *http.ServeMux) http.Handler {
c := middleware.NewChain()
c = c.Append(metrics.HTTPMetricsHandler("proxy"))
c = c.Append(log.NewHandler(log.Logger))
c = c.Append(log.AccessHandler(func(r *http.Request, status, size int, duration time.Duration) {
log.FromRequest(r).Debug().

View file

@ -162,6 +162,27 @@ Otherwise the proxy responds with `400 Bad Request` to all websocket connections
**Use with caution:** By definition, websockets are long-lived connections, so [global timeouts](#global-timeouts) are not enforced.
Allowing websocket connections to the proxy could result in abuse via DOS attacks.
### Metrics Address
- Environmental Variable: `METRICS_ADDRESS`
- Config File Key: `metrics_address`
- Type: `string`
- Example: `:8080`, `127.0.0.1:9090`, ``
- Default: `disabled`
- Optional
Expose a prometheus format HTTP endpoint on the specified port. Disabled by default.
**Use with caution:** the endpoint can expose frontend and backend server names or addresses. Do not expose the metrics port publicly.
#### Metrics tracked
| Name | Type | Description |
|:------------- |:-------------|:-----|
|http_server_requests_total| Counter | Total HTTP server requests handled by service|
|http_server_response_size_bytes| Histogram | HTTP server response size by service|
|http_server_request_duration_ms| Histogram | HTTP server request duration by service\
### Policy
- Environmental Variable: `POLICY`

4
go.mod
View file

@ -4,6 +4,7 @@ go 1.12
require (
cloud.google.com/go v0.40.0 // indirect
contrib.go.opencensus.io/exporter/prometheus v0.1.0
github.com/fsnotify/fsnotify v1.4.7
github.com/golang/mock v1.3.1
github.com/golang/protobuf v1.3.1
@ -13,12 +14,13 @@ require (
github.com/pelletier/go-toml v1.4.0 // indirect
github.com/pomerium/go-oidc v2.0.0+incompatible
github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
github.com/prometheus/client_golang v0.9.3
github.com/rs/zerolog v1.14.3
github.com/spf13/afero v1.2.2 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/viper v1.4.0
github.com/stretchr/testify v1.3.0 // indirect
go.opencensus.io v0.22.0 // indirect
go.opencensus.io v0.22.0
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8
golang.org/x/net v0.0.0-20190611141213-3f473d35a33a
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45

12
go.sum
View file

@ -4,6 +4,8 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
cloud.google.com/go v0.40.0 h1:FjSY7bOj+WzJe6TZRVtXI2b9kAYvtNg4lMbcH2+MUkk=
cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlro=
contrib.go.opencensus.io/exporter/prometheus v0.1.0 h1:SByaIoWwNgMdPSgl5sMqM2KDE5H/ukPWBRo314xiDvg=
contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
@ -11,6 +13,7 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@ -78,6 +81,7 @@ github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDe
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9dGS02Q3Y=
github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ=
@ -98,12 +102,19 @@ github.com/pomerium/go-oidc v2.0.0+incompatible/go.mod h1:DRsGVw6MOgxbfq4Y57jKOE
github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 h1:J9b7z+QKAmPf4YLrFg6oQUotqHQeUNWwkvo7jZp1GLU=
github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8=
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
@ -159,6 +170,7 @@ golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHl
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

View file

@ -133,6 +133,9 @@ type Options struct {
// Enable proxying of websocket connections. Defaults to "false".
// Caution: Enabling this feature could result in abuse via DOS attacks.
AllowWebsockets bool `mapstructure:"allow_websockets"`
// Address/Port to bind to for prometheus metrics
MetricsAddr string `mapstructure:"metrics_address"`
}
// NewOptions returns a new options struct with default values

View file

@ -0,0 +1,31 @@
package metrics
import (
"net/http"
ocProm "contrib.go.opencensus.io/exporter/prometheus"
prom "github.com/prometheus/client_golang/prometheus"
"go.opencensus.io/stats/view"
)
//NewPromHTTPListener creates a prometheus exporter on ListenAddr
func NewPromHTTPListener(ListenAddr string) error {
return http.ListenAndServe(ListenAddr, newPromHTTPHandler())
}
// newPromHTTPHandler creates a new prometheus exporter handler for /metrics
func newPromHTTPHandler() http.Handler {
// TODO this is a cheap way to get thorough go process
// stats. It will not work with additional exporters.
// It should turn into an FR to the OC framework
var reg *prom.Registry
reg = prom.DefaultRegisterer.(*prom.Registry)
pe, _ := ocProm.NewExporter(ocProm.Options{
Namespace: "pomerium",
Registry: reg,
})
view.RegisterExporter(pe)
mux := http.NewServeMux()
mux.Handle("/metrics", pe)
return mux
}

View file

@ -0,0 +1,27 @@
package metrics
import (
"bytes"
"io/ioutil"
"net/http/httptest"
"regexp"
"testing"
)
func Test_newPromHTTPHandler(t *testing.T) {
h := newPromHTTPHandler()
req := httptest.NewRequest("GET", "http://test.local/metrics", new(bytes.Buffer))
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
resp := rec.Result()
b, _ := ioutil.ReadAll(resp.Body)
if resp == nil || resp.StatusCode != 200 {
t.Errorf("Metrics endpoint failed to respond: %s", b)
}
if m, _ := regexp.Match("^# HELP .*", b); !m {
t.Errorf("Metrics endpoint did not contain any help messages: %s", b)
}
}

View file

@ -0,0 +1,90 @@
package metrics
import (
"context"
"net/http"
"strconv"
"time"
"github.com/pomerium/pomerium/internal/middleware/responsewriter"
"go.opencensus.io/tag"
"go.opencensus.io/stats/view"
"go.opencensus.io/stats"
)
var (
keyMethod, _ = tag.NewKey("method")
keyStatus, _ = tag.NewKey("status")
keyService, _ = tag.NewKey("service")
keyHost, _ = tag.NewKey("host")
httpRequestCount = stats.Int64("http_server_requests_total", "Total HTTP Requests", "1")
httpResponseSize = stats.Int64("http_server_response_size_bytes", "HTTP Server Response Size in bytes", "bytes")
httpRequestDuration = stats.Int64("http_server_request_duration_ms", "HTTP Request duration in ms", "ms")
views = []*view.View{
&view.View{
Name: httpRequestCount.Name(),
Measure: httpRequestCount,
Description: httpRequestCount.Description(),
TagKeys: []tag.Key{keyService, keyHost, keyMethod, keyStatus},
Aggregation: view.Count(),
},
&view.View{
Name: httpRequestDuration.Name(),
Measure: httpRequestDuration,
Description: httpRequestDuration.Description(),
TagKeys: []tag.Key{keyService, keyHost, keyMethod, keyStatus},
Aggregation: view.Distribution(
1, 2, 5, 7, 10, 25, 500, 750,
100, 250, 500, 750,
1000, 2500, 5000, 7500,
10000, 25000, 50000, 75000,
100000,
),
},
&view.View{
Name: httpResponseSize.Name(),
Measure: httpResponseSize,
Description: httpResponseSize.Description(),
TagKeys: []tag.Key{keyService, keyHost, keyMethod, keyStatus},
Aggregation: view.Distribution(
1, 256, 512, 1024, 2048, 8192, 16384, 32768, 65536, 131072, 262144, 524288,
1048576, 2097152, 4194304, 8388608,
),
},
}
)
func init() {
view.Register(views...)
}
// HTTPMetricsHandler creates a metrics middleware for incoming HTTP requests
func HTTPMetricsHandler(service string) func(next http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
startTime := time.Now()
m := responsewriter.NewWrapResponseWriter(w, 1)
next.ServeHTTP(m, r)
ctx, _ := tag.New(
context.Background(),
tag.Insert(keyService, service),
tag.Insert(keyHost, r.Host),
tag.Insert(keyMethod, r.Method),
tag.Insert(keyStatus, strconv.Itoa(m.Status())),
)
stats.Record(ctx,
httpRequestCount.M(1),
httpRequestDuration.M(time.Since(startTime).Nanoseconds()/int64(time.Millisecond)),
httpResponseSize.M(int64(m.BytesWritten())),
)
})
}
}

View file

@ -0,0 +1,110 @@
package metrics
import (
"bytes"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/pomerium/pomerium/internal/middleware"
"go.opencensus.io/stats/view"
)
type measure struct {
Name string
Tags map[string]string
Measure int
}
func newTestMux() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/good", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "Hello")
})
return mux
}
func Test_HTTPMetricsHandler(t *testing.T) {
chain := middleware.NewChain()
chain = chain.Append(HTTPMetricsHandler("test_service"))
chainHandler := chain.Then(newTestMux())
tests := []struct {
name string
url string
verb string
wanthttpResponseSize string
wanthttpRequestDuration string
wanthttpRequestCount string
}{
{
name: "good get",
url: "http://test.local/good",
verb: "GET",
wanthttpResponseSize: "{ { {host test.local}{method GET}{service test_service}{status 200} }&{1 5 5 5 0 [0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]",
wanthttpRequestDuration: "{ { {host test.local}{method GET}{service test_service}{status 200} }&{1",
wanthttpRequestCount: "{ { {host test.local}{method GET}{service test_service}{status 200} }&{1",
},
{
name: "good post",
url: "http://test.local/good",
verb: "POST",
wanthttpResponseSize: "{ { {host test.local}{method POST}{service test_service}{status 200} }&{1 5 5 5 0 [0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]",
wanthttpRequestDuration: "{ { {host test.local}{method POST}{service test_service}{status 200} }&{1",
wanthttpRequestCount: "{ { {host test.local}{method POST}{service test_service}{status 200} }&{1",
},
{
name: "bad post",
url: "http://test.local/bad",
verb: "POST",
wanthttpResponseSize: "{ { {host test.local}{method POST}{service test_service}{status 404} }&{1 19 19 19 0 [0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]",
wanthttpRequestDuration: "{ { {host test.local}{method POST}{service test_service}{status 404} }&{1",
wanthttpRequestCount: "{ { {host test.local}{method POST}{service test_service}{status 404} }&{1",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
view.Unregister(views...)
view.Register(views...)
req := httptest.NewRequest(tt.verb, tt.url, new(bytes.Buffer))
rec := httptest.NewRecorder()
chainHandler.ServeHTTP(rec, req)
// httpResponseSize
data, _ := view.RetrieveData(httpResponseSize.Name())
if len(data) != 1 {
t.Errorf("httpResponseSize: received wrong number of data rows: %d", len(data))
return
}
if !strings.HasPrefix(data[0].String(), tt.wanthttpResponseSize) {
t.Errorf("httpResponseSize: Found unexpected data row: \nwant: %s\ngot: %s\n", tt.wanthttpResponseSize, data[0].String())
}
// httpResponseSize
data, _ = view.RetrieveData(httpRequestDuration.Name())
if len(data) != 1 {
t.Errorf("httpRequestDuration: received too many data rows: %d", len(data))
}
if !strings.HasPrefix(data[0].String(), tt.wanthttpRequestDuration) {
t.Errorf("httpRequestDuration: Found unexpected data row: \nwant: %s\ngot: %s\n", tt.wanthttpRequestDuration, data[0].String())
}
// httpRequestCount
data, _ = view.RetrieveData(httpRequestCount.Name())
if len(data) != 1 {
t.Errorf("httpRequestCount: received too many data rows: %d", len(data))
}
if !strings.HasPrefix(data[0].String(), tt.wanthttpRequestCount) {
t.Errorf("httpRequestCount: Found unexpected data row: \nwant: %s\ngot: %s\n", tt.wanthttpRequestCount, data[0].String())
}
})
}
}