Add metrics implementation* Covers proxy service on server side* Update documentation

internal/config/options.go

@@ -133,6 +133,9 @@ type Options struct {
 	// Enable proxying of websocket connections. Defaults to "false".
 	// Caution: Enabling this feature could result in abuse via DOS attacks.
 	AllowWebsockets bool `mapstructure:"allow_websockets"`
+
+	// Address/Port to bind to for prometheus metrics
+	MetricsAddr string `mapstructure:"metrics_address"`
 }
 
 // NewOptions returns a new options struct with default values

internal/metrics/exporter.go

@@ -0,0 +1,31 @@
+package metrics
+
+import (
+	"net/http"
+
+	ocProm "contrib.go.opencensus.io/exporter/prometheus"
+	prom "github.com/prometheus/client_golang/prometheus"
+	"go.opencensus.io/stats/view"
+)
+
+//NewPromHTTPListener creates a prometheus exporter on ListenAddr
+func NewPromHTTPListener(ListenAddr string) error {
+	return http.ListenAndServe(ListenAddr, newPromHTTPHandler())
+}
+
+// newPromHTTPHandler creates a new prometheus exporter handler for /metrics
+func newPromHTTPHandler() http.Handler {
+	// TODO this is a cheap way to get thorough go process
+	// stats.  It will not work with additional exporters.
+	// It should turn into an FR to the OC framework
+	var reg *prom.Registry
+	reg = prom.DefaultRegisterer.(*prom.Registry)
+	pe, _ := ocProm.NewExporter(ocProm.Options{
+		Namespace: "pomerium",
+		Registry:  reg,
+	})
+	view.RegisterExporter(pe)
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", pe)
+	return mux
+}

internal/metrics/exporter_test.go

@@ -0,0 +1,27 @@
+package metrics
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http/httptest"
+	"regexp"
+	"testing"
+)
+
+func Test_newPromHTTPHandler(t *testing.T) {
+	h := newPromHTTPHandler()
+
+	req := httptest.NewRequest("GET", "http://test.local/metrics", new(bytes.Buffer))
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, req)
+	resp := rec.Result()
+	b, _ := ioutil.ReadAll(resp.Body)
+
+	if resp == nil || resp.StatusCode != 200 {
+		t.Errorf("Metrics endpoint failed to respond: %s", b)
+	}
+
+	if m, _ := regexp.Match("^# HELP .*", b); !m {
+		t.Errorf("Metrics endpoint did not contain any help messages: %s", b)
+	}
+}

internal/metrics/middleware.go

@@ -0,0 +1,90 @@
+package metrics
+
+import (
+	"context"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/pomerium/pomerium/internal/middleware/responsewriter"
+
+	"go.opencensus.io/tag"
+
+	"go.opencensus.io/stats/view"
+
+	"go.opencensus.io/stats"
+)
+
+var (
+	keyMethod, _  = tag.NewKey("method")
+	keyStatus, _  = tag.NewKey("status")
+	keyService, _ = tag.NewKey("service")
+	keyHost, _    = tag.NewKey("host")
+
+	httpRequestCount    = stats.Int64("http_server_requests_total", "Total HTTP Requests", "1")
+	httpResponseSize    = stats.Int64("http_server_response_size_bytes", "HTTP Server Response Size in bytes", "bytes")
+	httpRequestDuration = stats.Int64("http_server_request_duration_ms", "HTTP Request duration in ms", "ms")
+
+	views = []*view.View{
+		&view.View{
+			Name:        httpRequestCount.Name(),
+			Measure:     httpRequestCount,
+			Description: httpRequestCount.Description(),
+			TagKeys:     []tag.Key{keyService, keyHost, keyMethod, keyStatus},
+			Aggregation: view.Count(),
+		},
+		&view.View{
+			Name:        httpRequestDuration.Name(),
+			Measure:     httpRequestDuration,
+			Description: httpRequestDuration.Description(),
+			TagKeys:     []tag.Key{keyService, keyHost, keyMethod, keyStatus},
+			Aggregation: view.Distribution(
+				1, 2, 5, 7, 10, 25, 500, 750,
+				100, 250, 500, 750,
+				1000, 2500, 5000, 7500,
+				10000, 25000, 50000, 75000,
+				100000,
+			),
+		},
+		&view.View{
+			Name:        httpResponseSize.Name(),
+			Measure:     httpResponseSize,
+			Description: httpResponseSize.Description(),
+			TagKeys:     []tag.Key{keyService, keyHost, keyMethod, keyStatus},
+			Aggregation: view.Distribution(
+				1, 256, 512, 1024, 2048, 8192, 16384, 32768, 65536, 131072, 262144, 524288,
+				1048576, 2097152, 4194304, 8388608,
+			),
+		},
+	}
+)
+
+func init() {
+	view.Register(views...)
+}
+
+// HTTPMetricsHandler creates a metrics middleware for incoming HTTP requests
+func HTTPMetricsHandler(service string) func(next http.Handler) http.Handler {
+
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			startTime := time.Now()
+			m := responsewriter.NewWrapResponseWriter(w, 1)
+
+			next.ServeHTTP(m, r)
+
+			ctx, _ := tag.New(
+				context.Background(),
+				tag.Insert(keyService, service),
+				tag.Insert(keyHost, r.Host),
+				tag.Insert(keyMethod, r.Method),
+				tag.Insert(keyStatus, strconv.Itoa(m.Status())),
+			)
+			stats.Record(ctx,
+				httpRequestCount.M(1),
+				httpRequestDuration.M(time.Since(startTime).Nanoseconds()/int64(time.Millisecond)),
+				httpResponseSize.M(int64(m.BytesWritten())),
+			)
+		})
+	}
+}

internal/metrics/middleware_test.go

@@ -0,0 +1,110 @@
+package metrics
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/pomerium/pomerium/internal/middleware"
+	"go.opencensus.io/stats/view"
+)
+
+type measure struct {
+	Name    string
+	Tags    map[string]string
+	Measure int
+}
+
+func newTestMux() http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/good", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "Hello")
+	})
+
+	return mux
+}
+
+func Test_HTTPMetricsHandler(t *testing.T) {
+
+	chain := middleware.NewChain()
+	chain = chain.Append(HTTPMetricsHandler("test_service"))
+	chainHandler := chain.Then(newTestMux())
+
+	tests := []struct {
+		name                    string
+		url                     string
+		verb                    string
+		wanthttpResponseSize    string
+		wanthttpRequestDuration string
+		wanthttpRequestCount    string
+	}{
+		{
+			name:                    "good get",
+			url:                     "http://test.local/good",
+			verb:                    "GET",
+			wanthttpResponseSize:    "{ { {host test.local}{method GET}{service test_service}{status 200} }&{1 5 5 5 0 [0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]",
+			wanthttpRequestDuration: "{ { {host test.local}{method GET}{service test_service}{status 200} }&{1",
+			wanthttpRequestCount:    "{ { {host test.local}{method GET}{service test_service}{status 200} }&{1",
+		},
+		{
+			name:                    "good post",
+			url:                     "http://test.local/good",
+			verb:                    "POST",
+			wanthttpResponseSize:    "{ { {host test.local}{method POST}{service test_service}{status 200} }&{1 5 5 5 0 [0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]",
+			wanthttpRequestDuration: "{ { {host test.local}{method POST}{service test_service}{status 200} }&{1",
+			wanthttpRequestCount:    "{ { {host test.local}{method POST}{service test_service}{status 200} }&{1",
+		},
+		{
+			name:                    "bad post",
+			url:                     "http://test.local/bad",
+			verb:                    "POST",
+			wanthttpResponseSize:    "{ { {host test.local}{method POST}{service test_service}{status 404} }&{1 19 19 19 0 [0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]",
+			wanthttpRequestDuration: "{ { {host test.local}{method POST}{service test_service}{status 404} }&{1",
+			wanthttpRequestCount:    "{ { {host test.local}{method POST}{service test_service}{status 404} }&{1",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			view.Unregister(views...)
+			view.Register(views...)
+
+			req := httptest.NewRequest(tt.verb, tt.url, new(bytes.Buffer))
+			rec := httptest.NewRecorder()
+			chainHandler.ServeHTTP(rec, req)
+
+			// httpResponseSize
+			data, _ := view.RetrieveData(httpResponseSize.Name())
+			if len(data) != 1 {
+				t.Errorf("httpResponseSize: received wrong number of data rows: %d", len(data))
+				return
+			}
+
+			if !strings.HasPrefix(data[0].String(), tt.wanthttpResponseSize) {
+				t.Errorf("httpResponseSize: Found unexpected data row: \nwant: %s\ngot: %s\n", tt.wanthttpResponseSize, data[0].String())
+			}
+
+			// httpResponseSize
+			data, _ = view.RetrieveData(httpRequestDuration.Name())
+			if len(data) != 1 {
+				t.Errorf("httpRequestDuration: received too many data rows: %d", len(data))
+			}
+
+			if !strings.HasPrefix(data[0].String(), tt.wanthttpRequestDuration) {
+				t.Errorf("httpRequestDuration: Found unexpected data row: \nwant: %s\ngot: %s\n", tt.wanthttpRequestDuration, data[0].String())
+			}
+
+			// httpRequestCount
+			data, _ = view.RetrieveData(httpRequestCount.Name())
+			if len(data) != 1 {
+				t.Errorf("httpRequestCount: received too many data rows: %d", len(data))
+			}
+
+			if !strings.HasPrefix(data[0].String(), tt.wanthttpRequestCount) {
+				t.Errorf("httpRequestCount: Found unexpected data row: \nwant: %s\ngot: %s\n", tt.wanthttpRequestCount, data[0].String())
+			}
+		})
+	}
+}