From ff528e8c7b8d84652a3b0de690f8b845c5b10c3f Mon Sep 17 00:00:00 2001 From: Travis Groth Date: Mon, 10 Jun 2019 21:17:32 -0400 Subject: [PATCH] Add metrics implementation* Covers proxy service on server side* Update documentation --- CHANGELOG.md | 1 + cmd/pomerium/main.go | 12 +++ docs/reference/readme.md | 21 ++++++ go.mod | 4 +- go.sum | 12 +++ internal/config/options.go | 3 + internal/metrics/exporter.go | 31 ++++++++ internal/metrics/exporter_test.go | 27 +++++++ internal/metrics/middleware.go | 90 +++++++++++++++++++++++ internal/metrics/middleware_test.go | 110 ++++++++++++++++++++++++++++ 10 files changed, 310 insertions(+), 1 deletion(-) create mode 100644 internal/metrics/exporter.go create mode 100644 internal/metrics/exporter_test.go create mode 100644 internal/metrics/middleware.go create mode 100644 internal/metrics/middleware_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 995589d4c..a9085de43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### NEW - Add programmatic authentication support. [GH-177] +- Add Prometheus format metrics endpoint. [GH-35] ### CHANGED diff --git a/cmd/pomerium/main.go b/cmd/pomerium/main.go index c56816ac1..51d23c2bb 100644 --- a/cmd/pomerium/main.go +++ b/cmd/pomerium/main.go @@ -8,6 +8,8 @@ import ( "os" "time" + "github.com/pomerium/pomerium/internal/metrics" + "github.com/fsnotify/fsnotify" "github.com/pomerium/pomerium/authenticate" "github.com/pomerium/pomerium/authorize" @@ -83,6 +85,10 @@ func main() { IdleTimeout: opt.IdleTimeout, } + if opt.MetricsAddr != "" { + go newPromListener(opt.MetricsAddr) + } + if srv, err := startRedirectServer(opt.HTTPRedirectAddr); err != nil { log.Debug().Err(err).Msg("cmd/pomerium: http redirect server not started") } else { @@ -151,8 +157,14 @@ func newProxyService(opt config.Options, mux *http.ServeMux) (*proxy.Proxy, erro return service, nil } +func newPromListener(addr string) { + log.Info().Str("MetricsAddr", addr).Msg("cmd/pomerium: starting prometheus endpoint") + log.Error().Err(metrics.NewPromHTTPListener(addr)).Str("MetricsAddr", addr).Msg("cmd/pomerium: could not start metrics exporter") +} + func wrapMiddleware(o config.Options, mux *http.ServeMux) http.Handler { c := middleware.NewChain() + c = c.Append(metrics.HTTPMetricsHandler("proxy")) c = c.Append(log.NewHandler(log.Logger)) c = c.Append(log.AccessHandler(func(r *http.Request, status, size int, duration time.Duration) { log.FromRequest(r).Debug(). diff --git a/docs/reference/readme.md b/docs/reference/readme.md index 4e261c6f9..ee75bf1c2 100644 --- a/docs/reference/readme.md +++ b/docs/reference/readme.md @@ -162,6 +162,27 @@ Otherwise the proxy responds with `400 Bad Request` to all websocket connections **Use with caution:** By definition, websockets are long-lived connections, so [global timeouts](#global-timeouts) are not enforced. Allowing websocket connections to the proxy could result in abuse via DOS attacks. +### Metrics Address + +- Environmental Variable: `METRICS_ADDRESS` +- Config File Key: `metrics_address` +- Type: `string` +- Example: `:8080`, `127.0.0.1:9090`, `` +- Default: `disabled` +- Optional + +Expose a prometheus format HTTP endpoint on the specified port. Disabled by default. + +**Use with caution:** the endpoint can expose frontend and backend server names or addresses. Do not expose the metrics port publicly. + +#### Metrics tracked + +| Name | Type | Description | +|:------------- |:-------------|:-----| +|http_server_requests_total| Counter | Total HTTP server requests handled by service| +|http_server_response_size_bytes| Histogram | HTTP server response size by service| +|http_server_request_duration_ms| Histogram | HTTP server request duration by service\ + ### Policy - Environmental Variable: `POLICY` diff --git a/go.mod b/go.mod index 461ae3780..2918a1757 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.12 require ( cloud.google.com/go v0.40.0 // indirect + contrib.go.opencensus.io/exporter/prometheus v0.1.0 github.com/fsnotify/fsnotify v1.4.7 github.com/golang/mock v1.3.1 github.com/golang/protobuf v1.3.1 @@ -13,12 +14,13 @@ require ( github.com/pelletier/go-toml v1.4.0 // indirect github.com/pomerium/go-oidc v2.0.0+incompatible github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect + github.com/prometheus/client_golang v0.9.3 github.com/rs/zerolog v1.14.3 github.com/spf13/afero v1.2.2 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.4.0 github.com/stretchr/testify v1.3.0 // indirect - go.opencensus.io v0.22.0 // indirect + go.opencensus.io v0.22.0 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 golang.org/x/net v0.0.0-20190611141213-3f473d35a33a golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 diff --git a/go.sum b/go.sum index 6dcef31a4..15a8ce1d2 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.40.0 h1:FjSY7bOj+WzJe6TZRVtXI2b9kAYvtNg4lMbcH2+MUkk= cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlro= +contrib.go.opencensus.io/exporter/prometheus v0.1.0 h1:SByaIoWwNgMdPSgl5sMqM2KDE5H/ukPWBRo314xiDvg= +contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -11,6 +13,7 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -78,6 +81,7 @@ github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDe github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9dGS02Q3Y= github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= @@ -98,12 +102,19 @@ github.com/pomerium/go-oidc v2.0.0+incompatible/go.mod h1:DRsGVw6MOgxbfq4Y57jKOE github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 h1:J9b7z+QKAmPf4YLrFg6oQUotqHQeUNWwkvo7jZp1GLU= github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -159,6 +170,7 @@ golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/internal/config/options.go b/internal/config/options.go index 923671933..75e6bb509 100644 --- a/internal/config/options.go +++ b/internal/config/options.go @@ -133,6 +133,9 @@ type Options struct { // Enable proxying of websocket connections. Defaults to "false". // Caution: Enabling this feature could result in abuse via DOS attacks. AllowWebsockets bool `mapstructure:"allow_websockets"` + + // Address/Port to bind to for prometheus metrics + MetricsAddr string `mapstructure:"metrics_address"` } // NewOptions returns a new options struct with default values diff --git a/internal/metrics/exporter.go b/internal/metrics/exporter.go new file mode 100644 index 000000000..09b9f18c8 --- /dev/null +++ b/internal/metrics/exporter.go @@ -0,0 +1,31 @@ +package metrics + +import ( + "net/http" + + ocProm "contrib.go.opencensus.io/exporter/prometheus" + prom "github.com/prometheus/client_golang/prometheus" + "go.opencensus.io/stats/view" +) + +//NewPromHTTPListener creates a prometheus exporter on ListenAddr +func NewPromHTTPListener(ListenAddr string) error { + return http.ListenAndServe(ListenAddr, newPromHTTPHandler()) +} + +// newPromHTTPHandler creates a new prometheus exporter handler for /metrics +func newPromHTTPHandler() http.Handler { + // TODO this is a cheap way to get thorough go process + // stats. It will not work with additional exporters. + // It should turn into an FR to the OC framework + var reg *prom.Registry + reg = prom.DefaultRegisterer.(*prom.Registry) + pe, _ := ocProm.NewExporter(ocProm.Options{ + Namespace: "pomerium", + Registry: reg, + }) + view.RegisterExporter(pe) + mux := http.NewServeMux() + mux.Handle("/metrics", pe) + return mux +} diff --git a/internal/metrics/exporter_test.go b/internal/metrics/exporter_test.go new file mode 100644 index 000000000..6738dcd43 --- /dev/null +++ b/internal/metrics/exporter_test.go @@ -0,0 +1,27 @@ +package metrics + +import ( + "bytes" + "io/ioutil" + "net/http/httptest" + "regexp" + "testing" +) + +func Test_newPromHTTPHandler(t *testing.T) { + h := newPromHTTPHandler() + + req := httptest.NewRequest("GET", "http://test.local/metrics", new(bytes.Buffer)) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + resp := rec.Result() + b, _ := ioutil.ReadAll(resp.Body) + + if resp == nil || resp.StatusCode != 200 { + t.Errorf("Metrics endpoint failed to respond: %s", b) + } + + if m, _ := regexp.Match("^# HELP .*", b); !m { + t.Errorf("Metrics endpoint did not contain any help messages: %s", b) + } +} diff --git a/internal/metrics/middleware.go b/internal/metrics/middleware.go new file mode 100644 index 000000000..c9546b6d7 --- /dev/null +++ b/internal/metrics/middleware.go @@ -0,0 +1,90 @@ +package metrics + +import ( + "context" + "net/http" + "strconv" + "time" + + "github.com/pomerium/pomerium/internal/middleware/responsewriter" + + "go.opencensus.io/tag" + + "go.opencensus.io/stats/view" + + "go.opencensus.io/stats" +) + +var ( + keyMethod, _ = tag.NewKey("method") + keyStatus, _ = tag.NewKey("status") + keyService, _ = tag.NewKey("service") + keyHost, _ = tag.NewKey("host") + + httpRequestCount = stats.Int64("http_server_requests_total", "Total HTTP Requests", "1") + httpResponseSize = stats.Int64("http_server_response_size_bytes", "HTTP Server Response Size in bytes", "bytes") + httpRequestDuration = stats.Int64("http_server_request_duration_ms", "HTTP Request duration in ms", "ms") + + views = []*view.View{ + &view.View{ + Name: httpRequestCount.Name(), + Measure: httpRequestCount, + Description: httpRequestCount.Description(), + TagKeys: []tag.Key{keyService, keyHost, keyMethod, keyStatus}, + Aggregation: view.Count(), + }, + &view.View{ + Name: httpRequestDuration.Name(), + Measure: httpRequestDuration, + Description: httpRequestDuration.Description(), + TagKeys: []tag.Key{keyService, keyHost, keyMethod, keyStatus}, + Aggregation: view.Distribution( + 1, 2, 5, 7, 10, 25, 500, 750, + 100, 250, 500, 750, + 1000, 2500, 5000, 7500, + 10000, 25000, 50000, 75000, + 100000, + ), + }, + &view.View{ + Name: httpResponseSize.Name(), + Measure: httpResponseSize, + Description: httpResponseSize.Description(), + TagKeys: []tag.Key{keyService, keyHost, keyMethod, keyStatus}, + Aggregation: view.Distribution( + 1, 256, 512, 1024, 2048, 8192, 16384, 32768, 65536, 131072, 262144, 524288, + 1048576, 2097152, 4194304, 8388608, + ), + }, + } +) + +func init() { + view.Register(views...) +} + +// HTTPMetricsHandler creates a metrics middleware for incoming HTTP requests +func HTTPMetricsHandler(service string) func(next http.Handler) http.Handler { + + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + startTime := time.Now() + m := responsewriter.NewWrapResponseWriter(w, 1) + + next.ServeHTTP(m, r) + + ctx, _ := tag.New( + context.Background(), + tag.Insert(keyService, service), + tag.Insert(keyHost, r.Host), + tag.Insert(keyMethod, r.Method), + tag.Insert(keyStatus, strconv.Itoa(m.Status())), + ) + stats.Record(ctx, + httpRequestCount.M(1), + httpRequestDuration.M(time.Since(startTime).Nanoseconds()/int64(time.Millisecond)), + httpResponseSize.M(int64(m.BytesWritten())), + ) + }) + } +} diff --git a/internal/metrics/middleware_test.go b/internal/metrics/middleware_test.go new file mode 100644 index 000000000..83c25d952 --- /dev/null +++ b/internal/metrics/middleware_test.go @@ -0,0 +1,110 @@ +package metrics + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/pomerium/pomerium/internal/middleware" + "go.opencensus.io/stats/view" +) + +type measure struct { + Name string + Tags map[string]string + Measure int +} + +func newTestMux() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/good", func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "Hello") + }) + + return mux +} + +func Test_HTTPMetricsHandler(t *testing.T) { + + chain := middleware.NewChain() + chain = chain.Append(HTTPMetricsHandler("test_service")) + chainHandler := chain.Then(newTestMux()) + + tests := []struct { + name string + url string + verb string + wanthttpResponseSize string + wanthttpRequestDuration string + wanthttpRequestCount string + }{ + { + name: "good get", + url: "http://test.local/good", + verb: "GET", + wanthttpResponseSize: "{ { {host test.local}{method GET}{service test_service}{status 200} }&{1 5 5 5 0 [0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]", + wanthttpRequestDuration: "{ { {host test.local}{method GET}{service test_service}{status 200} }&{1", + wanthttpRequestCount: "{ { {host test.local}{method GET}{service test_service}{status 200} }&{1", + }, + { + name: "good post", + url: "http://test.local/good", + verb: "POST", + wanthttpResponseSize: "{ { {host test.local}{method POST}{service test_service}{status 200} }&{1 5 5 5 0 [0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]", + wanthttpRequestDuration: "{ { {host test.local}{method POST}{service test_service}{status 200} }&{1", + wanthttpRequestCount: "{ { {host test.local}{method POST}{service test_service}{status 200} }&{1", + }, + { + name: "bad post", + url: "http://test.local/bad", + verb: "POST", + wanthttpResponseSize: "{ { {host test.local}{method POST}{service test_service}{status 404} }&{1 19 19 19 0 [0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]", + wanthttpRequestDuration: "{ { {host test.local}{method POST}{service test_service}{status 404} }&{1", + wanthttpRequestCount: "{ { {host test.local}{method POST}{service test_service}{status 404} }&{1", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + view.Unregister(views...) + view.Register(views...) + + req := httptest.NewRequest(tt.verb, tt.url, new(bytes.Buffer)) + rec := httptest.NewRecorder() + chainHandler.ServeHTTP(rec, req) + + // httpResponseSize + data, _ := view.RetrieveData(httpResponseSize.Name()) + if len(data) != 1 { + t.Errorf("httpResponseSize: received wrong number of data rows: %d", len(data)) + return + } + + if !strings.HasPrefix(data[0].String(), tt.wanthttpResponseSize) { + t.Errorf("httpResponseSize: Found unexpected data row: \nwant: %s\ngot: %s\n", tt.wanthttpResponseSize, data[0].String()) + } + + // httpResponseSize + data, _ = view.RetrieveData(httpRequestDuration.Name()) + if len(data) != 1 { + t.Errorf("httpRequestDuration: received too many data rows: %d", len(data)) + } + + if !strings.HasPrefix(data[0].String(), tt.wanthttpRequestDuration) { + t.Errorf("httpRequestDuration: Found unexpected data row: \nwant: %s\ngot: %s\n", tt.wanthttpRequestDuration, data[0].String()) + } + + // httpRequestCount + data, _ = view.RetrieveData(httpRequestCount.Name()) + if len(data) != 1 { + t.Errorf("httpRequestCount: received too many data rows: %d", len(data)) + } + + if !strings.HasPrefix(data[0].String(), tt.wanthttpRequestCount) { + t.Errorf("httpRequestCount: Found unexpected data row: \nwant: %s\ngot: %s\n", tt.wanthttpRequestCount, data[0].String()) + } + }) + } +}