diff --git a/CHANGELOG.md b/CHANGELOG.md index f29f06909..69809a301 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Pomerium Changelog +## vUNRELEASED + +### New + + +### Changed + +- GRPC Metrics Implementation [GH-218] + - Additional GRPC server metrics and request sizes + - Improved GRPC metrics implementation internals + - The GRPC method label is now 'grpc_method' and GRPC status is now `grpc_client_status` and `grpc_server_status` + + ## v0.1.0 ### NEW diff --git a/cmd/pomerium/main.go b/cmd/pomerium/main.go index e5c811502..2d00cd168 100644 --- a/cmd/pomerium/main.go +++ b/cmd/pomerium/main.go @@ -41,7 +41,7 @@ func main() { } log.Info().Str("version", version.FullVersion()).Msg("cmd/pomerium") grpcAuth := middleware.NewSharedSecretCred(opt.SharedKey) - grpcOpts := []grpc.ServerOption{grpc.UnaryInterceptor(grpcAuth.ValidateRequest)} + grpcOpts := []grpc.ServerOption{grpc.UnaryInterceptor(grpcAuth.ValidateRequest), grpc.StatsHandler(metrics.NewGRPCServerStatsHandler(opt.Services))} grpcServer := grpc.NewServer(grpcOpts...) mux := http.NewServeMux() @@ -158,9 +158,10 @@ func newProxyService(opt config.Options, mux *http.ServeMux) (*proxy.Proxy, erro } func newPromListener(addr string) { - metrics.RegisterGRPCClientView() - metrics.RegisterHTTPClientView() - metrics.RegisterHTTPServerView() + metrics.RegisterView(metrics.HTTPClientViews) + metrics.RegisterView(metrics.HTTPServerViews) + metrics.RegisterView(metrics.GRPCClientViews) + metrics.RegisterView(metrics.GRPCServerViews) log.Info().Str("MetricsAddr", addr).Msg("cmd/pomerium: starting prometheus endpoint") log.Error().Err(metrics.NewPromHTTPListener(addr)).Str("MetricsAddr", addr).Msg("cmd/pomerium: could not start metrics exporter") diff --git a/docs/reference/readme.md b/docs/reference/readme.md index 344ee2666..20e9e803e 100644 --- a/docs/reference/readme.md +++ b/docs/reference/readme.md @@ -160,7 +160,7 @@ If set, the HTTP Redirect Address specifies the host and port to redirect http t Expose a prometheus format HTTP endpoint on the specified port. Disabled by default. -**Use with caution:** the endpoint can expose frontend and backend server names or addresses. Do not expose the metrics port publicly. +**Use with caution:** the endpoint can expose frontend and backend server names or addresses. Do not expose the metrics port if this is sensitive information. #### Metrics tracked @@ -174,7 +174,12 @@ http_client_response_size_bytes | Histogram | HTTP client response size by servi http_client_request_duration_ms | Histogram | HTTP client request duration by service grpc_client_requests_total | Counter | Total GRPC client requests made by service grpc_client_response_size_bytes | Histogram | GRPC client response size by service +grpc_client_request_size_bytes | Histogram | GRPC client request size by service grpc_client_request_duration_ms | Histogram | GRPC client request duration by service +grpc_server_requests_total | Counter | Total GRPC server requests made by service +grpc_server_response_size_bytes | Histogram | GRPC server response size by service +grpc_server_request_size_bytes | Histogram | GRPC server request size by service +grpc_server_request_duration_ms | Histogram | GRPC server request duration by service ### Policy diff --git a/internal/metrics/helpers_test.go b/internal/metrics/helpers_test.go index 04c06d1eb..7171da858 100644 --- a/internal/metrics/helpers_test.go +++ b/internal/metrics/helpers_test.go @@ -4,12 +4,14 @@ import ( "strings" "testing" - "go.opencensus.io/stats" "go.opencensus.io/stats/view" ) -func testDataRetrieval(measure stats.Measure, t *testing.T, want string) { - name := measure.Name() +func testDataRetrieval(v *view.View, t *testing.T, want string) { + if v == nil { + t.Fatalf("%s: nil view passed", t.Name()) + } + name := v.Name data, err := view.RetrieveData(name) if err != nil { diff --git a/internal/metrics/interceptors.go b/internal/metrics/interceptors.go index 04bf20ce1..0cb62557c 100644 --- a/internal/metrics/interceptors.go +++ b/internal/metrics/interceptors.go @@ -3,100 +3,100 @@ package metrics // import "github.com/pomerium/pomerium/internal/metrics" import ( "context" "strings" - "time" - "github.com/golang/protobuf/proto" "github.com/pomerium/pomerium/internal/log" - "go.opencensus.io/stats" + "go.opencensus.io/plugin/ocgrpc" "go.opencensus.io/stats/view" "go.opencensus.io/tag" "google.golang.org/grpc" - "google.golang.org/grpc/status" + grpcstats "google.golang.org/grpc/stats" ) var ( - grpcServerRequestCount = stats.Int64("grpc_server_requests_total", "Total grpc Requests", "1") - grpcServerResponseSize = stats.Int64("grpc_server_response_size_bytes", "grpc Server Response Size in bytes", "bytes") - grpcServerRequestDuration = stats.Int64("grpc_server_request_duration_ms", "grpc Request duration in ms", "ms") + grpcSizeDistribution = view.Distribution( + 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, + 2048, 4096, 8192, 16384, + ) + grcpLatencyDistribution = view.Distribution( + 1, 2, 5, 7, 10, 25, 50, 75, + 100, 250, 500, 750, 1000, + ) - grpcClientRequestCount = stats.Int64("grpc_client_requests_total", "Total grpc Client Requests", "1") - grpcClientResponseSize = stats.Int64("grpc_client_response_size_bytes", "grpc Client Response Size in bytes", "bytes") - grpcClientRequestDuration = stats.Int64("grpc_client_request_duration_ms", "grpc Client Request duration in ms", "ms") - - // GRPCServerRequestCountView is an OpenCensus view which tracks GRPC Server requests by pomerium service, host, grpc service, grpc method, and status + // GRPCServerRequestCountView is an OpenCensus view which counts GRPC Server requests by pomerium service, grpc service, grpc method, and status GRPCServerRequestCountView = &view.View{ - Name: grpcServerRequestCount.Name(), - Measure: grpcServerRequestCount, - Description: grpcServerRequestCount.Description(), - TagKeys: []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService}, + Name: "grpc_server_requests_total", + Measure: ocgrpc.ServerLatency, + Description: "Total grpc Requests", + TagKeys: []tag.Key{keyService, keyGRPCMethod, ocgrpc.KeyServerStatus, keyGRPCService}, Aggregation: view.Count(), } - // GRPCServerRequestDurationView is an OpenCensus view which tracks GRPC Server request duration by pomerium service, host, grpc service, grpc method, and statu + // GRPCServerRequestDurationView is an OpenCensus view which tracks GRPC Server request duration by pomerium service, grpc service, grpc method, and status GRPCServerRequestDurationView = &view.View{ - Name: grpcServerRequestDuration.Name(), - Measure: grpcServerRequestDuration, - Description: grpcServerRequestDuration.Description(), - TagKeys: []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService}, - Aggregation: view.Distribution( - 1, 2, 5, 7, 10, 25, 500, 750, - 100, 250, 500, 750, - 1000, 2500, 5000, 7500, - 10000, 25000, 50000, 75000, - 100000, - ), + Name: "grpc_server_request_duration_ms", + Measure: ocgrpc.ServerLatency, + Description: "grpc Request duration in ms", + TagKeys: []tag.Key{keyService, keyGRPCMethod, ocgrpc.KeyServerStatus, keyGRPCService}, + Aggregation: grcpLatencyDistribution, } - // GRPCServerResponseSizeView is an OpenCensus view which tracks GRPC Server request duration by pomerium service, host, grpc service, grpc method, and statu + // GRPCServerResponseSizeView is an OpenCensus view which tracks GRPC Server response size by pomerium service, grpc service, grpc method, and status GRPCServerResponseSizeView = &view.View{ - Name: grpcServerResponseSize.Name(), - Measure: grpcServerResponseSize, - Description: grpcServerResponseSize.Description(), - TagKeys: []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService}, - Aggregation: view.Distribution( - 1, 256, 512, 1024, 2048, 8192, 16384, 32768, 65536, 131072, 262144, 524288, - 1048576, 2097152, 4194304, 8388608, - ), + Name: "grpc_server_response_size_bytes", + Measure: ocgrpc.ServerSentBytesPerRPC, + Description: "grpc Server Response Size in bytes", + TagKeys: []tag.Key{keyService, keyGRPCMethod, ocgrpc.KeyServerStatus, keyGRPCService}, + Aggregation: grpcSizeDistribution, } - // GRPCClientRequestCountView is an OpenCensus view which tracks GRPC Client requests by pomerium service, target host, grpc service, grpc method, and statu + // GRPCServerRequestSizeView is an OpenCensus view which tracks GRPC Server request size by pomerium service, grpc service, grpc method, and status + GRPCServerRequestSizeView = &view.View{ + Name: "grpc_server_request_size_bytes", + Measure: ocgrpc.ServerReceivedBytesPerRPC, + Description: "grpc Server Request Size in bytes", + TagKeys: []tag.Key{keyService, keyGRPCMethod, ocgrpc.KeyServerStatus, keyGRPCService}, + Aggregation: grpcSizeDistribution, + } + + // GRPCClientRequestCountView is an OpenCensus view which tracks GRPC Client requests by pomerium service, target host, grpc service, grpc method, and status GRPCClientRequestCountView = &view.View{ - Name: grpcClientRequestCount.Name(), - Measure: grpcClientRequestCount, - Description: grpcClientRequestCount.Description(), - TagKeys: []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService}, + Name: "grpc_client_requests_total", + Measure: ocgrpc.ClientRoundtripLatency, + Description: "Total grpc Client Requests", + TagKeys: []tag.Key{keyService, keyHost, keyGRPCMethod, keyGRPCService, ocgrpc.KeyClientStatus}, Aggregation: view.Count(), } - // GRPCClientRequestDurationView is an OpenCensus view which tracks GRPC Client request duration by pomerium service, target host, grpc service, grpc method, and statu + // GRPCClientRequestDurationView is an OpenCensus view which tracks GRPC Client request duration by pomerium service, target host, grpc service, grpc method, and status GRPCClientRequestDurationView = &view.View{ - Name: grpcClientRequestDuration.Name(), - Measure: grpcClientRequestDuration, - Description: grpcClientRequestDuration.Description(), - TagKeys: []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService}, - Aggregation: view.Distribution( - 1, 2, 5, 7, 10, 25, 500, 750, - 100, 250, 500, 750, - 1000, 2500, 5000, 7500, - 10000, 25000, 50000, 75000, - 100000, - ), + Name: "grpc_client_request_duration_ms", + Measure: ocgrpc.ClientRoundtripLatency, + Description: "grpc Client Request duration in ms", + TagKeys: []tag.Key{keyService, keyHost, keyGRPCMethod, keyGRPCService, ocgrpc.KeyClientStatus}, + Aggregation: grcpLatencyDistribution, } - // GRPCClientResponseSizeView is an OpenCensus view which tracks GRPC Client response size by pomerium service, target host, grpc service, grpc method, and statu + // GRPCClientResponseSizeView is an OpenCensus view which tracks GRPC Client response size by pomerium service, target host, grpc service, grpc method, and status GRPCClientResponseSizeView = &view.View{ - Name: grpcClientResponseSize.Name(), - Measure: grpcClientResponseSize, - Description: grpcClientResponseSize.Description(), - TagKeys: []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService}, - Aggregation: view.Distribution( - 1, 256, 512, 1024, 2048, 8192, 16384, 32768, 65536, 131072, 262144, 524288, - 1048576, 2097152, 4194304, 8388608, - ), + Name: "grpc_client_response_size_bytes", + Measure: ocgrpc.ClientReceivedBytesPerRPC, + Description: "grpc Client Response Size in bytes", + TagKeys: []tag.Key{keyService, keyHost, keyGRPCMethod, keyGRPCService, ocgrpc.KeyClientStatus}, + Aggregation: grpcSizeDistribution, + } + + // GRPCClientRequestSizeView is an OpenCensus view which tracks GRPC Client request size by pomerium service, target host, grpc service, grpc method, and status + GRPCClientRequestSizeView = &view.View{ + Name: "grpc_client_request_size_bytes", + Measure: ocgrpc.ClientSentBytesPerRPC, + Description: "grpc Client Request Size in bytes", + TagKeys: []tag.Key{keyService, keyHost, keyGRPCMethod, keyGRPCService, ocgrpc.KeyClientStatus}, + Aggregation: grpcSizeDistribution, } ) -// GRPCClientInterceptor creates a UnaryClientInterceptor which tracks metrics of grpc client requests +// GRPCClientInterceptor creates a UnaryClientInterceptor which updates the RPC context with metric tag +// metadata func GRPCClientInterceptor(service string) grpc.UnaryClientInterceptor { return func( ctx context.Context, @@ -107,11 +107,6 @@ func GRPCClientInterceptor(service string) grpc.UnaryClientInterceptor { invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { - startTime := time.Now() - - // Calls the invoker to execute RPC - err := invoker(ctx, method, req, reply, cc, opts...) - // Split the method into parts for better slicing rpcInfo := strings.SplitN(method, "/", 3) var rpcMethod string @@ -121,30 +116,62 @@ func GRPCClientInterceptor(service string) grpc.UnaryClientInterceptor { rpcMethod = rpcInfo[2] } - responseStatus, _ := status.FromError(err) - ctx, tagErr := tag.New( - context.Background(), + taggedCtx, tagErr := tag.New( + ctx, tag.Insert(keyService, service), tag.Insert(keyHost, cc.Target()), - tag.Insert(keyMethod, rpcMethod), + tag.Insert(keyGRPCMethod, rpcMethod), tag.Insert(keyGRPCService, rpcService), - tag.Insert(keyStatus, responseStatus.Code().String()), ) - if tagErr != nil { - log.Warn().Err(tagErr).Str("context", "HTTPMetricsRoundTripper").Msg("Failed to create context tag") - } else { - responseProto := reply.(proto.Message) - responseSize := proto.Size(responseProto) - - stats.Record(ctx, - grpcClientRequestCount.M(1), - grpcClientRequestDuration.M(time.Since(startTime).Nanoseconds()/int64(time.Millisecond)), - grpcClientResponseSize.M(int64(responseSize)), - ) + log.Warn().Err(tagErr).Str("context", "GRPCClientInterceptor").Msg("internal/metrics: Failed to create context") + return invoker(ctx, method, req, reply, cc, opts...) } - return err + // Calls the invoker to execute RPC + return invoker(taggedCtx, method, req, reply, cc, opts...) } } + +// GRPCServerStatsHandler provides a grpc stats.Handler for a pomerium service to add tags and track +// metrics to server side calls +type GRPCServerStatsHandler struct { + service string + grpcstats.Handler +} + +// TagRPC implements grpc.stats.Handler and adds tags to the context of a given RPC +func (h *GRPCServerStatsHandler) TagRPC(ctx context.Context, tagInfo *grpcstats.RPCTagInfo) context.Context { + + handledCtx := h.Handler.TagRPC(ctx, tagInfo) + + // Split the method into parts for better slicing + rpcInfo := strings.SplitN(tagInfo.FullMethodName, "/", 3) + var rpcMethod string + var rpcService string + if len(rpcInfo) == 3 { + rpcService = rpcInfo[1] + rpcMethod = rpcInfo[2] + } + + taggedCtx, tagErr := tag.New( + handledCtx, + tag.Insert(keyService, h.service), + tag.Insert(keyGRPCMethod, rpcMethod), + tag.Insert(keyGRPCService, rpcService), + ) + if tagErr != nil { + log.Warn().Err(tagErr).Str("context", "GRPCServerStatsHandler").Msg("internal/metrics: Failed to create context") + return handledCtx + + } + + return taggedCtx +} + +// NewGRPCServerStatsHandler creates a new GRPCServerStatsHandler for a pomerium service +func NewGRPCServerStatsHandler(service string) grpcstats.Handler { + + return &GRPCServerStatsHandler{service: service, Handler: &ocgrpc.ServerHandler{}} +} diff --git a/internal/metrics/interceptors_test.go b/internal/metrics/interceptors_test.go index 5f8e3a027..7a3470cc1 100644 --- a/internal/metrics/interceptors_test.go +++ b/internal/metrics/interceptors_test.go @@ -4,8 +4,9 @@ import ( "context" "testing" - "go.opencensus.io/stats/view" + "go.opencensus.io/plugin/ocgrpc" "google.golang.org/grpc" + "google.golang.org/grpc/stats" "google.golang.org/grpc/status" ) @@ -29,11 +30,18 @@ func (t testProto) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { type testInvoker struct { invokeResult error + statsHandler stats.Handler } func (t testInvoker) UnaryInvoke(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, opts ...grpc.CallOption) error { r := reply.(*testProto) r.message = "hello" + + ctx = t.statsHandler.TagRPC(ctx, &stats.RPCTagInfo{FullMethodName: method}) + t.statsHandler.HandleRPC(ctx, &stats.InPayload{Client: true, Length: len(r.message)}) + t.statsHandler.HandleRPC(ctx, &stats.OutPayload{Client: true, Length: len(r.message)}) + t.statsHandler.HandleRPC(ctx, &stats.End{Client: true, Error: t.invokeResult}) + return t.invokeResult } @@ -55,49 +63,121 @@ func Test_GRPCClientInterceptor(t *testing.T) { wantgrpcClientResponseSize string wantgrpcClientRequestDuration string wantgrpcClientRequestCount string + wantgrpcClientRequestSize string }{ { name: "ok authorize", method: "/authorize.Authorizer/Authorize", errorCode: nil, - wantgrpcClientResponseSize: "{ { {grpc_service authorize.Authorizer}{host dns:localhost:9999}{method Authorize}{service test_service}{status OK} }&{1 5 5 5 0 [0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]", - wantgrpcClientRequestDuration: "{ { {grpc_service authorize.Authorizer}{host dns:localhost:9999}{method Authorize}{service test_service}{status OK} }&{1", - wantgrpcClientRequestCount: "{ { {grpc_service authorize.Authorizer}{host dns:localhost:9999}{method Authorize}{service test_service}{status OK} }&{1", + wantgrpcClientResponseSize: "{ { {grpc_client_status OK}{grpc_method Authorize}{grpc_service authorize.Authorizer}{host dns:localhost:9999}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]", + wantgrpcClientRequestDuration: "{ { {grpc_client_status OK}{grpc_method Authorize}{grpc_service authorize.Authorizer}{host dns:localhost:9999}{service test_service} }&{1", + wantgrpcClientRequestCount: "{ { {grpc_client_status OK}{grpc_method Authorize}{grpc_service authorize.Authorizer}{host dns:localhost:9999}{service test_service} }&{1", + wantgrpcClientRequestSize: "{ { {grpc_client_status OK}{grpc_method Authorize}{grpc_service authorize.Authorizer}{host dns:localhost:9999}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]", }, { name: "unknown validate", method: "/authenticate.Authenticator/Validate", errorCode: status.Error(14, ""), - wantgrpcClientResponseSize: "{ { {grpc_service authenticate.Authenticator}{host dns:localhost:9999}{method Validate}{service test_service}{status Unavailable} }&{1 5 5 5 0 [0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]", - wantgrpcClientRequestDuration: "{ { {grpc_service authenticate.Authenticator}{host dns:localhost:9999}{method Validate}{service test_service}{status Unavailable} }&{1", - wantgrpcClientRequestCount: "{ { {grpc_service authenticate.Authenticator}{host dns:localhost:9999}{method Validate}{service test_service}{status Unavailable} }&{1", + wantgrpcClientResponseSize: "{ { {grpc_client_status UNAVAILABLE}{grpc_method Validate}{grpc_service authenticate.Authenticator}{host dns:localhost:9999}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]", + wantgrpcClientRequestDuration: "{ { {grpc_client_status UNAVAILABLE}{grpc_method Validate}{grpc_service authenticate.Authenticator}{host dns:localhost:9999}{service test_service} }&{1", + wantgrpcClientRequestCount: "{ { {grpc_client_status UNAVAILABLE}{grpc_method Validate}{grpc_service authenticate.Authenticator}{host dns:localhost:9999}{service test_service} }&{1", + wantgrpcClientRequestSize: "{ { {grpc_client_status UNAVAILABLE}{grpc_method Validate}{grpc_service authenticate.Authenticator}{host dns:localhost:9999}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]", }, { name: "broken method parsing", method: "f", errorCode: status.Error(14, ""), - wantgrpcClientResponseSize: "{ { {host dns:localhost:9999}{service test_service}{status Unavailable} }&{1 5 5 5 0 [0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]", - wantgrpcClientRequestDuration: "{ { {host dns:localhost:9999}{service test_service}{status Unavailable} }&{1", - wantgrpcClientRequestCount: "{ { {host dns:localhost:9999}{service test_service}{status Unavailable} }&{1", + wantgrpcClientResponseSize: "{ { {grpc_client_status UNAVAILABLE}{host dns:localhost:9999}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]", + wantgrpcClientRequestDuration: "{ { {grpc_client_status UNAVAILABLE}{host dns:localhost:9999}{service test_service} }&{1", + wantgrpcClientRequestCount: "{ { {grpc_client_status UNAVAILABLE}{host dns:localhost:9999}{service test_service} }&{1", + wantgrpcClientRequestSize: "{ { {grpc_client_status UNAVAILABLE}{host dns:localhost:9999}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - view.Unregister(GRPCClientRequestCountView, GRPCClientRequestDurationView, GRPCClientResponseSizeView) - view.Register(GRPCClientRequestCountView, GRPCClientRequestDurationView, GRPCClientResponseSizeView) + UnRegisterView(GRPCClientViews) + RegisterView(GRPCClientViews) invoker := testInvoker{ invokeResult: tt.errorCode, + statsHandler: &ocgrpc.ClientHandler{}, } var reply testProto interceptor(context.Background(), tt.method, nil, &reply, newTestCC(t), invoker.UnaryInvoke) - testDataRetrieval(grpcClientResponseSize, t, tt.wantgrpcClientResponseSize) - testDataRetrieval(grpcClientRequestDuration, t, tt.wantgrpcClientRequestDuration) - testDataRetrieval(grpcClientRequestCount, t, tt.wantgrpcClientRequestCount) + testDataRetrieval(GRPCClientResponseSizeView, t, tt.wantgrpcClientResponseSize) + testDataRetrieval(GRPCClientRequestDurationView, t, tt.wantgrpcClientRequestDuration) + testDataRetrieval(GRPCClientRequestCountView, t, tt.wantgrpcClientRequestCount) + testDataRetrieval(GRPCClientRequestSizeView, t, tt.wantgrpcClientRequestSize) + + }) + } +} + +func mockServerRPCHandle(statsHandler stats.Handler, method string, errorCode error) { + message := "hello" + ctx := statsHandler.TagRPC(context.Background(), &stats.RPCTagInfo{FullMethodName: method}) + statsHandler.HandleRPC(ctx, &stats.InPayload{Client: false, Length: len(message)}) + statsHandler.HandleRPC(ctx, &stats.OutPayload{Client: false, Length: len(message)}) + statsHandler.HandleRPC(ctx, &stats.End{Client: false, Error: errorCode}) + +} +func Test_GRPCServerStatsHandler(t *testing.T) { + tests := []struct { + name string + method string + errorCode error + wantgrpcServerResponseSize string + wantgrpcServerRequestDuration string + wantgrpcServerRequestCount string + wantgrpcServerRequestSizeView string + }{ + { + name: "ok authorize", + method: "/authorize.Authorizer/Authorize", + errorCode: nil, + wantgrpcServerResponseSize: "{ { {grpc_method Authorize}{grpc_server_status OK}{grpc_service authorize.Authorizer}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]", + wantgrpcServerRequestDuration: "{ { {grpc_method Authorize}{grpc_server_status OK}{grpc_service authorize.Authorizer}{service test_service} }&{1", + wantgrpcServerRequestCount: "{ { {grpc_method Authorize}{grpc_server_status OK}{grpc_service authorize.Authorizer}{service test_service} }&{1", + wantgrpcServerRequestSizeView: "{ { {grpc_method Authorize}{grpc_server_status OK}{grpc_service authorize.Authorizer}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]", + }, + { + name: "unknown validate", + method: "/authenticate.Authenticator/Validate", + errorCode: status.Error(14, ""), + wantgrpcServerResponseSize: "{ { {grpc_method Validate}{grpc_server_status UNAVAILABLE}{grpc_service authenticate.Authenticator}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]", + wantgrpcServerRequestDuration: "{ { {grpc_method Validate}{grpc_server_status UNAVAILABLE}{grpc_service authenticate.Authenticator}{service test_service} }&{1", + wantgrpcServerRequestCount: "{ { {grpc_method Validate}{grpc_server_status UNAVAILABLE}{grpc_service authenticate.Authenticator}{service test_service} }&{1", + wantgrpcServerRequestSizeView: "{ { {grpc_method Validate}{grpc_server_status UNAVAILABLE}{grpc_service authenticate.Authenticator}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]", + }, + { + name: "broken method parsing", + method: "f", + errorCode: status.Error(14, ""), + wantgrpcServerResponseSize: "{ { {grpc_server_status UNAVAILABLE}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]", + wantgrpcServerRequestDuration: "{ { {grpc_server_status UNAVAILABLE}{service test_service} }&{1", + wantgrpcServerRequestCount: "{ { {grpc_server_status UNAVAILABLE}{service test_service} }&{1", + wantgrpcServerRequestSizeView: "{ { {grpc_server_status UNAVAILABLE}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + UnRegisterView(GRPCServerViews) + RegisterView(GRPCServerViews) + + statsHandler := NewGRPCServerStatsHandler("test_service") + mockServerRPCHandle(statsHandler, tt.method, tt.errorCode) + + testDataRetrieval(GRPCServerResponseSizeView, t, tt.wantgrpcServerResponseSize) + testDataRetrieval(GRPCServerRequestDurationView, t, tt.wantgrpcServerRequestDuration) + testDataRetrieval(GRPCServerRequestCountView, t, tt.wantgrpcServerRequestCount) + testDataRetrieval(GRPCServerRequestSizeView, t, tt.wantgrpcServerRequestSizeView) + }) } } diff --git a/internal/metrics/middleware_test.go b/internal/metrics/middleware_test.go index 5898473eb..1454d953d 100644 --- a/internal/metrics/middleware_test.go +++ b/internal/metrics/middleware_test.go @@ -70,9 +70,9 @@ func Test_HTTPMetricsHandler(t *testing.T) { rec := httptest.NewRecorder() chainHandler.ServeHTTP(rec, req) - testDataRetrieval(httpServerResponseSize, t, tt.wanthttpServerResponseSize) - testDataRetrieval(httpServerRequestDuration, t, tt.wanthttpServerRequestDuration) - testDataRetrieval(httpServerRequestCount, t, tt.wanthttpServerRequestCount) + testDataRetrieval(HTTPServerRequestSizeView, t, tt.wanthttpServerResponseSize) + testDataRetrieval(HTTPServerRequestDurationView, t, tt.wanthttpServerRequestDuration) + testDataRetrieval(HTTPServerRequestCountView, t, tt.wanthttpServerRequestCount) }) } } @@ -144,9 +144,9 @@ func Test_HTTPMetricsRoundTripper(t *testing.T) { resp, err := client.Do(req) t.Logf("response: %#v, %#v", resp, err) - testDataRetrieval(httpClientResponseSize, t, tt.wanthttpClientResponseSize) - testDataRetrieval(httpClientRequestDuration, t, tt.wanthttpClientRequestDuration) - testDataRetrieval(httpClientRequestCount, t, tt.wanthttpClientRequestCount) + testDataRetrieval(HTTPClientResponseSizeView, t, tt.wanthttpClientResponseSize) + testDataRetrieval(HTTPClientRequestDurationView, t, tt.wanthttpClientRequestDuration) + testDataRetrieval(HTTPClientRequestCountView, t, tt.wanthttpClientRequestCount) }) } diff --git a/internal/metrics/tags.go b/internal/metrics/tags.go index f610dd5bf..e68f1fa58 100644 --- a/internal/metrics/tags.go +++ b/internal/metrics/tags.go @@ -9,5 +9,6 @@ var ( keyStatus tag.Key = tag.MustNewKey("status") keyService tag.Key = tag.MustNewKey("service") keyGRPCService tag.Key = tag.MustNewKey("grpc_service") + keyGRPCMethod tag.Key = tag.MustNewKey("grpc_method") keyHost tag.Key = tag.MustNewKey("host") ) diff --git a/internal/metrics/view.go b/internal/metrics/view.go index 0dbbc2ea4..23914eab3 100644 --- a/internal/metrics/view.go +++ b/internal/metrics/view.go @@ -5,26 +5,26 @@ import ( "go.opencensus.io/stats/view" ) -// RegisterHTTPClientView registers the standard HTTPClient view. -// It must be called to see metrics in the configured exporters -func RegisterHTTPClientView() { - if err := view.Register(HTTPClientRequestCountView, HTTPClientRequestDurationView, HTTPClientResponseSizeView); err != nil { - log.Warn().Err(err).Msg("Could not register HTTPClientView") +var ( + // HTTPClientViews contains opencensus views for HTTP Client metrics + HTTPClientViews = []*view.View{HTTPClientRequestCountView, HTTPClientRequestDurationView, HTTPClientResponseSizeView} + // HTTPServerViews contains opencensus views for HTTP Server metrics + HTTPServerViews = []*view.View{HTTPServerRequestCountView, HTTPServerRequestDurationView, HTTPServerRequestSizeView} + // GRPCClientViews contains opencensus views for GRPC Client metrics + GRPCClientViews = []*view.View{GRPCClientRequestCountView, GRPCClientRequestDurationView, GRPCClientResponseSizeView, GRPCClientRequestSizeView} + // GRPCServerViews contains opencensus views for GRPC Server metrics + GRPCServerViews = []*view.View{GRPCServerRequestCountView, GRPCServerRequestDurationView, GRPCServerResponseSizeView, GRPCServerRequestSizeView} +) + +// RegisterView registers one of the defined metrics views. It must be called for metrics to see metrics +// in the configured exporters +func RegisterView(v []*view.View) { + if err := view.Register(v...); err != nil { + log.Warn().Str("context", "RegisterView").Err(err).Msg("internal/metrics: Could not register view") } } -// RegisterHTTPServerView registers the standard HTTPServer view. -// It must be called to see metrics in the configured exporters -func RegisterHTTPServerView() { - if err := view.Register(HTTPServerRequestCountView, HTTPServerRequestDurationView, HTTPServerRequestSizeView); err != nil { - log.Warn().Err(err).Msg("Could not register HTTPServerView") - } -} - -// RegisterGRPCClientView registers the standard GRPCClient view. -// It must be called to see metrics in the configured exporters -func RegisterGRPCClientView() { - if err := view.Register(GRPCClientRequestCountView, GRPCClientRequestDurationView, GRPCClientResponseSizeView); err != nil { - log.Warn().Err(err).Msg("Could not register GRPCClientView") - } +// UnRegisterView unregisters one of the defined metrics views. +func UnRegisterView(v []*view.View) { + view.Unregister(v...) } diff --git a/internal/metrics/view_test.go b/internal/metrics/view_test.go index 95e4d6aaa..3d34ea85d 100644 --- a/internal/metrics/view_test.go +++ b/internal/metrics/view_test.go @@ -6,29 +6,20 @@ import ( "go.opencensus.io/stats/view" ) -func Test_RegisterHTTPClientView(t *testing.T) { - RegisterHTTPClientView() - for _, v := range []*view.View{HTTPClientRequestCountView, HTTPClientRequestDurationView, HTTPClientResponseSizeView} { +func Test_RegisterView(t *testing.T) { + RegisterView(HTTPClientViews) + for _, v := range HTTPClientViews { if view.Find(v.Name) != v { t.Errorf("Failed to find registered view %s", v.Name) } } } -func Test_RegisterHTTPServerView(t *testing.T) { - RegisterHTTPServerView() - for _, v := range []*view.View{HTTPServerRequestCountView, HTTPServerRequestDurationView, HTTPServerRequestSizeView} { - if view.Find(v.Name) != v { - t.Errorf("Failed to find registered view %s", v.Name) - } - } -} - -func Test_RegisterGRPCClientView(t *testing.T) { - RegisterGRPCClientView() - for _, v := range []*view.View{GRPCClientRequestCountView, GRPCClientRequestDurationView, GRPCClientResponseSizeView} { - if view.Find(v.Name) != v { - t.Errorf("Failed to find registered view %s", v.Name) +func Test_UnregisterView(t *testing.T) { + UnRegisterView(HTTPClientViews) + for _, v := range HTTPClientViews { + if view.Find(v.Name) == v { + t.Errorf("Found unregistered view %s", v.Name) } } } diff --git a/proxy/clients/clients.go b/proxy/clients/clients.go index d0b6367ec..8f6d234fa 100644 --- a/proxy/clients/clients.go +++ b/proxy/clients/clients.go @@ -13,7 +13,7 @@ import ( "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/metrics" "github.com/pomerium/pomerium/internal/middleware" - + "go.opencensus.io/plugin/ocgrpc" "google.golang.org/grpc" "google.golang.org/grpc/credentials" ) @@ -106,5 +106,6 @@ func NewGRPCClientConn(opts *Options) (*grpc.ClientConn, error) { grpc.WithTransportCredentials(cert), grpc.WithPerRPCCredentials(grpcAuth), grpc.WithUnaryInterceptor(metrics.GRPCClientInterceptor("proxy")), + grpc.WithStatsHandler(&ocgrpc.ClientHandler{}), ) }