GRPC metrics improvements

- change to ocgrpc plugin
- rename labels to be more consistent
- refactor view registration patterns
- add server metrics- add client request size metrics

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Pomerium Changelog
 
+## vUNRELEASED
+
+### New
+
+
+### Changed
+
+- GRPC Metrics Implementation [GH-218]
+  - Additional GRPC server metrics and request sizes
+  - Improved GRPC metrics implementation internals
+  - The GRPC method label is now 'grpc_method' and GRPC status is now `grpc_client_status` and `grpc_server_status`
+
+
 ## v0.1.0
 
 ### NEW

cmd/pomerium/main.go

@@ -41,7 +41,7 @@ func main() {
 	}
 	log.Info().Str("version", version.FullVersion()).Msg("cmd/pomerium")
 	grpcAuth := middleware.NewSharedSecretCred(opt.SharedKey)
-	grpcOpts := []grpc.ServerOption{grpc.UnaryInterceptor(grpcAuth.ValidateRequest)}
+	grpcOpts := []grpc.ServerOption{grpc.UnaryInterceptor(grpcAuth.ValidateRequest), grpc.StatsHandler(metrics.NewGRPCServerStatsHandler(opt.Services))}
 	grpcServer := grpc.NewServer(grpcOpts...)
 
 	mux := http.NewServeMux()
@@ -158,9 +158,10 @@ func newProxyService(opt config.Options, mux *http.ServeMux) (*proxy.Proxy, erro
 }
 
 func newPromListener(addr string) {
-	metrics.RegisterGRPCClientView()
+	metrics.RegisterView(metrics.HTTPClientViews)
-	metrics.RegisterHTTPClientView()
+	metrics.RegisterView(metrics.HTTPServerViews)
-	metrics.RegisterHTTPServerView()
+	metrics.RegisterView(metrics.GRPCClientViews)
+	metrics.RegisterView(metrics.GRPCServerViews)
 
 	log.Info().Str("MetricsAddr", addr).Msg("cmd/pomerium: starting prometheus endpoint")
 	log.Error().Err(metrics.NewPromHTTPListener(addr)).Str("MetricsAddr", addr).Msg("cmd/pomerium: could not start metrics exporter")

docs/reference/readme.md

@@ -160,7 +160,7 @@ If set, the HTTP Redirect Address specifies the host and port to redirect http t
 
 Expose a prometheus format HTTP endpoint on the specified port. Disabled by default.
 
-**Use with caution:** the endpoint can expose frontend and backend server names or addresses. Do not expose the metrics port publicly.
+**Use with caution:** the endpoint can expose frontend and backend server names or addresses. Do not expose the metrics port if this is sensitive information.
 
 #### Metrics tracked
 
@@ -174,7 +174,12 @@ http_client_response_size_bytes | Histogram | HTTP client response size by servi
 http_client_request_duration_ms | Histogram | HTTP client request duration by service
 grpc_client_requests_total      | Counter   | Total GRPC client requests made by service
 grpc_client_response_size_bytes | Histogram | GRPC client response size by service
+grpc_client_request_size_bytes | Histogram | GRPC client request size by service
 grpc_client_request_duration_ms | Histogram | GRPC client request duration by service
+grpc_server_requests_total      | Counter   | Total GRPC server requests made by service
+grpc_server_response_size_bytes | Histogram | GRPC server response size by service
+grpc_server_request_size_bytes | Histogram | GRPC server request size by service
+grpc_server_request_duration_ms | Histogram | GRPC server request duration by service
 
 ### Policy
 

internal/metrics/helpers_test.go

@@ -4,12 +4,14 @@ import (
 	"strings"
 	"testing"
 
-	"go.opencensus.io/stats"
 	"go.opencensus.io/stats/view"
 )
 
-func testDataRetrieval(measure stats.Measure, t *testing.T, want string) {
+func testDataRetrieval(v *view.View, t *testing.T, want string) {
-	name := measure.Name()
+	if v == nil {
+		t.Fatalf("%s: nil view passed", t.Name())
+	}
+	name := v.Name
 	data, err := view.RetrieveData(name)
 
 	if err != nil {

internal/metrics/interceptors.go

@@ -3,100 +3,100 @@ package metrics // import "github.com/pomerium/pomerium/internal/metrics"
 import (
 	"context"
 	"strings"
-	"time"
 
-	"github.com/golang/protobuf/proto"
 	"github.com/pomerium/pomerium/internal/log"
-	"go.opencensus.io/stats"
+	"go.opencensus.io/plugin/ocgrpc"
 	"go.opencensus.io/stats/view"
 	"go.opencensus.io/tag"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/status"
+	grpcstats "google.golang.org/grpc/stats"
 )
 
 var (
-	grpcServerRequestCount    = stats.Int64("grpc_server_requests_total", "Total grpc Requests", "1")
+	grpcSizeDistribution = view.Distribution(
-	grpcServerResponseSize    = stats.Int64("grpc_server_response_size_bytes", "grpc Server Response Size in bytes", "bytes")
+		1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024,
-	grpcServerRequestDuration = stats.Int64("grpc_server_request_duration_ms", "grpc Request duration in ms", "ms")
+		2048, 4096, 8192, 16384,
+	)
+	grcpLatencyDistribution = view.Distribution(
+		1, 2, 5, 7, 10, 25, 50, 75,
+		100, 250, 500, 750, 1000,
+	)
 
-	grpcClientRequestCount    = stats.Int64("grpc_client_requests_total", "Total grpc Client Requests", "1")
+	// GRPCServerRequestCountView is an OpenCensus view which counts GRPC Server requests by pomerium service, grpc service, grpc method, and status
-	grpcClientResponseSize    = stats.Int64("grpc_client_response_size_bytes", "grpc Client Response Size in bytes", "bytes")
-	grpcClientRequestDuration = stats.Int64("grpc_client_request_duration_ms", "grpc Client Request duration in ms", "ms")
-
-	// GRPCServerRequestCountView is an OpenCensus view which tracks GRPC Server requests by pomerium service, host, grpc service, grpc method, and status
 	GRPCServerRequestCountView = &view.View{
-		Name:        grpcServerRequestCount.Name(),
+		Name:        "grpc_server_requests_total",
-		Measure:     grpcServerRequestCount,
+		Measure:     ocgrpc.ServerLatency,
-		Description: grpcServerRequestCount.Description(),
+		Description: "Total grpc Requests",
-		TagKeys:     []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService},
+		TagKeys:     []tag.Key{keyService, keyGRPCMethod, ocgrpc.KeyServerStatus, keyGRPCService},
 		Aggregation: view.Count(),
 	}
 
-	// GRPCServerRequestDurationView is an OpenCensus view which tracks GRPC Server request duration by pomerium service, host, grpc service, grpc method, and statu
+	// GRPCServerRequestDurationView is an OpenCensus view which tracks GRPC Server request duration by pomerium service, grpc service, grpc method, and status
 	GRPCServerRequestDurationView = &view.View{
-		Name:        grpcServerRequestDuration.Name(),
+		Name:        "grpc_server_request_duration_ms",
-		Measure:     grpcServerRequestDuration,
+		Measure:     ocgrpc.ServerLatency,
-		Description: grpcServerRequestDuration.Description(),
+		Description: "grpc Request duration in ms",
-		TagKeys:     []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService},
+		TagKeys:     []tag.Key{keyService, keyGRPCMethod, ocgrpc.KeyServerStatus, keyGRPCService},
-		Aggregation: view.Distribution(
+		Aggregation: grcpLatencyDistribution,
-			1, 2, 5, 7, 10, 25, 500, 750,
-			100, 250, 500, 750,
-			1000, 2500, 5000, 7500,
-			10000, 25000, 50000, 75000,
-			100000,
-		),
 	}
 
-	// GRPCServerResponseSizeView is an OpenCensus view which tracks GRPC Server request duration by pomerium service, host, grpc service, grpc method, and statu
+	// GRPCServerResponseSizeView is an OpenCensus view which tracks GRPC Server response size by pomerium service, grpc service, grpc method, and status
 	GRPCServerResponseSizeView = &view.View{
-		Name:        grpcServerResponseSize.Name(),
+		Name:        "grpc_server_response_size_bytes",
-		Measure:     grpcServerResponseSize,
+		Measure:     ocgrpc.ServerSentBytesPerRPC,
-		Description: grpcServerResponseSize.Description(),
+		Description: "grpc Server Response Size in bytes",
-		TagKeys:     []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService},
+		TagKeys:     []tag.Key{keyService, keyGRPCMethod, ocgrpc.KeyServerStatus, keyGRPCService},
-		Aggregation: view.Distribution(
+		Aggregation: grpcSizeDistribution,
-			1, 256, 512, 1024, 2048, 8192, 16384, 32768, 65536, 131072, 262144, 524288,
-			1048576, 2097152, 4194304, 8388608,
-		),
 	}
 
-	// GRPCClientRequestCountView is an OpenCensus view which tracks GRPC Client requests by pomerium service, target host, grpc service, grpc method, and statu
+	// GRPCServerRequestSizeView is an OpenCensus view which tracks GRPC Server request size by pomerium service, grpc service, grpc method, and status
+	GRPCServerRequestSizeView = &view.View{
+		Name:        "grpc_server_request_size_bytes",
+		Measure:     ocgrpc.ServerReceivedBytesPerRPC,
+		Description: "grpc Server Request Size in bytes",
+		TagKeys:     []tag.Key{keyService, keyGRPCMethod, ocgrpc.KeyServerStatus, keyGRPCService},
+		Aggregation: grpcSizeDistribution,
+	}
+
+	// GRPCClientRequestCountView is an OpenCensus view which tracks GRPC Client requests by pomerium service, target host, grpc service, grpc method, and status
 	GRPCClientRequestCountView = &view.View{
-		Name:        grpcClientRequestCount.Name(),
+		Name:        "grpc_client_requests_total",
-		Measure:     grpcClientRequestCount,
+		Measure:     ocgrpc.ClientRoundtripLatency,
-		Description: grpcClientRequestCount.Description(),
+		Description: "Total grpc Client Requests",
-		TagKeys:     []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService},
+		TagKeys:     []tag.Key{keyService, keyHost, keyGRPCMethod, keyGRPCService, ocgrpc.KeyClientStatus},
 		Aggregation: view.Count(),
 	}
 
-	// GRPCClientRequestDurationView is an OpenCensus view which tracks GRPC Client request duration by pomerium service, target host, grpc service, grpc method, and statu
+	// GRPCClientRequestDurationView is an OpenCensus view which tracks GRPC Client request duration by pomerium service, target host, grpc service, grpc method, and status
 	GRPCClientRequestDurationView = &view.View{
-		Name:        grpcClientRequestDuration.Name(),
+		Name:        "grpc_client_request_duration_ms",
-		Measure:     grpcClientRequestDuration,
+		Measure:     ocgrpc.ClientRoundtripLatency,
-		Description: grpcClientRequestDuration.Description(),
+		Description: "grpc Client Request duration in ms",
-		TagKeys:     []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService},
+		TagKeys:     []tag.Key{keyService, keyHost, keyGRPCMethod, keyGRPCService, ocgrpc.KeyClientStatus},
-		Aggregation: view.Distribution(
+		Aggregation: grcpLatencyDistribution,
-			1, 2, 5, 7, 10, 25, 500, 750,
-			100, 250, 500, 750,
-			1000, 2500, 5000, 7500,
-			10000, 25000, 50000, 75000,
-			100000,
-		),
 	}
 
-	// GRPCClientResponseSizeView  is an OpenCensus view which tracks GRPC Client response size by pomerium service, target host, grpc service, grpc method, and statu
+	// GRPCClientResponseSizeView  is an OpenCensus view which tracks GRPC Client response size by pomerium service, target host, grpc service, grpc method, and status
 	GRPCClientResponseSizeView = &view.View{
-		Name:        grpcClientResponseSize.Name(),
+		Name:        "grpc_client_response_size_bytes",
-		Measure:     grpcClientResponseSize,
+		Measure:     ocgrpc.ClientReceivedBytesPerRPC,
-		Description: grpcClientResponseSize.Description(),
+		Description: "grpc Client Response Size in bytes",
-		TagKeys:     []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService},
+		TagKeys:     []tag.Key{keyService, keyHost, keyGRPCMethod, keyGRPCService, ocgrpc.KeyClientStatus},
-		Aggregation: view.Distribution(
+		Aggregation: grpcSizeDistribution,
-			1, 256, 512, 1024, 2048, 8192, 16384, 32768, 65536, 131072, 262144, 524288,
+	}
-			1048576, 2097152, 4194304, 8388608,
+
-		),
+	// GRPCClientRequestSizeView  is an OpenCensus view which tracks GRPC Client request size by pomerium service, target host, grpc service, grpc method, and status
+	GRPCClientRequestSizeView = &view.View{
+		Name:        "grpc_client_request_size_bytes",
+		Measure:     ocgrpc.ClientSentBytesPerRPC,
+		Description: "grpc Client Request Size in bytes",
+		TagKeys:     []tag.Key{keyService, keyHost, keyGRPCMethod, keyGRPCService, ocgrpc.KeyClientStatus},
+		Aggregation: grpcSizeDistribution,
 	}
 )
 
-// GRPCClientInterceptor creates a UnaryClientInterceptor which tracks metrics of grpc client requests
+// GRPCClientInterceptor creates a UnaryClientInterceptor which updates the RPC context with metric tag
+// metadata
 func GRPCClientInterceptor(service string) grpc.UnaryClientInterceptor {
 	return func(
 		ctx context.Context,
@@ -107,11 +107,6 @@ func GRPCClientInterceptor(service string) grpc.UnaryClientInterceptor {
 		invoker grpc.UnaryInvoker,
 		opts ...grpc.CallOption) error {
 
-		startTime := time.Now()
-
-		// Calls the invoker to execute RPC
-		err := invoker(ctx, method, req, reply, cc, opts...)
-
 		// Split the method into parts for better slicing
 		rpcInfo := strings.SplitN(method, "/", 3)
 		var rpcMethod string
@@ -121,30 +116,62 @@ func GRPCClientInterceptor(service string) grpc.UnaryClientInterceptor {
 			rpcMethod = rpcInfo[2]
 		}
 
-		responseStatus, _ := status.FromError(err)
+		taggedCtx, tagErr := tag.New(
-		ctx, tagErr := tag.New(
+			ctx,
-			context.Background(),
 			tag.Insert(keyService, service),
 			tag.Insert(keyHost, cc.Target()),
-			tag.Insert(keyMethod, rpcMethod),
+			tag.Insert(keyGRPCMethod, rpcMethod),
 			tag.Insert(keyGRPCService, rpcService),
-			tag.Insert(keyStatus, responseStatus.Code().String()),
 		)
-
 		if tagErr != nil {
-			log.Warn().Err(tagErr).Str("context", "HTTPMetricsRoundTripper").Msg("Failed to create context tag")
+			log.Warn().Err(tagErr).Str("context", "GRPCClientInterceptor").Msg("internal/metrics: Failed to create context")
-		} else {
+			return invoker(ctx, method, req, reply, cc, opts...)
-			responseProto := reply.(proto.Message)
-			responseSize := proto.Size(responseProto)
-
-			stats.Record(ctx,
-				grpcClientRequestCount.M(1),
-				grpcClientRequestDuration.M(time.Since(startTime).Nanoseconds()/int64(time.Millisecond)),
-				grpcClientResponseSize.M(int64(responseSize)),
-			)
 		}
 
-		return err
+		// Calls the invoker to execute RPC
+		return invoker(taggedCtx, method, req, reply, cc, opts...)
 	}
 
 }
+
+// GRPCServerStatsHandler provides a grpc stats.Handler for a pomerium service to add tags and track
+// metrics to server side calls
+type GRPCServerStatsHandler struct {
+	service string
+	grpcstats.Handler
+}
+
+// TagRPC implements grpc.stats.Handler and adds tags to the context of a given RPC
+func (h *GRPCServerStatsHandler) TagRPC(ctx context.Context, tagInfo *grpcstats.RPCTagInfo) context.Context {
+
+	handledCtx := h.Handler.TagRPC(ctx, tagInfo)
+
+	// Split the method into parts for better slicing
+	rpcInfo := strings.SplitN(tagInfo.FullMethodName, "/", 3)
+	var rpcMethod string
+	var rpcService string
+	if len(rpcInfo) == 3 {
+		rpcService = rpcInfo[1]
+		rpcMethod = rpcInfo[2]
+	}
+
+	taggedCtx, tagErr := tag.New(
+		handledCtx,
+		tag.Insert(keyService, h.service),
+		tag.Insert(keyGRPCMethod, rpcMethod),
+		tag.Insert(keyGRPCService, rpcService),
+	)
+	if tagErr != nil {
+		log.Warn().Err(tagErr).Str("context", "GRPCServerStatsHandler").Msg("internal/metrics: Failed to create context")
+		return handledCtx
+
+	}
+
+	return taggedCtx
+}
+
+// NewGRPCServerStatsHandler creates a new GRPCServerStatsHandler for a pomerium service
+func NewGRPCServerStatsHandler(service string) grpcstats.Handler {
+
+	return &GRPCServerStatsHandler{service: service, Handler: &ocgrpc.ServerHandler{}}
+}

internal/metrics/interceptors_test.go

@@ -4,8 +4,9 @@ import (
 	"context"
 	"testing"
 
-	"go.opencensus.io/stats/view"
+	"go.opencensus.io/plugin/ocgrpc"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/stats"
 	"google.golang.org/grpc/status"
 )
 
@@ -29,11 +30,18 @@ func (t testProto) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
 
 type testInvoker struct {
 	invokeResult error
+	statsHandler stats.Handler
 }
 
 func (t testInvoker) UnaryInvoke(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, opts ...grpc.CallOption) error {
 	r := reply.(*testProto)
 	r.message = "hello"
+
+	ctx = t.statsHandler.TagRPC(ctx, &stats.RPCTagInfo{FullMethodName: method})
+	t.statsHandler.HandleRPC(ctx, &stats.InPayload{Client: true, Length: len(r.message)})
+	t.statsHandler.HandleRPC(ctx, &stats.OutPayload{Client: true, Length: len(r.message)})
+	t.statsHandler.HandleRPC(ctx, &stats.End{Client: true, Error: t.invokeResult})
+
 	return t.invokeResult
 }
 
@@ -55,49 +63,121 @@ func Test_GRPCClientInterceptor(t *testing.T) {
 		wantgrpcClientResponseSize    string
 		wantgrpcClientRequestDuration string
 		wantgrpcClientRequestCount    string
+		wantgrpcClientRequestSize     string
 	}{
 		{
 			name:                          "ok authorize",
 			method:                        "/authorize.Authorizer/Authorize",
 			errorCode:                     nil,
-			wantgrpcClientResponseSize:    "{ { {grpc_service authorize.Authorizer}{host dns:localhost:9999}{method Authorize}{service test_service}{status OK} }&{1 5 5 5 0 [0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]",
+			wantgrpcClientResponseSize:    "{ { {grpc_client_status OK}{grpc_method Authorize}{grpc_service authorize.Authorizer}{host dns:localhost:9999}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
-			wantgrpcClientRequestDuration: "{ { {grpc_service authorize.Authorizer}{host dns:localhost:9999}{method Authorize}{service test_service}{status OK} }&{1",
+			wantgrpcClientRequestDuration: "{ { {grpc_client_status OK}{grpc_method Authorize}{grpc_service authorize.Authorizer}{host dns:localhost:9999}{service test_service} }&{1",
-			wantgrpcClientRequestCount:    "{ { {grpc_service authorize.Authorizer}{host dns:localhost:9999}{method Authorize}{service test_service}{status OK} }&{1",
+			wantgrpcClientRequestCount:    "{ { {grpc_client_status OK}{grpc_method Authorize}{grpc_service authorize.Authorizer}{host dns:localhost:9999}{service test_service} }&{1",
+			wantgrpcClientRequestSize:     "{ { {grpc_client_status OK}{grpc_method Authorize}{grpc_service authorize.Authorizer}{host dns:localhost:9999}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
 		},
 		{
 			name:                          "unknown validate",
 			method:                        "/authenticate.Authenticator/Validate",
 			errorCode:                     status.Error(14, ""),
-			wantgrpcClientResponseSize:    "{ { {grpc_service authenticate.Authenticator}{host dns:localhost:9999}{method Validate}{service test_service}{status Unavailable} }&{1 5 5 5 0 [0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]",
+			wantgrpcClientResponseSize:    "{ { {grpc_client_status UNAVAILABLE}{grpc_method Validate}{grpc_service authenticate.Authenticator}{host dns:localhost:9999}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
-			wantgrpcClientRequestDuration: "{ { {grpc_service authenticate.Authenticator}{host dns:localhost:9999}{method Validate}{service test_service}{status Unavailable} }&{1",
+			wantgrpcClientRequestDuration: "{ { {grpc_client_status UNAVAILABLE}{grpc_method Validate}{grpc_service authenticate.Authenticator}{host dns:localhost:9999}{service test_service} }&{1",
-			wantgrpcClientRequestCount:    "{ { {grpc_service authenticate.Authenticator}{host dns:localhost:9999}{method Validate}{service test_service}{status Unavailable} }&{1",
+			wantgrpcClientRequestCount:    "{ { {grpc_client_status UNAVAILABLE}{grpc_method Validate}{grpc_service authenticate.Authenticator}{host dns:localhost:9999}{service test_service} }&{1",
+			wantgrpcClientRequestSize:     "{ { {grpc_client_status UNAVAILABLE}{grpc_method Validate}{grpc_service authenticate.Authenticator}{host dns:localhost:9999}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
 		},
 		{
 			name:                          "broken method parsing",
 			method:                        "f",
 			errorCode:                     status.Error(14, ""),
-			wantgrpcClientResponseSize:    "{ { {host dns:localhost:9999}{service test_service}{status Unavailable} }&{1 5 5 5 0 [0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]",
+			wantgrpcClientResponseSize:    "{ { {grpc_client_status UNAVAILABLE}{host dns:localhost:9999}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
-			wantgrpcClientRequestDuration: "{ { {host dns:localhost:9999}{service test_service}{status Unavailable} }&{1",
+			wantgrpcClientRequestDuration: "{ { {grpc_client_status UNAVAILABLE}{host dns:localhost:9999}{service test_service} }&{1",
-			wantgrpcClientRequestCount:    "{ { {host dns:localhost:9999}{service test_service}{status Unavailable} }&{1",
+			wantgrpcClientRequestCount:    "{ { {grpc_client_status UNAVAILABLE}{host dns:localhost:9999}{service test_service} }&{1",
+			wantgrpcClientRequestSize:     "{ { {grpc_client_status UNAVAILABLE}{host dns:localhost:9999}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 
-			view.Unregister(GRPCClientRequestCountView, GRPCClientRequestDurationView, GRPCClientResponseSizeView)
+			UnRegisterView(GRPCClientViews)
-			view.Register(GRPCClientRequestCountView, GRPCClientRequestDurationView, GRPCClientResponseSizeView)
+			RegisterView(GRPCClientViews)
 
 			invoker := testInvoker{
 				invokeResult: tt.errorCode,
+				statsHandler: &ocgrpc.ClientHandler{},
 			}
 			var reply testProto
 
 			interceptor(context.Background(), tt.method, nil, &reply, newTestCC(t), invoker.UnaryInvoke)
 
-			testDataRetrieval(grpcClientResponseSize, t, tt.wantgrpcClientResponseSize)
+			testDataRetrieval(GRPCClientResponseSizeView, t, tt.wantgrpcClientResponseSize)
-			testDataRetrieval(grpcClientRequestDuration, t, tt.wantgrpcClientRequestDuration)
+			testDataRetrieval(GRPCClientRequestDurationView, t, tt.wantgrpcClientRequestDuration)
-			testDataRetrieval(grpcClientRequestCount, t, tt.wantgrpcClientRequestCount)
+			testDataRetrieval(GRPCClientRequestCountView, t, tt.wantgrpcClientRequestCount)
+			testDataRetrieval(GRPCClientRequestSizeView, t, tt.wantgrpcClientRequestSize)
+
+		})
+	}
+}
+
+func mockServerRPCHandle(statsHandler stats.Handler, method string, errorCode error) {
+	message := "hello"
+	ctx := statsHandler.TagRPC(context.Background(), &stats.RPCTagInfo{FullMethodName: method})
+	statsHandler.HandleRPC(ctx, &stats.InPayload{Client: false, Length: len(message)})
+	statsHandler.HandleRPC(ctx, &stats.OutPayload{Client: false, Length: len(message)})
+	statsHandler.HandleRPC(ctx, &stats.End{Client: false, Error: errorCode})
+
+}
+func Test_GRPCServerStatsHandler(t *testing.T) {
+	tests := []struct {
+		name                          string
+		method                        string
+		errorCode                     error
+		wantgrpcServerResponseSize    string
+		wantgrpcServerRequestDuration string
+		wantgrpcServerRequestCount    string
+		wantgrpcServerRequestSizeView string
+	}{
+		{
+			name:                          "ok authorize",
+			method:                        "/authorize.Authorizer/Authorize",
+			errorCode:                     nil,
+			wantgrpcServerResponseSize:    "{ { {grpc_method Authorize}{grpc_server_status OK}{grpc_service authorize.Authorizer}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
+			wantgrpcServerRequestDuration: "{ { {grpc_method Authorize}{grpc_server_status OK}{grpc_service authorize.Authorizer}{service test_service} }&{1",
+			wantgrpcServerRequestCount:    "{ { {grpc_method Authorize}{grpc_server_status OK}{grpc_service authorize.Authorizer}{service test_service} }&{1",
+			wantgrpcServerRequestSizeView: "{ { {grpc_method Authorize}{grpc_server_status OK}{grpc_service authorize.Authorizer}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
+		},
+		{
+			name:                          "unknown validate",
+			method:                        "/authenticate.Authenticator/Validate",
+			errorCode:                     status.Error(14, ""),
+			wantgrpcServerResponseSize:    "{ { {grpc_method Validate}{grpc_server_status UNAVAILABLE}{grpc_service authenticate.Authenticator}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
+			wantgrpcServerRequestDuration: "{ { {grpc_method Validate}{grpc_server_status UNAVAILABLE}{grpc_service authenticate.Authenticator}{service test_service} }&{1",
+			wantgrpcServerRequestCount:    "{ { {grpc_method Validate}{grpc_server_status UNAVAILABLE}{grpc_service authenticate.Authenticator}{service test_service} }&{1",
+			wantgrpcServerRequestSizeView: "{ { {grpc_method Validate}{grpc_server_status UNAVAILABLE}{grpc_service authenticate.Authenticator}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
+		},
+		{
+			name:                          "broken method parsing",
+			method:                        "f",
+			errorCode:                     status.Error(14, ""),
+			wantgrpcServerResponseSize:    "{ { {grpc_server_status UNAVAILABLE}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
+			wantgrpcServerRequestDuration: "{ { {grpc_server_status UNAVAILABLE}{service test_service} }&{1",
+			wantgrpcServerRequestCount:    "{ { {grpc_server_status UNAVAILABLE}{service test_service} }&{1",
+			wantgrpcServerRequestSizeView: "{ { {grpc_server_status UNAVAILABLE}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+
+			UnRegisterView(GRPCServerViews)
+			RegisterView(GRPCServerViews)
+
+			statsHandler := NewGRPCServerStatsHandler("test_service")
+			mockServerRPCHandle(statsHandler, tt.method, tt.errorCode)
+
+			testDataRetrieval(GRPCServerResponseSizeView, t, tt.wantgrpcServerResponseSize)
+			testDataRetrieval(GRPCServerRequestDurationView, t, tt.wantgrpcServerRequestDuration)
+			testDataRetrieval(GRPCServerRequestCountView, t, tt.wantgrpcServerRequestCount)
+			testDataRetrieval(GRPCServerRequestSizeView, t, tt.wantgrpcServerRequestSizeView)
+
 		})
 	}
 }

internal/metrics/middleware_test.go

@@ -70,9 +70,9 @@ func Test_HTTPMetricsHandler(t *testing.T) {
 			rec := httptest.NewRecorder()
 			chainHandler.ServeHTTP(rec, req)
 
-			testDataRetrieval(httpServerResponseSize, t, tt.wanthttpServerResponseSize)
+			testDataRetrieval(HTTPServerRequestSizeView, t, tt.wanthttpServerResponseSize)
-			testDataRetrieval(httpServerRequestDuration, t, tt.wanthttpServerRequestDuration)
+			testDataRetrieval(HTTPServerRequestDurationView, t, tt.wanthttpServerRequestDuration)
-			testDataRetrieval(httpServerRequestCount, t, tt.wanthttpServerRequestCount)
+			testDataRetrieval(HTTPServerRequestCountView, t, tt.wanthttpServerRequestCount)
 		})
 	}
 }
@@ -144,9 +144,9 @@ func Test_HTTPMetricsRoundTripper(t *testing.T) {
 			resp, err := client.Do(req)
 
 			t.Logf("response: %#v, %#v", resp, err)
-			testDataRetrieval(httpClientResponseSize, t, tt.wanthttpClientResponseSize)
+			testDataRetrieval(HTTPClientResponseSizeView, t, tt.wanthttpClientResponseSize)
-			testDataRetrieval(httpClientRequestDuration, t, tt.wanthttpClientRequestDuration)
+			testDataRetrieval(HTTPClientRequestDurationView, t, tt.wanthttpClientRequestDuration)
-			testDataRetrieval(httpClientRequestCount, t, tt.wanthttpClientRequestCount)
+			testDataRetrieval(HTTPClientRequestCountView, t, tt.wanthttpClientRequestCount)
 		})
 	}
 

internal/metrics/tags.go

@@ -9,5 +9,6 @@ var (
 	keyStatus      tag.Key = tag.MustNewKey("status")
 	keyService     tag.Key = tag.MustNewKey("service")
 	keyGRPCService tag.Key = tag.MustNewKey("grpc_service")
+	keyGRPCMethod  tag.Key = tag.MustNewKey("grpc_method")
 	keyHost        tag.Key = tag.MustNewKey("host")
 )

internal/metrics/view.go

@@ -5,26 +5,26 @@ import (
 	"go.opencensus.io/stats/view"
 )
 
-// RegisterHTTPClientView registers the standard HTTPClient view.
+var (
-// It must be called to see metrics in the configured exporters
+	// HTTPClientViews contains opencensus views for HTTP Client metrics
-func RegisterHTTPClientView() {
+	HTTPClientViews = []*view.View{HTTPClientRequestCountView, HTTPClientRequestDurationView, HTTPClientResponseSizeView}
-	if err := view.Register(HTTPClientRequestCountView, HTTPClientRequestDurationView, HTTPClientResponseSizeView); err != nil {
+	// HTTPServerViews contains opencensus views for HTTP Server metrics
-		log.Warn().Err(err).Msg("Could not register HTTPClientView")
+	HTTPServerViews = []*view.View{HTTPServerRequestCountView, HTTPServerRequestDurationView, HTTPServerRequestSizeView}
+	// GRPCClientViews contains opencensus views for GRPC Client metrics
+	GRPCClientViews = []*view.View{GRPCClientRequestCountView, GRPCClientRequestDurationView, GRPCClientResponseSizeView, GRPCClientRequestSizeView}
+	// GRPCServerViews contains opencensus views for GRPC Server metrics
+	GRPCServerViews = []*view.View{GRPCServerRequestCountView, GRPCServerRequestDurationView, GRPCServerResponseSizeView, GRPCServerRequestSizeView}
+)
+
+// RegisterView registers one of the defined metrics views.  It must be called for metrics to see metrics
+// in the configured exporters
+func RegisterView(v []*view.View) {
+	if err := view.Register(v...); err != nil {
+		log.Warn().Str("context", "RegisterView").Err(err).Msg("internal/metrics: Could not register view")
 	}
 }
 
-// RegisterHTTPServerView registers the standard HTTPServer view.
+// UnRegisterView unregisters one of the defined metrics views.
-// It must be called to see metrics in the configured exporters
+func UnRegisterView(v []*view.View) {
-func RegisterHTTPServerView() {
+	view.Unregister(v...)
-	if err := view.Register(HTTPServerRequestCountView, HTTPServerRequestDurationView, HTTPServerRequestSizeView); err != nil {
-		log.Warn().Err(err).Msg("Could not register HTTPServerView")
-	}
-}
-
-// RegisterGRPCClientView registers the standard GRPCClient view.
-// It must be called to see metrics in the configured exporters
-func RegisterGRPCClientView() {
-	if err := view.Register(GRPCClientRequestCountView, GRPCClientRequestDurationView, GRPCClientResponseSizeView); err != nil {
-		log.Warn().Err(err).Msg("Could not register GRPCClientView")
-	}
 }

internal/metrics/view_test.go

@@ -6,29 +6,20 @@ import (
 	"go.opencensus.io/stats/view"
 )
 
-func Test_RegisterHTTPClientView(t *testing.T) {
+func Test_RegisterView(t *testing.T) {
-	RegisterHTTPClientView()
+	RegisterView(HTTPClientViews)
-	for _, v := range []*view.View{HTTPClientRequestCountView, HTTPClientRequestDurationView, HTTPClientResponseSizeView} {
+	for _, v := range HTTPClientViews {
 		if view.Find(v.Name) != v {
 			t.Errorf("Failed to find registered view %s", v.Name)
 		}
 	}
 }
 
-func Test_RegisterHTTPServerView(t *testing.T) {
+func Test_UnregisterView(t *testing.T) {
-	RegisterHTTPServerView()
+	UnRegisterView(HTTPClientViews)
-	for _, v := range []*view.View{HTTPServerRequestCountView, HTTPServerRequestDurationView, HTTPServerRequestSizeView} {
+	for _, v := range HTTPClientViews {
-		if view.Find(v.Name) != v {
+		if view.Find(v.Name) == v {
-			t.Errorf("Failed to find registered view %s", v.Name)
+			t.Errorf("Found unregistered view %s", v.Name)
-		}
-	}
-}
-
-func Test_RegisterGRPCClientView(t *testing.T) {
-	RegisterGRPCClientView()
-	for _, v := range []*view.View{GRPCClientRequestCountView, GRPCClientRequestDurationView, GRPCClientResponseSizeView} {
-		if view.Find(v.Name) != v {
-			t.Errorf("Failed to find registered view %s", v.Name)
 		}
 	}
 }

proxy/clients/clients.go

@@ -13,7 +13,7 @@ import (
 	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/internal/metrics"
 	"github.com/pomerium/pomerium/internal/middleware"
-
+	"go.opencensus.io/plugin/ocgrpc"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 )
@@ -106,5 +106,6 @@ func NewGRPCClientConn(opts *Options) (*grpc.ClientConn, error) {
 		grpc.WithTransportCredentials(cert),
 		grpc.WithPerRPCCredentials(grpcAuth),
 		grpc.WithUnaryInterceptor(metrics.GRPCClientInterceptor("proxy")),
+		grpc.WithStatsHandler(&ocgrpc.ClientHandler{}),
 	)
 }