Ensure service name is passed to grpc metrics handlers (#510)

cmd/pomerium/main.go

@@ -150,6 +150,7 @@ func newGRPCServer(opt config.Options, as *authorize.Authorize, cs *cache.Cache,
 	}
 	so := &pgrpc.ServerOptions{
 		Addr:        opt.GRPCAddr,
+		ServiceName: opt.Services,
 		KeepaliveParams: keepalive.ServerParameters{
 			MaxConnectionAge:      opt.GRPCServerMaxConnectionAge,
 			MaxConnectionAgeGrace: opt.GRPCServerMaxConnectionAgeGrace,

go.sum

@@ -510,6 +510,7 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk=
 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

internal/grpc/client.go

@@ -43,6 +43,9 @@ type Options struct {
 	// WithInsecure disables transport security for this ClientConn.
 	// Note that transport security is required unless WithInsecure is set.
 	WithInsecure bool
+
+	// ServiceName specifies the service name for telemetry exposition
+	ServiceName string
 }
 
 // NewGRPCClientConn returns a new gRPC pomerium service client connection.
@@ -57,7 +60,7 @@ func NewGRPCClientConn(opts *Options) (*grpc.ClientConn, error) {
 		connAddr = fmt.Sprintf("%s:%d", connAddr, defaultGRPCPort)
 	}
 	dialOptions := []grpc.DialOption{
-		grpc.WithChainUnaryInterceptor(metrics.GRPCClientInterceptor(connAddr), grpcTimeoutInterceptor(opts.RequestTimeout)),
+		grpc.WithChainUnaryInterceptor(metrics.GRPCClientInterceptor(opts.ServiceName), grpcTimeoutInterceptor(opts.RequestTimeout)),
 		grpc.WithStatsHandler(&ocgrpc.ClientHandler{}),
 		grpc.WithDefaultCallOptions([]grpc.CallOption{grpc.WaitForReady(true)}...),
 	}

internal/grpc/server.go

@@ -29,7 +29,7 @@ func NewServer(opt *ServerOptions, registrationFn func(s *grpc.Server), wg *sync
 		return nil, err
 	}
 	grpcOpts := []grpc.ServerOption{
-		grpc.StatsHandler(metrics.NewGRPCServerStatsHandler(opt.Addr)),
+		grpc.StatsHandler(metrics.NewGRPCServerStatsHandler(opt.ServiceName)),
 		grpc.KeepaliveParams(opt.KeepaliveParams),
 	}
 
@@ -70,7 +70,11 @@ type ServerOptions struct {
 	// This should be used only for testing.
 	InsecureServer bool
 
+	// KeepaliveParams sets GRPC keepalive.ServerParameters
 	KeepaliveParams keepalive.ServerParameters
+
+	// ServiceName specifies the service name for telemetry exposition
+	ServiceName string
 }
 
 var defaultServerOptions = &ServerOptions{

internal/telemetry/metrics/grpc.go

@@ -131,10 +131,10 @@ func GRPCClientInterceptor(service string) grpc.UnaryClientInterceptor {
 
 		taggedCtx, tagErr := tag.New(
 			ctx,
-			tag.Insert(TagKeyService, service),
+			tag.Upsert(TagKeyService, service),
-			tag.Insert(TagKeyHost, cc.Target()),
+			tag.Upsert(TagKeyHost, cc.Target()),
-			tag.Insert(TagKeyGRPCMethod, rpcMethod),
+			tag.Upsert(TagKeyGRPCMethod, rpcMethod),
-			tag.Insert(TagKeyGRPCService, rpcService),
+			tag.Upsert(TagKeyGRPCService, rpcService),
 		)
 		if tagErr != nil {
 			log.Warn().Err(tagErr).Str("context", "GRPCClientInterceptor").Msg("telemetry/metrics: failed to create context")
@@ -170,9 +170,9 @@ func (h *GRPCServerStatsHandler) TagRPC(ctx context.Context, tagInfo *grpcstats.
 
 	taggedCtx, tagErr := tag.New(
 		handledCtx,
-		tag.Insert(TagKeyService, h.service),
+		tag.Upsert(TagKeyService, h.service),
-		tag.Insert(TagKeyGRPCMethod, rpcMethod),
+		tag.Upsert(TagKeyGRPCMethod, rpcMethod),
-		tag.Insert(TagKeyGRPCService, rpcService),
+		tag.Upsert(TagKeyGRPCService, rpcService),
 	)
 	if tagErr != nil {
 		log.Warn().Err(tagErr).Str("context", "GRPCServerStatsHandler").Msg("telemetry/metrics: failed to create context")

internal/telemetry/metrics/http.go

@@ -112,9 +112,9 @@ func HTTPMetricsHandler(service string) func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			ctx, tagErr := tag.New(
 				r.Context(),
-				tag.Insert(TagKeyService, service),
+				tag.Upsert(TagKeyService, service),
-				tag.Insert(TagKeyHost, r.Host),
+				tag.Upsert(TagKeyHost, r.Host),
-				tag.Insert(TagKeyHTTPMethod, r.Method),
+				tag.Upsert(TagKeyHTTPMethod, r.Method),
 			)
 			if tagErr != nil {
 				log.Warn().Err(tagErr).Str("context", "HTTPMetricsHandler").Msg("telemetry/metrics: failed to create metrics tag")
@@ -139,10 +139,10 @@ func HTTPMetricsRoundTripper(service string, destination string) func(next http.
 		return tripper.RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
 			ctx, tagErr := tag.New(
 				r.Context(),
-				tag.Insert(TagKeyService, service),
+				tag.Upsert(TagKeyService, service),
-				tag.Insert(TagKeyHost, r.Host),
+				tag.Upsert(TagKeyHost, r.Host),
-				tag.Insert(TagKeyHTTPMethod, r.Method),
+				tag.Upsert(TagKeyHTTPMethod, r.Method),
-				tag.Insert(TagKeyDestination, destination),
+				tag.Upsert(TagKeyDestination, destination),
 			)
 			if tagErr != nil {
 				log.Warn().Err(tagErr).Str("context", "HTTPMetricsRoundTripper").Msg("telemetry/metrics: failed to create metrics tag")