Add GRPC Client Metrics

2025-08-03 00:40:25 +02:00 · 2019-06-21 18:54:39 -04:00 · 2019-06-21 18:54:39 -04:00 · 77338bd4e9
commit 77338bd4e9
parent 7191ed6fb1
9 changed files with 311 additions and 69 deletions
--- a/internal/metrics/interceptors.go
+++ b/internal/metrics/interceptors.go
@ -0,0 +1,148 @@
+package metrics // import "github.com/pomerium/pomerium/internal/metrics"
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/pomerium/pomerium/internal/log"
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/status"
+)
+
+var (
+	grpcServerRequestCount    = stats.Int64("grpc_server_requests_total", "Total grpc Requests", "1")
+	grpcServerResponseSize    = stats.Int64("grpc_server_response_size_bytes", "grpc Server Response Size in bytes", "bytes")
+	grpcServerRequestDuration = stats.Int64("grpc_server_request_duration_ms", "grpc Request duration in ms", "ms")
+
+	grpcClientRequestCount    = stats.Int64("grpc_client_requests_total", "Total grpc Client Requests", "1")
+	grpcClientResponseSize    = stats.Int64("grpc_client_response_size_bytes", "grpc Client Response Size in bytes", "bytes")
+	grpcClientRequestDuration = stats.Int64("grpc_client_request_duration_ms", "grpc Client Request duration in ms", "ms")
+
+	grpcViews = []*view.View{
+		//grpc Server
+		{
+			Name:        grpcServerRequestCount.Name(),
+			Measure:     grpcServerRequestCount,
+			Description: grpcServerRequestCount.Description(),
+			TagKeys:     []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService},
+			Aggregation: view.Count(),
+		},
+		{
+			Name:        grpcServerRequestDuration.Name(),
+			Measure:     grpcServerRequestDuration,
+			Description: grpcServerRequestDuration.Description(),
+			TagKeys:     []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService},
+			Aggregation: view.Distribution(
+				1, 2, 5, 7, 10, 25, 500, 750,
+				100, 250, 500, 750,
+				1000, 2500, 5000, 7500,
+				10000, 25000, 50000, 75000,
+				100000,
+			),
+		},
+		{
+			Name:        grpcServerResponseSize.Name(),
+			Measure:     grpcServerResponseSize,
+			Description: grpcServerResponseSize.Description(),
+			TagKeys:     []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService},
+			Aggregation: view.Distribution(
+				1, 256, 512, 1024, 2048, 8192, 16384, 32768, 65536, 131072, 262144, 524288,
+				1048576, 2097152, 4194304, 8388608,
+			),
+		},
+
+		//grpc Client
+		{
+			Name:        grpcClientRequestCount.Name(),
+			Measure:     grpcClientRequestCount,
+			Description: grpcClientRequestCount.Description(),
+			TagKeys:     []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService},
+			Aggregation: view.Count(),
+		},
+		{
+			Name:        grpcClientRequestDuration.Name(),
+			Measure:     grpcClientRequestDuration,
+			Description: grpcClientRequestDuration.Description(),
+			TagKeys:     []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService},
+			Aggregation: view.Distribution(
+				1, 2, 5, 7, 10, 25, 500, 750,
+				100, 250, 500, 750,
+				1000, 2500, 5000, 7500,
+				10000, 25000, 50000, 75000,
+				100000,
+			),
+		},
+		{
+			Name:        grpcClientResponseSize.Name(),
+			Measure:     grpcClientResponseSize,
+			Description: grpcClientResponseSize.Description(),
+			TagKeys:     []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService},
+			Aggregation: view.Distribution(
+				1, 256, 512, 1024, 2048, 8192, 16384, 32768, 65536, 131072, 262144, 524288,
+				1048576, 2097152, 4194304, 8388608,
+			),
+		},
+	}
+)
+
+func init() {
+	view.Register(grpcViews...)
+}
+
+// GRPCClientInterceptor creates a UnaryClientInterceptor which tracks metrics of grpc client requests
+func GRPCClientInterceptor(service string) grpc.UnaryClientInterceptor {
+	return func(
+		ctx context.Context,
+		method string,
+		req interface{},
+		reply interface{},
+		cc *grpc.ClientConn,
+		invoker grpc.UnaryInvoker,
+		opts ...grpc.CallOption) error {
+
+		startTime := time.Now()
+
+		// Calls the invoker to execute RPC
+		err := invoker(ctx, method, req, reply, cc, opts...)
+
+		// Split the method into parts for better slicing
+		rpcInfo := strings.SplitN(method, "/", 3)
+		var rpcMethod string
+		var rpcService string
+		if len(rpcInfo) == 3 {
+			rpcService = rpcInfo[1]
+			rpcMethod = rpcInfo[2]
+		}
+
+		responseStatus, _ := status.FromError(err)
+		ctx, tagErr := tag.New(
+			context.Background(),
+			tag.Insert(keyService, service),
+			tag.Insert(keyHost, cc.Target()),
+			tag.Insert(keyMethod, rpcMethod),
+			tag.Insert(keyGRPCService, rpcService),
+			tag.Insert(keyStatus, responseStatus.Code().String()),
+		)
+
+		if tagErr != nil {
+			log.Warn().Err(tagErr).Str("context", "HTTPMetricsRoundTripper").Msg("Failed to create context tag")
+		} else {
+			responseProto := reply.(proto.Message)
+			responseSize := proto.Size(responseProto)
+
+			stats.Record(ctx,
+				grpcClientRequestCount.M(1),
+				grpcClientRequestDuration.M(time.Since(startTime).Nanoseconds()/int64(time.Millisecond)),
+				grpcClientResponseSize.M(int64(responseSize)),
+			)
+		}
+
+		return err
+	}
+
+}