mirror of
https://github.com/pomerium/pomerium.git
synced 2025-05-11 16:17:39 +02:00
GRPC metrics improvements
- change to ocgrpc plugin - rename labels to be more consistent - refactor view registration patterns - add server metrics- add client request size metrics
This commit is contained in:
Travis Groth
parent
d0f1314286
commit
4bd4b27f28
11 changed files with 273 additions and 152 deletions
13
CHANGELOG.md
13
CHANGELOG.md
|
|
@ -1,5 +1,18 @@
|
|||
# Pomerium Changelog
|
||||
|
||||
## vUNRELEASED
|
||||
|
||||
### New
|
||||
|
||||
|
||||
### Changed
|
||||
|
||||
- GRPC Metrics Implementation [GH-218]
|
||||
- Additional GRPC server metrics and request sizes
|
||||
- Improved GRPC metrics implementation internals
|
||||
- The GRPC method label is now 'grpc_method' and GRPC status is now `grpc_client_status` and `grpc_server_status`
|
||||
|
||||
|
||||
## v0.1.0
|
||||
|
||||
### NEW
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ func main() {
|
|||
}
|
||||
log.Info().Str("version", version.FullVersion()).Msg("cmd/pomerium")
|
||||
grpcAuth := middleware.NewSharedSecretCred(opt.SharedKey)
|
||||
grpcOpts := []grpc.ServerOption{grpc.UnaryInterceptor(grpcAuth.ValidateRequest)}
|
||||
grpcOpts := []grpc.ServerOption{grpc.UnaryInterceptor(grpcAuth.ValidateRequest), grpc.StatsHandler(metrics.NewGRPCServerStatsHandler(opt.Services))}
|
||||
grpcServer := grpc.NewServer(grpcOpts...)
|
||||
|
||||
mux := http.NewServeMux()
|
||||
|
|
@ -158,9 +158,10 @@ func newProxyService(opt config.Options, mux *http.ServeMux) (*proxy.Proxy, erro
|
|||
}
|
||||
|
||||
func newPromListener(addr string) {
|
||||
metrics.RegisterGRPCClientView()
|
||||
metrics.RegisterHTTPClientView()
|
||||
metrics.RegisterHTTPServerView()
|
||||
metrics.RegisterView(metrics.HTTPClientViews)
|
||||
metrics.RegisterView(metrics.HTTPServerViews)
|
||||
metrics.RegisterView(metrics.GRPCClientViews)
|
||||
metrics.RegisterView(metrics.GRPCServerViews)
|
||||
|
||||
log.Info().Str("MetricsAddr", addr).Msg("cmd/pomerium: starting prometheus endpoint")
|
||||
log.Error().Err(metrics.NewPromHTTPListener(addr)).Str("MetricsAddr", addr).Msg("cmd/pomerium: could not start metrics exporter")
|
||||
|
|
|
|||
|
|
@ -160,7 +160,7 @@ If set, the HTTP Redirect Address specifies the host and port to redirect http t
|
|||
|
||||
Expose a prometheus format HTTP endpoint on the specified port. Disabled by default.
|
||||
|
||||
**Use with caution:** the endpoint can expose frontend and backend server names or addresses. Do not expose the metrics port publicly.
|
||||
**Use with caution:** the endpoint can expose frontend and backend server names or addresses. Do not expose the metrics port if this is sensitive information.
|
||||
|
||||
#### Metrics tracked
|
||||
|
||||
|
|
@ -174,7 +174,12 @@ http_client_response_size_bytes | Histogram | HTTP client response size by servi
|
|||
http_client_request_duration_ms | Histogram | HTTP client request duration by service
|
||||
grpc_client_requests_total | Counter | Total GRPC client requests made by service
|
||||
grpc_client_response_size_bytes | Histogram | GRPC client response size by service
|
||||
grpc_client_request_size_bytes | Histogram | GRPC client request size by service
|
||||
grpc_client_request_duration_ms | Histogram | GRPC client request duration by service
|
||||
grpc_server_requests_total | Counter | Total GRPC server requests made by service
|
||||
grpc_server_response_size_bytes | Histogram | GRPC server response size by service
|
||||
grpc_server_request_size_bytes | Histogram | GRPC server request size by service
|
||||
grpc_server_request_duration_ms | Histogram | GRPC server request duration by service
|
||||
|
||||
### Policy
|
||||
|
||||
|
|
|
|||
|
|
@ -4,12 +4,14 @@ import (
|
|||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.opencensus.io/stats"
|
||||
"go.opencensus.io/stats/view"
|
||||
)
|
||||
|
||||
func testDataRetrieval(measure stats.Measure, t *testing.T, want string) {
|
||||
name := measure.Name()
|
||||
func testDataRetrieval(v *view.View, t *testing.T, want string) {
|
||||
if v == nil {
|
||||
t.Fatalf("%s: nil view passed", t.Name())
|
||||
}
|
||||
name := v.Name
|
||||
data, err := view.RetrieveData(name)
|
||||
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -3,100 +3,100 @@ package metrics // import "github.com/pomerium/pomerium/internal/metrics"
|
|||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/pomerium/pomerium/internal/log"
|
||||
"go.opencensus.io/stats"
|
||||
"go.opencensus.io/plugin/ocgrpc"
|
||||
"go.opencensus.io/stats/view"
|
||||
"go.opencensus.io/tag"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/status"
|
||||
grpcstats "google.golang.org/grpc/stats"
|
||||
)
|
||||
|
||||
var (
|
||||
grpcServerRequestCount = stats.Int64("grpc_server_requests_total", "Total grpc Requests", "1")
|
||||
grpcServerResponseSize = stats.Int64("grpc_server_response_size_bytes", "grpc Server Response Size in bytes", "bytes")
|
||||
grpcServerRequestDuration = stats.Int64("grpc_server_request_duration_ms", "grpc Request duration in ms", "ms")
|
||||
grpcSizeDistribution = view.Distribution(
|
||||
1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024,
|
||||
2048, 4096, 8192, 16384,
|
||||
)
|
||||
grcpLatencyDistribution = view.Distribution(
|
||||
1, 2, 5, 7, 10, 25, 50, 75,
|
||||
100, 250, 500, 750, 1000,
|
||||
)
|
||||
|
||||
grpcClientRequestCount = stats.Int64("grpc_client_requests_total", "Total grpc Client Requests", "1")
|
||||
grpcClientResponseSize = stats.Int64("grpc_client_response_size_bytes", "grpc Client Response Size in bytes", "bytes")
|
||||
grpcClientRequestDuration = stats.Int64("grpc_client_request_duration_ms", "grpc Client Request duration in ms", "ms")
|
||||
|
||||
// GRPCServerRequestCountView is an OpenCensus view which tracks GRPC Server requests by pomerium service, host, grpc service, grpc method, and status
|
||||
// GRPCServerRequestCountView is an OpenCensus view which counts GRPC Server requests by pomerium service, grpc service, grpc method, and status
|
||||
GRPCServerRequestCountView = &view.View{
|
||||
Name: grpcServerRequestCount.Name(),
|
||||
Measure: grpcServerRequestCount,
|
||||
Description: grpcServerRequestCount.Description(),
|
||||
TagKeys: []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService},
|
||||
Name: "grpc_server_requests_total",
|
||||
Measure: ocgrpc.ServerLatency,
|
||||
Description: "Total grpc Requests",
|
||||
TagKeys: []tag.Key{keyService, keyGRPCMethod, ocgrpc.KeyServerStatus, keyGRPCService},
|
||||
Aggregation: view.Count(),
|
||||
}
|
||||
|
||||
// GRPCServerRequestDurationView is an OpenCensus view which tracks GRPC Server request duration by pomerium service, host, grpc service, grpc method, and statu
|
||||
// GRPCServerRequestDurationView is an OpenCensus view which tracks GRPC Server request duration by pomerium service, grpc service, grpc method, and status
|
||||
GRPCServerRequestDurationView = &view.View{
|
||||
Name: grpcServerRequestDuration.Name(),
|
||||
Measure: grpcServerRequestDuration,
|
||||
Description: grpcServerRequestDuration.Description(),
|
||||
TagKeys: []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService},
|
||||
Aggregation: view.Distribution(
|
||||
1, 2, 5, 7, 10, 25, 500, 750,
|
||||
100, 250, 500, 750,
|
||||
1000, 2500, 5000, 7500,
|
||||
10000, 25000, 50000, 75000,
|
||||
100000,
|
||||
),
|
||||
Name: "grpc_server_request_duration_ms",
|
||||
Measure: ocgrpc.ServerLatency,
|
||||
Description: "grpc Request duration in ms",
|
||||
TagKeys: []tag.Key{keyService, keyGRPCMethod, ocgrpc.KeyServerStatus, keyGRPCService},
|
||||
Aggregation: grcpLatencyDistribution,
|
||||
}
|
||||
|
||||
// GRPCServerResponseSizeView is an OpenCensus view which tracks GRPC Server request duration by pomerium service, host, grpc service, grpc method, and statu
|
||||
// GRPCServerResponseSizeView is an OpenCensus view which tracks GRPC Server response size by pomerium service, grpc service, grpc method, and status
|
||||
GRPCServerResponseSizeView = &view.View{
|
||||
Name: grpcServerResponseSize.Name(),
|
||||
Measure: grpcServerResponseSize,
|
||||
Description: grpcServerResponseSize.Description(),
|
||||
TagKeys: []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService},
|
||||
Aggregation: view.Distribution(
|
||||
1, 256, 512, 1024, 2048, 8192, 16384, 32768, 65536, 131072, 262144, 524288,
|
||||
1048576, 2097152, 4194304, 8388608,
|
||||
),
|
||||
Name: "grpc_server_response_size_bytes",
|
||||
Measure: ocgrpc.ServerSentBytesPerRPC,
|
||||
Description: "grpc Server Response Size in bytes",
|
||||
TagKeys: []tag.Key{keyService, keyGRPCMethod, ocgrpc.KeyServerStatus, keyGRPCService},
|
||||
Aggregation: grpcSizeDistribution,
|
||||
}
|
||||
|
||||
// GRPCClientRequestCountView is an OpenCensus view which tracks GRPC Client requests by pomerium service, target host, grpc service, grpc method, and statu
|
||||
// GRPCServerRequestSizeView is an OpenCensus view which tracks GRPC Server request size by pomerium service, grpc service, grpc method, and status
|
||||
GRPCServerRequestSizeView = &view.View{
|
||||
Name: "grpc_server_request_size_bytes",
|
||||
Measure: ocgrpc.ServerReceivedBytesPerRPC,
|
||||
Description: "grpc Server Request Size in bytes",
|
||||
TagKeys: []tag.Key{keyService, keyGRPCMethod, ocgrpc.KeyServerStatus, keyGRPCService},
|
||||
Aggregation: grpcSizeDistribution,
|
||||
}
|
||||
|
||||
// GRPCClientRequestCountView is an OpenCensus view which tracks GRPC Client requests by pomerium service, target host, grpc service, grpc method, and status
|
||||
GRPCClientRequestCountView = &view.View{
|
||||
Name: grpcClientRequestCount.Name(),
|
||||
Measure: grpcClientRequestCount,
|
||||
Description: grpcClientRequestCount.Description(),
|
||||
TagKeys: []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService},
|
||||
Name: "grpc_client_requests_total",
|
||||
Measure: ocgrpc.ClientRoundtripLatency,
|
||||
Description: "Total grpc Client Requests",
|
||||
TagKeys: []tag.Key{keyService, keyHost, keyGRPCMethod, keyGRPCService, ocgrpc.KeyClientStatus},
|
||||
Aggregation: view.Count(),
|
||||
}
|
||||
|
||||
// GRPCClientRequestDurationView is an OpenCensus view which tracks GRPC Client request duration by pomerium service, target host, grpc service, grpc method, and statu
|
||||
// GRPCClientRequestDurationView is an OpenCensus view which tracks GRPC Client request duration by pomerium service, target host, grpc service, grpc method, and status
|
||||
GRPCClientRequestDurationView = &view.View{
|
||||
Name: grpcClientRequestDuration.Name(),
|
||||
Measure: grpcClientRequestDuration,
|
||||
Description: grpcClientRequestDuration.Description(),
|
||||
TagKeys: []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService},
|
||||
Aggregation: view.Distribution(
|
||||
1, 2, 5, 7, 10, 25, 500, 750,
|
||||
100, 250, 500, 750,
|
||||
1000, 2500, 5000, 7500,
|
||||
10000, 25000, 50000, 75000,
|
||||
100000,
|
||||
),
|
||||
Name: "grpc_client_request_duration_ms",
|
||||
Measure: ocgrpc.ClientRoundtripLatency,
|
||||
Description: "grpc Client Request duration in ms",
|
||||
TagKeys: []tag.Key{keyService, keyHost, keyGRPCMethod, keyGRPCService, ocgrpc.KeyClientStatus},
|
||||
Aggregation: grcpLatencyDistribution,
|
||||
}
|
||||
|
||||
// GRPCClientResponseSizeView is an OpenCensus view which tracks GRPC Client response size by pomerium service, target host, grpc service, grpc method, and statu
|
||||
// GRPCClientResponseSizeView is an OpenCensus view which tracks GRPC Client response size by pomerium service, target host, grpc service, grpc method, and status
|
||||
GRPCClientResponseSizeView = &view.View{
|
||||
Name: grpcClientResponseSize.Name(),
|
||||
Measure: grpcClientResponseSize,
|
||||
Description: grpcClientResponseSize.Description(),
|
||||
TagKeys: []tag.Key{keyService, keyHost, keyMethod, keyStatus, keyGRPCService},
|
||||
Aggregation: view.Distribution(
|
||||
1, 256, 512, 1024, 2048, 8192, 16384, 32768, 65536, 131072, 262144, 524288,
|
||||
1048576, 2097152, 4194304, 8388608,
|
||||
),
|
||||
Name: "grpc_client_response_size_bytes",
|
||||
Measure: ocgrpc.ClientReceivedBytesPerRPC,
|
||||
Description: "grpc Client Response Size in bytes",
|
||||
TagKeys: []tag.Key{keyService, keyHost, keyGRPCMethod, keyGRPCService, ocgrpc.KeyClientStatus},
|
||||
Aggregation: grpcSizeDistribution,
|
||||
}
|
||||
|
||||
// GRPCClientRequestSizeView is an OpenCensus view which tracks GRPC Client request size by pomerium service, target host, grpc service, grpc method, and status
|
||||
GRPCClientRequestSizeView = &view.View{
|
||||
Name: "grpc_client_request_size_bytes",
|
||||
Measure: ocgrpc.ClientSentBytesPerRPC,
|
||||
Description: "grpc Client Request Size in bytes",
|
||||
TagKeys: []tag.Key{keyService, keyHost, keyGRPCMethod, keyGRPCService, ocgrpc.KeyClientStatus},
|
||||
Aggregation: grpcSizeDistribution,
|
||||
}
|
||||
)
|
||||
|
||||
// GRPCClientInterceptor creates a UnaryClientInterceptor which tracks metrics of grpc client requests
|
||||
// GRPCClientInterceptor creates a UnaryClientInterceptor which updates the RPC context with metric tag
|
||||
// metadata
|
||||
func GRPCClientInterceptor(service string) grpc.UnaryClientInterceptor {
|
||||
return func(
|
||||
ctx context.Context,
|
||||
|
|
@ -107,11 +107,6 @@ func GRPCClientInterceptor(service string) grpc.UnaryClientInterceptor {
|
|||
invoker grpc.UnaryInvoker,
|
||||
opts ...grpc.CallOption) error {
|
||||
|
||||
startTime := time.Now()
|
||||
|
||||
// Calls the invoker to execute RPC
|
||||
err := invoker(ctx, method, req, reply, cc, opts...)
|
||||
|
||||
// Split the method into parts for better slicing
|
||||
rpcInfo := strings.SplitN(method, "/", 3)
|
||||
var rpcMethod string
|
||||
|
|
@ -121,30 +116,62 @@ func GRPCClientInterceptor(service string) grpc.UnaryClientInterceptor {
|
|||
rpcMethod = rpcInfo[2]
|
||||
}
|
||||
|
||||
responseStatus, _ := status.FromError(err)
|
||||
ctx, tagErr := tag.New(
|
||||
context.Background(),
|
||||
taggedCtx, tagErr := tag.New(
|
||||
ctx,
|
||||
tag.Insert(keyService, service),
|
||||
tag.Insert(keyHost, cc.Target()),
|
||||
tag.Insert(keyMethod, rpcMethod),
|
||||
tag.Insert(keyGRPCMethod, rpcMethod),
|
||||
tag.Insert(keyGRPCService, rpcService),
|
||||
tag.Insert(keyStatus, responseStatus.Code().String()),
|
||||
)
|
||||
|
||||
if tagErr != nil {
|
||||
log.Warn().Err(tagErr).Str("context", "HTTPMetricsRoundTripper").Msg("Failed to create context tag")
|
||||
} else {
|
||||
responseProto := reply.(proto.Message)
|
||||
responseSize := proto.Size(responseProto)
|
||||
|
||||
stats.Record(ctx,
|
||||
grpcClientRequestCount.M(1),
|
||||
grpcClientRequestDuration.M(time.Since(startTime).Nanoseconds()/int64(time.Millisecond)),
|
||||
grpcClientResponseSize.M(int64(responseSize)),
|
||||
)
|
||||
log.Warn().Err(tagErr).Str("context", "GRPCClientInterceptor").Msg("internal/metrics: Failed to create context")
|
||||
return invoker(ctx, method, req, reply, cc, opts...)
|
||||
}
|
||||
|
||||
return err
|
||||
// Calls the invoker to execute RPC
|
||||
return invoker(taggedCtx, method, req, reply, cc, opts...)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// GRPCServerStatsHandler provides a grpc stats.Handler for a pomerium service to add tags and track
|
||||
// metrics to server side calls
|
||||
type GRPCServerStatsHandler struct {
|
||||
service string
|
||||
grpcstats.Handler
|
||||
}
|
||||
|
||||
// TagRPC implements grpc.stats.Handler and adds tags to the context of a given RPC
|
||||
func (h *GRPCServerStatsHandler) TagRPC(ctx context.Context, tagInfo *grpcstats.RPCTagInfo) context.Context {
|
||||
|
||||
handledCtx := h.Handler.TagRPC(ctx, tagInfo)
|
||||
|
||||
// Split the method into parts for better slicing
|
||||
rpcInfo := strings.SplitN(tagInfo.FullMethodName, "/", 3)
|
||||
var rpcMethod string
|
||||
var rpcService string
|
||||
if len(rpcInfo) == 3 {
|
||||
rpcService = rpcInfo[1]
|
||||
rpcMethod = rpcInfo[2]
|
||||
}
|
||||
|
||||
taggedCtx, tagErr := tag.New(
|
||||
handledCtx,
|
||||
tag.Insert(keyService, h.service),
|
||||
tag.Insert(keyGRPCMethod, rpcMethod),
|
||||
tag.Insert(keyGRPCService, rpcService),
|
||||
)
|
||||
if tagErr != nil {
|
||||
log.Warn().Err(tagErr).Str("context", "GRPCServerStatsHandler").Msg("internal/metrics: Failed to create context")
|
||||
return handledCtx
|
||||
|
||||
}
|
||||
|
||||
return taggedCtx
|
||||
}
|
||||
|
||||
// NewGRPCServerStatsHandler creates a new GRPCServerStatsHandler for a pomerium service
|
||||
func NewGRPCServerStatsHandler(service string) grpcstats.Handler {
|
||||
|
||||
return &GRPCServerStatsHandler{service: service, Handler: &ocgrpc.ServerHandler{}}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,8 +4,9 @@ import (
|
|||
"context"
|
||||
"testing"
|
||||
|
||||
"go.opencensus.io/stats/view"
|
||||
"go.opencensus.io/plugin/ocgrpc"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/stats"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
|
|
@ -29,11 +30,18 @@ func (t testProto) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
|
|||
|
||||
type testInvoker struct {
|
||||
invokeResult error
|
||||
statsHandler stats.Handler
|
||||
}
|
||||
|
||||
func (t testInvoker) UnaryInvoke(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, opts ...grpc.CallOption) error {
|
||||
r := reply.(*testProto)
|
||||
r.message = "hello"
|
||||
|
||||
ctx = t.statsHandler.TagRPC(ctx, &stats.RPCTagInfo{FullMethodName: method})
|
||||
t.statsHandler.HandleRPC(ctx, &stats.InPayload{Client: true, Length: len(r.message)})
|
||||
t.statsHandler.HandleRPC(ctx, &stats.OutPayload{Client: true, Length: len(r.message)})
|
||||
t.statsHandler.HandleRPC(ctx, &stats.End{Client: true, Error: t.invokeResult})
|
||||
|
||||
return t.invokeResult
|
||||
}
|
||||
|
||||
|
|
@ -55,49 +63,121 @@ func Test_GRPCClientInterceptor(t *testing.T) {
|
|||
wantgrpcClientResponseSize string
|
||||
wantgrpcClientRequestDuration string
|
||||
wantgrpcClientRequestCount string
|
||||
wantgrpcClientRequestSize string
|
||||
}{
|
||||
{
|
||||
name: "ok authorize",
|
||||
method: "/authorize.Authorizer/Authorize",
|
||||
errorCode: nil,
|
||||
wantgrpcClientResponseSize: "{ { {grpc_service authorize.Authorizer}{host dns:localhost:9999}{method Authorize}{service test_service}{status OK} }&{1 5 5 5 0 [0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]",
|
||||
wantgrpcClientRequestDuration: "{ { {grpc_service authorize.Authorizer}{host dns:localhost:9999}{method Authorize}{service test_service}{status OK} }&{1",
|
||||
wantgrpcClientRequestCount: "{ { {grpc_service authorize.Authorizer}{host dns:localhost:9999}{method Authorize}{service test_service}{status OK} }&{1",
|
||||
wantgrpcClientResponseSize: "{ { {grpc_client_status OK}{grpc_method Authorize}{grpc_service authorize.Authorizer}{host dns:localhost:9999}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
|
||||
wantgrpcClientRequestDuration: "{ { {grpc_client_status OK}{grpc_method Authorize}{grpc_service authorize.Authorizer}{host dns:localhost:9999}{service test_service} }&{1",
|
||||
wantgrpcClientRequestCount: "{ { {grpc_client_status OK}{grpc_method Authorize}{grpc_service authorize.Authorizer}{host dns:localhost:9999}{service test_service} }&{1",
|
||||
wantgrpcClientRequestSize: "{ { {grpc_client_status OK}{grpc_method Authorize}{grpc_service authorize.Authorizer}{host dns:localhost:9999}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
|
||||
},
|
||||
{
|
||||
name: "unknown validate",
|
||||
method: "/authenticate.Authenticator/Validate",
|
||||
errorCode: status.Error(14, ""),
|
||||
wantgrpcClientResponseSize: "{ { {grpc_service authenticate.Authenticator}{host dns:localhost:9999}{method Validate}{service test_service}{status Unavailable} }&{1 5 5 5 0 [0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]",
|
||||
wantgrpcClientRequestDuration: "{ { {grpc_service authenticate.Authenticator}{host dns:localhost:9999}{method Validate}{service test_service}{status Unavailable} }&{1",
|
||||
wantgrpcClientRequestCount: "{ { {grpc_service authenticate.Authenticator}{host dns:localhost:9999}{method Validate}{service test_service}{status Unavailable} }&{1",
|
||||
wantgrpcClientResponseSize: "{ { {grpc_client_status UNAVAILABLE}{grpc_method Validate}{grpc_service authenticate.Authenticator}{host dns:localhost:9999}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
|
||||
wantgrpcClientRequestDuration: "{ { {grpc_client_status UNAVAILABLE}{grpc_method Validate}{grpc_service authenticate.Authenticator}{host dns:localhost:9999}{service test_service} }&{1",
|
||||
wantgrpcClientRequestCount: "{ { {grpc_client_status UNAVAILABLE}{grpc_method Validate}{grpc_service authenticate.Authenticator}{host dns:localhost:9999}{service test_service} }&{1",
|
||||
wantgrpcClientRequestSize: "{ { {grpc_client_status UNAVAILABLE}{grpc_method Validate}{grpc_service authenticate.Authenticator}{host dns:localhost:9999}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
|
||||
},
|
||||
{
|
||||
name: "broken method parsing",
|
||||
method: "f",
|
||||
errorCode: status.Error(14, ""),
|
||||
wantgrpcClientResponseSize: "{ { {host dns:localhost:9999}{service test_service}{status Unavailable} }&{1 5 5 5 0 [0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]",
|
||||
wantgrpcClientRequestDuration: "{ { {host dns:localhost:9999}{service test_service}{status Unavailable} }&{1",
|
||||
wantgrpcClientRequestCount: "{ { {host dns:localhost:9999}{service test_service}{status Unavailable} }&{1",
|
||||
wantgrpcClientResponseSize: "{ { {grpc_client_status UNAVAILABLE}{host dns:localhost:9999}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
|
||||
wantgrpcClientRequestDuration: "{ { {grpc_client_status UNAVAILABLE}{host dns:localhost:9999}{service test_service} }&{1",
|
||||
wantgrpcClientRequestCount: "{ { {grpc_client_status UNAVAILABLE}{host dns:localhost:9999}{service test_service} }&{1",
|
||||
wantgrpcClientRequestSize: "{ { {grpc_client_status UNAVAILABLE}{host dns:localhost:9999}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
view.Unregister(GRPCClientRequestCountView, GRPCClientRequestDurationView, GRPCClientResponseSizeView)
|
||||
view.Register(GRPCClientRequestCountView, GRPCClientRequestDurationView, GRPCClientResponseSizeView)
|
||||
UnRegisterView(GRPCClientViews)
|
||||
RegisterView(GRPCClientViews)
|
||||
|
||||
invoker := testInvoker{
|
||||
invokeResult: tt.errorCode,
|
||||
statsHandler: &ocgrpc.ClientHandler{},
|
||||
}
|
||||
var reply testProto
|
||||
|
||||
interceptor(context.Background(), tt.method, nil, &reply, newTestCC(t), invoker.UnaryInvoke)
|
||||
|
||||
testDataRetrieval(grpcClientResponseSize, t, tt.wantgrpcClientResponseSize)
|
||||
testDataRetrieval(grpcClientRequestDuration, t, tt.wantgrpcClientRequestDuration)
|
||||
testDataRetrieval(grpcClientRequestCount, t, tt.wantgrpcClientRequestCount)
|
||||
testDataRetrieval(GRPCClientResponseSizeView, t, tt.wantgrpcClientResponseSize)
|
||||
testDataRetrieval(GRPCClientRequestDurationView, t, tt.wantgrpcClientRequestDuration)
|
||||
testDataRetrieval(GRPCClientRequestCountView, t, tt.wantgrpcClientRequestCount)
|
||||
testDataRetrieval(GRPCClientRequestSizeView, t, tt.wantgrpcClientRequestSize)
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func mockServerRPCHandle(statsHandler stats.Handler, method string, errorCode error) {
|
||||
message := "hello"
|
||||
ctx := statsHandler.TagRPC(context.Background(), &stats.RPCTagInfo{FullMethodName: method})
|
||||
statsHandler.HandleRPC(ctx, &stats.InPayload{Client: false, Length: len(message)})
|
||||
statsHandler.HandleRPC(ctx, &stats.OutPayload{Client: false, Length: len(message)})
|
||||
statsHandler.HandleRPC(ctx, &stats.End{Client: false, Error: errorCode})
|
||||
|
||||
}
|
||||
func Test_GRPCServerStatsHandler(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
method string
|
||||
errorCode error
|
||||
wantgrpcServerResponseSize string
|
||||
wantgrpcServerRequestDuration string
|
||||
wantgrpcServerRequestCount string
|
||||
wantgrpcServerRequestSizeView string
|
||||
}{
|
||||
{
|
||||
name: "ok authorize",
|
||||
method: "/authorize.Authorizer/Authorize",
|
||||
errorCode: nil,
|
||||
wantgrpcServerResponseSize: "{ { {grpc_method Authorize}{grpc_server_status OK}{grpc_service authorize.Authorizer}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
|
||||
wantgrpcServerRequestDuration: "{ { {grpc_method Authorize}{grpc_server_status OK}{grpc_service authorize.Authorizer}{service test_service} }&{1",
|
||||
wantgrpcServerRequestCount: "{ { {grpc_method Authorize}{grpc_server_status OK}{grpc_service authorize.Authorizer}{service test_service} }&{1",
|
||||
wantgrpcServerRequestSizeView: "{ { {grpc_method Authorize}{grpc_server_status OK}{grpc_service authorize.Authorizer}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
|
||||
},
|
||||
{
|
||||
name: "unknown validate",
|
||||
method: "/authenticate.Authenticator/Validate",
|
||||
errorCode: status.Error(14, ""),
|
||||
wantgrpcServerResponseSize: "{ { {grpc_method Validate}{grpc_server_status UNAVAILABLE}{grpc_service authenticate.Authenticator}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
|
||||
wantgrpcServerRequestDuration: "{ { {grpc_method Validate}{grpc_server_status UNAVAILABLE}{grpc_service authenticate.Authenticator}{service test_service} }&{1",
|
||||
wantgrpcServerRequestCount: "{ { {grpc_method Validate}{grpc_server_status UNAVAILABLE}{grpc_service authenticate.Authenticator}{service test_service} }&{1",
|
||||
wantgrpcServerRequestSizeView: "{ { {grpc_method Validate}{grpc_server_status UNAVAILABLE}{grpc_service authenticate.Authenticator}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
|
||||
},
|
||||
{
|
||||
name: "broken method parsing",
|
||||
method: "f",
|
||||
errorCode: status.Error(14, ""),
|
||||
wantgrpcServerResponseSize: "{ { {grpc_server_status UNAVAILABLE}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
|
||||
wantgrpcServerRequestDuration: "{ { {grpc_server_status UNAVAILABLE}{service test_service} }&{1",
|
||||
wantgrpcServerRequestCount: "{ { {grpc_server_status UNAVAILABLE}{service test_service} }&{1",
|
||||
wantgrpcServerRequestSizeView: "{ { {grpc_server_status UNAVAILABLE}{service test_service} }&{1 5 5 5 0 [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
UnRegisterView(GRPCServerViews)
|
||||
RegisterView(GRPCServerViews)
|
||||
|
||||
statsHandler := NewGRPCServerStatsHandler("test_service")
|
||||
mockServerRPCHandle(statsHandler, tt.method, tt.errorCode)
|
||||
|
||||
testDataRetrieval(GRPCServerResponseSizeView, t, tt.wantgrpcServerResponseSize)
|
||||
testDataRetrieval(GRPCServerRequestDurationView, t, tt.wantgrpcServerRequestDuration)
|
||||
testDataRetrieval(GRPCServerRequestCountView, t, tt.wantgrpcServerRequestCount)
|
||||
testDataRetrieval(GRPCServerRequestSizeView, t, tt.wantgrpcServerRequestSizeView)
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -70,9 +70,9 @@ func Test_HTTPMetricsHandler(t *testing.T) {
|
|||
rec := httptest.NewRecorder()
|
||||
chainHandler.ServeHTTP(rec, req)
|
||||
|
||||
testDataRetrieval(httpServerResponseSize, t, tt.wanthttpServerResponseSize)
|
||||
testDataRetrieval(httpServerRequestDuration, t, tt.wanthttpServerRequestDuration)
|
||||
testDataRetrieval(httpServerRequestCount, t, tt.wanthttpServerRequestCount)
|
||||
testDataRetrieval(HTTPServerRequestSizeView, t, tt.wanthttpServerResponseSize)
|
||||
testDataRetrieval(HTTPServerRequestDurationView, t, tt.wanthttpServerRequestDuration)
|
||||
testDataRetrieval(HTTPServerRequestCountView, t, tt.wanthttpServerRequestCount)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -144,9 +144,9 @@ func Test_HTTPMetricsRoundTripper(t *testing.T) {
|
|||
resp, err := client.Do(req)
|
||||
|
||||
t.Logf("response: %#v, %#v", resp, err)
|
||||
testDataRetrieval(httpClientResponseSize, t, tt.wanthttpClientResponseSize)
|
||||
testDataRetrieval(httpClientRequestDuration, t, tt.wanthttpClientRequestDuration)
|
||||
testDataRetrieval(httpClientRequestCount, t, tt.wanthttpClientRequestCount)
|
||||
testDataRetrieval(HTTPClientResponseSizeView, t, tt.wanthttpClientResponseSize)
|
||||
testDataRetrieval(HTTPClientRequestDurationView, t, tt.wanthttpClientRequestDuration)
|
||||
testDataRetrieval(HTTPClientRequestCountView, t, tt.wanthttpClientRequestCount)
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -9,5 +9,6 @@ var (
|
|||
keyStatus tag.Key = tag.MustNewKey("status")
|
||||
keyService tag.Key = tag.MustNewKey("service")
|
||||
keyGRPCService tag.Key = tag.MustNewKey("grpc_service")
|
||||
keyGRPCMethod tag.Key = tag.MustNewKey("grpc_method")
|
||||
keyHost tag.Key = tag.MustNewKey("host")
|
||||
)
|
||||
|
|
|
|||
|
|
@ -5,26 +5,26 @@ import (
|
|||
"go.opencensus.io/stats/view"
|
||||
)
|
||||
|
||||
// RegisterHTTPClientView registers the standard HTTPClient view.
|
||||
// It must be called to see metrics in the configured exporters
|
||||
func RegisterHTTPClientView() {
|
||||
if err := view.Register(HTTPClientRequestCountView, HTTPClientRequestDurationView, HTTPClientResponseSizeView); err != nil {
|
||||
log.Warn().Err(err).Msg("Could not register HTTPClientView")
|
||||
var (
|
||||
// HTTPClientViews contains opencensus views for HTTP Client metrics
|
||||
HTTPClientViews = []*view.View{HTTPClientRequestCountView, HTTPClientRequestDurationView, HTTPClientResponseSizeView}
|
||||
// HTTPServerViews contains opencensus views for HTTP Server metrics
|
||||
HTTPServerViews = []*view.View{HTTPServerRequestCountView, HTTPServerRequestDurationView, HTTPServerRequestSizeView}
|
||||
// GRPCClientViews contains opencensus views for GRPC Client metrics
|
||||
GRPCClientViews = []*view.View{GRPCClientRequestCountView, GRPCClientRequestDurationView, GRPCClientResponseSizeView, GRPCClientRequestSizeView}
|
||||
// GRPCServerViews contains opencensus views for GRPC Server metrics
|
||||
GRPCServerViews = []*view.View{GRPCServerRequestCountView, GRPCServerRequestDurationView, GRPCServerResponseSizeView, GRPCServerRequestSizeView}
|
||||
)
|
||||
|
||||
// RegisterView registers one of the defined metrics views. It must be called for metrics to see metrics
|
||||
// in the configured exporters
|
||||
func RegisterView(v []*view.View) {
|
||||
if err := view.Register(v...); err != nil {
|
||||
log.Warn().Str("context", "RegisterView").Err(err).Msg("internal/metrics: Could not register view")
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterHTTPServerView registers the standard HTTPServer view.
|
||||
// It must be called to see metrics in the configured exporters
|
||||
func RegisterHTTPServerView() {
|
||||
if err := view.Register(HTTPServerRequestCountView, HTTPServerRequestDurationView, HTTPServerRequestSizeView); err != nil {
|
||||
log.Warn().Err(err).Msg("Could not register HTTPServerView")
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterGRPCClientView registers the standard GRPCClient view.
|
||||
// It must be called to see metrics in the configured exporters
|
||||
func RegisterGRPCClientView() {
|
||||
if err := view.Register(GRPCClientRequestCountView, GRPCClientRequestDurationView, GRPCClientResponseSizeView); err != nil {
|
||||
log.Warn().Err(err).Msg("Could not register GRPCClientView")
|
||||
}
|
||||
// UnRegisterView unregisters one of the defined metrics views.
|
||||
func UnRegisterView(v []*view.View) {
|
||||
view.Unregister(v...)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,29 +6,20 @@ import (
|
|||
"go.opencensus.io/stats/view"
|
||||
)
|
||||
|
||||
func Test_RegisterHTTPClientView(t *testing.T) {
|
||||
RegisterHTTPClientView()
|
||||
for _, v := range []*view.View{HTTPClientRequestCountView, HTTPClientRequestDurationView, HTTPClientResponseSizeView} {
|
||||
func Test_RegisterView(t *testing.T) {
|
||||
RegisterView(HTTPClientViews)
|
||||
for _, v := range HTTPClientViews {
|
||||
if view.Find(v.Name) != v {
|
||||
t.Errorf("Failed to find registered view %s", v.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func Test_RegisterHTTPServerView(t *testing.T) {
|
||||
RegisterHTTPServerView()
|
||||
for _, v := range []*view.View{HTTPServerRequestCountView, HTTPServerRequestDurationView, HTTPServerRequestSizeView} {
|
||||
if view.Find(v.Name) != v {
|
||||
t.Errorf("Failed to find registered view %s", v.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func Test_RegisterGRPCClientView(t *testing.T) {
|
||||
RegisterGRPCClientView()
|
||||
for _, v := range []*view.View{GRPCClientRequestCountView, GRPCClientRequestDurationView, GRPCClientResponseSizeView} {
|
||||
if view.Find(v.Name) != v {
|
||||
t.Errorf("Failed to find registered view %s", v.Name)
|
||||
func Test_UnregisterView(t *testing.T) {
|
||||
UnRegisterView(HTTPClientViews)
|
||||
for _, v := range HTTPClientViews {
|
||||
if view.Find(v.Name) == v {
|
||||
t.Errorf("Found unregistered view %s", v.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import (
|
|||
"github.com/pomerium/pomerium/internal/log"
|
||||
"github.com/pomerium/pomerium/internal/metrics"
|
||||
"github.com/pomerium/pomerium/internal/middleware"
|
||||
|
||||
"go.opencensus.io/plugin/ocgrpc"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials"
|
||||
)
|
||||
|
|
@ -106,5 +106,6 @@ func NewGRPCClientConn(opts *Options) (*grpc.ClientConn, error) {
|
|||
grpc.WithTransportCredentials(cert),
|
||||
grpc.WithPerRPCCredentials(grpcAuth),
|
||||
grpc.WithUnaryInterceptor(metrics.GRPCClientInterceptor("proxy")),
|
||||
grpc.WithStatsHandler(&ocgrpc.ClientHandler{}),
|
||||
)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue