diff --git a/authenticate/authenticate.go b/authenticate/authenticate.go index 79c6fdf01..7cf18f0ab 100644 --- a/authenticate/authenticate.go +++ b/authenticate/authenticate.go @@ -10,7 +10,9 @@ import ( "github.com/pomerium/pomerium/config" "github.com/pomerium/pomerium/internal/atomicutil" "github.com/pomerium/pomerium/internal/log" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/pkg/cryptutil" + oteltrace "go.opentelemetry.io/otel/trace" ) // ValidateOptions checks that configuration are complete and valid. @@ -38,23 +40,31 @@ func ValidateOptions(o *config.Options) error { // Authenticate contains data required to run the authenticate service. type Authenticate struct { - cfg *authenticateConfig - options *atomicutil.Value[*config.Options] - state *atomicutil.Value[*authenticateState] + cfg *authenticateConfig + options *atomicutil.Value[*config.Options] + state *atomicutil.Value[*authenticateState] + tracerProvider oteltrace.TracerProvider + tracer oteltrace.Tracer } // New validates and creates a new authenticate service from a set of Options. func New(ctx context.Context, cfg *config.Config, options ...Option) (*Authenticate, error) { authenticateConfig := getAuthenticateConfig(options...) + + tracerProvider := trace.NewTracerProvider(ctx, "Authenticate") + tracer := tracerProvider.Tracer(trace.PomeriumCoreTracer) + a := &Authenticate{ - cfg: authenticateConfig, - options: config.NewAtomicOptions(), - state: atomicutil.NewValue(newAuthenticateState()), + cfg: authenticateConfig, + options: config.NewAtomicOptions(), + state: atomicutil.NewValue(newAuthenticateState()), + tracerProvider: tracerProvider, + tracer: tracer, } a.options.Store(cfg.Options) - state, err := newAuthenticateStateFromConfig(ctx, cfg, authenticateConfig) + state, err := newAuthenticateStateFromConfig(ctx, tracerProvider, cfg, authenticateConfig) if err != nil { return nil, err } @@ -70,7 +80,7 @@ func (a *Authenticate) OnConfigChange(ctx context.Context, cfg *config.Config) { } a.options.Store(cfg.Options) - if state, err := newAuthenticateStateFromConfig(ctx, cfg, a.cfg); err != nil { + if state, err := newAuthenticateStateFromConfig(ctx, a.tracerProvider, cfg, a.cfg); err != nil { log.Ctx(ctx).Error().Err(err).Msg("authenticate: failed to update state") } else { a.state.Store(state) diff --git a/authenticate/config.go b/authenticate/config.go index ef293808b..11ce67bee 100644 --- a/authenticate/config.go +++ b/authenticate/config.go @@ -1,14 +1,17 @@ package authenticate import ( + "context" + "github.com/pomerium/pomerium/authenticate/events" "github.com/pomerium/pomerium/config" identitypb "github.com/pomerium/pomerium/pkg/grpc/identity" "github.com/pomerium/pomerium/pkg/identity" + oteltrace "go.opentelemetry.io/otel/trace" ) type authenticateConfig struct { - getIdentityProvider func(options *config.Options, idpID string) (identity.Authenticator, error) + getIdentityProvider func(ctx context.Context, tracerProvider oteltrace.TracerProvider, options *config.Options, idpID string) (identity.Authenticator, error) profileTrimFn func(*identitypb.Profile) authEventFn events.AuthEventFn } @@ -26,7 +29,7 @@ func getAuthenticateConfig(options ...Option) *authenticateConfig { } // WithGetIdentityProvider sets the getIdentityProvider function in the config. -func WithGetIdentityProvider(getIdentityProvider func(options *config.Options, idpID string) (identity.Authenticator, error)) Option { +func WithGetIdentityProvider(getIdentityProvider func(ctx context.Context, tracerProvider oteltrace.TracerProvider, options *config.Options, idpID string) (identity.Authenticator, error)) Option { return func(cfg *authenticateConfig) { cfg.getIdentityProvider = getIdentityProvider } diff --git a/authenticate/handlers.go b/authenticate/handlers.go index f394b3f1b..72f010f88 100644 --- a/authenticate/handlers.go +++ b/authenticate/handlers.go @@ -3,6 +3,7 @@ package authenticate import ( "context" "encoding/base64" + "encoding/hex" "errors" "fmt" "net/http" @@ -13,6 +14,9 @@ import ( "github.com/google/uuid" "github.com/gorilla/mux" "github.com/rs/cors" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel/attribute" + oteltrace "go.opentelemetry.io/otel/trace" "github.com/pomerium/csrf" "github.com/pomerium/pomerium/internal/authenticateflow" @@ -58,6 +62,7 @@ func (a *Authenticate) Mount(r *mux.Router) { } return csrf.Protect(state.cookieSecret, csrfOptions...)(h) }) + r.Use(trace.NewHTTPMiddleware(otelhttp.WithTracerProvider(a.tracerProvider))) // redirect / to /.pomerium/ r.Path("/").Handler(http.RedirectHandler("/.pomerium/", http.StatusFound)) @@ -114,7 +119,7 @@ func (a *Authenticate) RetrieveSession(next http.Handler) http.Handler { // session state is attached to the users's request context. func (a *Authenticate) VerifySession(next http.Handler) http.Handler { return httputil.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { - ctx, span := trace.StartSpan(r.Context(), "authenticate.VerifySession") + ctx, span := a.tracer.Start(r.Context(), "authenticate.VerifySession") defer span.End() state := a.state.Load() @@ -126,6 +131,8 @@ func (a *Authenticate) VerifySession(next http.Handler) http.Handler { Err(err). Str("idp_id", idpID). Msg("authenticate: session load error") + span.AddEvent("session load error", + oteltrace.WithAttributes(attribute.String("error", err.Error()))) return a.reauthenticateOrFail(w, r, err) } @@ -135,6 +142,7 @@ func (a *Authenticate) VerifySession(next http.Handler) http.Handler { Str("session_idp_id", sessionState.IdentityProviderID). Str("id", sessionState.ID). Msg("authenticate: session not associated with identity provider") + span.AddEvent("session not associated with identity provider") return a.reauthenticateOrFail(w, r, err) } @@ -143,6 +151,8 @@ func (a *Authenticate) VerifySession(next http.Handler) http.Handler { Err(err). Str("idp_id", idpID). Msg("authenticate: couldn't verify session") + span.AddEvent("couldn't verify session", + oteltrace.WithAttributes(attribute.String("error", err.Error()))) return a.reauthenticateOrFail(w, r, err) } @@ -160,7 +170,7 @@ func (a *Authenticate) RobotsTxt(w http.ResponseWriter, _ *http.Request) { // SignIn handles authenticating a user. func (a *Authenticate) SignIn(w http.ResponseWriter, r *http.Request) error { - ctx, span := trace.StartSpan(r.Context(), "authenticate.SignIn") + ctx, span := a.tracer.Start(r.Context(), "authenticate.SignIn") defer span.End() state := a.state.Load() @@ -197,13 +207,13 @@ func (a *Authenticate) SignOut(w http.ResponseWriter, r *http.Request) error { } func (a *Authenticate) signOutRedirect(w http.ResponseWriter, r *http.Request) error { - ctx, span := trace.StartSpan(r.Context(), "authenticate.SignOut") + ctx, span := a.tracer.Start(r.Context(), "authenticate.SignOut") defer span.End() options := a.options.Load() idpID := a.getIdentityProviderIDForRequest(r) - authenticator, err := a.cfg.getIdentityProvider(options, idpID) + authenticator, err := a.cfg.getIdentityProvider(ctx, a.tracerProvider, options, idpID) if err != nil { return err } @@ -274,7 +284,7 @@ func (a *Authenticate) reauthenticateOrFail(w http.ResponseWriter, r *http.Reque options := a.options.Load() idpID := a.getIdentityProviderIDForRequest(r) - authenticator, err := a.cfg.getIdentityProvider(options, idpID) + authenticator, err := a.cfg.getIdentityProvider(r.Context(), a.tracerProvider, options, idpID) if err != nil { return err } @@ -283,9 +293,20 @@ func (a *Authenticate) reauthenticateOrFail(w http.ResponseWriter, r *http.Reque state.sessionStore.ClearSession(w, r) redirectURL := state.redirectURL.ResolveReference(r.URL) + redirectURLValues := redirectURL.Query() + var traceID string + if tp := trace.PomeriumURLQueryCarrier(redirectURLValues).Get("traceparent"); len(tp) == 55 { + if traceIDBytes, err := hex.DecodeString(tp[3:35]); err == nil { + traceFlags, _ := hex.DecodeString(tp[53:55]) + if len(traceFlags) != 1 { + traceFlags = []byte{0} + } + traceID = base64.RawURLEncoding.EncodeToString(append(traceIDBytes, traceFlags[0])) + } + } nonce := csrf.Token(r) now := time.Now().Unix() - b := []byte(fmt.Sprintf("%s|%d|", nonce, now)) + b := []byte(fmt.Sprintf("%s|%d|%s|", nonce, now, traceID)) enc := cryptutil.Encrypt(state.cookieCipher, []byte(redirectURL.String()), b) b = append(b, enc...) encodedState := base64.URLEncoding.EncodeToString(b) @@ -321,7 +342,7 @@ func (a *Authenticate) statusForErrorCode(errorCode string) int { } func (a *Authenticate) getOAuthCallback(w http.ResponseWriter, r *http.Request) (*url.URL, error) { - ctx, span := trace.StartSpan(r.Context(), "authenticate.getOAuthCallback") + ctx, span := a.tracer.Start(r.Context(), "authenticate.OAuthCallback") defer span.End() state := a.state.Load() @@ -347,21 +368,20 @@ func (a *Authenticate) getOAuthCallback(w http.ResponseWriter, r *http.Request) } // split state into concat'd components - // (nonce|timestamp|redirect_url|encrypted_data(redirect_url)+mac(nonce,ts)) - statePayload := strings.SplitN(string(bytes), "|", 3) - if len(statePayload) != 3 { + // (nonce|timestamp|trace_id+flags|encrypted_data(redirect_url)+mac(nonce,ts)) + statePayload := strings.SplitN(string(bytes), "|", 4) + if len(statePayload) != 4 { return nil, httputil.NewError(http.StatusBadRequest, fmt.Errorf("state malformed, size: %d", len(statePayload))) } // Use our AEAD construct to enforce secrecy and authenticity: // mac: to validate the nonce again, and above timestamp // decrypt: to prevent leaking 'redirect_uri' to IdP or logs - b := []byte(fmt.Sprint(statePayload[0], "|", statePayload[1], "|")) - redirectString, err := cryptutil.Decrypt(state.cookieCipher, []byte(statePayload[2]), b) + b := []byte(fmt.Sprint(statePayload[0], "|", statePayload[1], "|", statePayload[2], "|")) + redirectString, err := cryptutil.Decrypt(state.cookieCipher, []byte(statePayload[3]), b) if err != nil { return nil, httputil.NewError(http.StatusBadRequest, err) } - redirectURL, err := urlutil.ParseAndValidateURL(string(redirectString)) if err != nil { return nil, httputil.NewError(http.StatusBadRequest, err) @@ -380,7 +400,7 @@ Or contact your administrator. idpID := state.flow.GetIdentityProviderIDForURLValues(redirectURL.Query()) - authenticator, err := a.cfg.getIdentityProvider(options, idpID) + authenticator, err := a.cfg.getIdentityProvider(ctx, a.tracerProvider, options, idpID) if err != nil { return nil, err } @@ -432,7 +452,7 @@ func (a *Authenticate) getSessionFromCtx(ctx context.Context) (*sessions.State, } func (a *Authenticate) userInfo(w http.ResponseWriter, r *http.Request) error { - ctx, span := trace.StartSpan(r.Context(), "authenticate.userInfo") + ctx, span := a.tracer.Start(r.Context(), "authenticate.userInfo") defer span.End() options := a.options.Load() @@ -484,7 +504,7 @@ func (a *Authenticate) revokeSession(ctx context.Context, w http.ResponseWriter, idpID := r.FormValue(urlutil.QueryIdentityProviderID) - authenticator, err := a.cfg.getIdentityProvider(options, idpID) + authenticator, err := a.cfg.getIdentityProvider(ctx, a.tracerProvider, options, idpID) if err != nil { return "" } diff --git a/authenticate/handlers_test.go b/authenticate/handlers_test.go index c8f5342a5..b9ae56066 100644 --- a/authenticate/handlers_test.go +++ b/authenticate/handlers_test.go @@ -16,6 +16,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + oteltrace "go.opentelemetry.io/otel/trace" "go.uber.org/mock/gomock" "golang.org/x/crypto/chacha20poly1305" "golang.org/x/oauth2" @@ -39,23 +40,21 @@ import ( "github.com/pomerium/pomerium/pkg/identity/oidc" ) -func testAuthenticate() *Authenticate { - redirectURL, _ := url.Parse("https://auth.example.com/oauth/callback") - var auth Authenticate - auth.state = atomicutil.NewValue(&authenticateState{ - redirectURL: redirectURL, - cookieSecret: cryptutil.NewKey(), - flow: new(stubFlow), +func testAuthenticate(t *testing.T) *Authenticate { + opts := newTestOptions(t) + opts.AuthenticateURLString = "https://auth.example.com/oauth/callback" + auth, err := New(context.Background(), &config.Config{ + Options: opts, }) - auth.options = config.NewAtomicOptions() - auth.options.Store(&config.Options{ - SharedKey: cryptutil.NewBase64Key(), - }) - return &auth + if err != nil { + panic(err) + } + auth.state.Load().flow = new(stubFlow) + return auth } func TestAuthenticate_RobotsTxt(t *testing.T) { - auth := testAuthenticate() + auth := testAuthenticate(t) req, err := http.NewRequest(http.MethodGet, "/robots.txt", nil) if err != nil { t.Fatal(err) @@ -73,7 +72,7 @@ func TestAuthenticate_RobotsTxt(t *testing.T) { } func TestAuthenticate_Handler(t *testing.T) { - auth := testAuthenticate() + auth := testAuthenticate(t) h := auth.Handler() if h == nil { @@ -224,17 +223,16 @@ func TestAuthenticate_SignOut(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - a := &Authenticate{ - cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ *config.Options, _ string) (identity.Authenticator, error) { - return tt.provider, nil - })), - state: atomicutil.NewValue(&authenticateState{ - sessionStore: tt.sessionStore, - sharedEncoder: mock.Encoder{}, - flow: new(stubFlow), - }), - options: config.NewAtomicOptions(), - } + a := testAuthenticate(t) + a.cfg = getAuthenticateConfig(WithGetIdentityProvider(func(_ context.Context, _ oteltrace.TracerProvider, _ *config.Options, _ string) (identity.Authenticator, error) { + return tt.provider, nil + })) + a.state = atomicutil.NewValue(&authenticateState{ + sessionStore: tt.sessionStore, + sharedEncoder: mock.Encoder{}, + flow: new(stubFlow), + }) + a.options = config.NewAtomicOptions() if tt.signoutRedirectURL != "" { opts := a.options.Load() opts.SignOutRedirectURLString = tt.signoutRedirectURL @@ -280,7 +278,7 @@ func TestAuthenticate_SignOutDoesNotRequireSession(t *testing.T) { sessionStore := &mstore.Store{LoadError: errors.New("no session")} a := &Authenticate{ - cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ *config.Options, _ string) (identity.Authenticator, error) { + cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ context.Context, _ oteltrace.TracerProvider, _ *config.Options, _ string) (identity.Authenticator, error) { return identity.MockProvider{}, nil })), state: atomicutil.NewValue(&authenticateState{ @@ -354,26 +352,24 @@ func TestAuthenticate_OAuthCallback(t *testing.T) { t.Fatal(err) } authURL, _ := url.Parse(tt.authenticateURL) - a := &Authenticate{ - cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ *config.Options, _ string) (identity.Authenticator, error) { - return tt.provider, nil - })), - state: atomicutil.NewValue(&authenticateState{ - redirectURL: authURL, - sessionStore: tt.session, - cookieCipher: aead, - flow: new(stubFlow), - }), - options: config.NewAtomicOptions(), - } + a := testAuthenticate(t) + a.cfg = getAuthenticateConfig(WithGetIdentityProvider(func(_ context.Context, _ oteltrace.TracerProvider, _ *config.Options, _ string) (identity.Authenticator, error) { + return tt.provider, nil + })) + a.state = atomicutil.NewValue(&authenticateState{ + redirectURL: authURL, + sessionStore: tt.session, + cookieCipher: aead, + flow: new(stubFlow), + }) + a.options = config.NewAtomicOptions() u, _ := url.Parse("/oauthGet") params, _ := url.ParseQuery(u.RawQuery) params.Add("error", tt.paramErr) params.Add("code", tt.code) nonce := cryptutil.NewBase64Key() // mock csrf - // (nonce|timestamp|redirect_url|encrypt(redirect_url),mac(nonce,ts)) - b := []byte(fmt.Sprintf("%s|%d|%s", nonce, tt.ts, tt.extraMac)) - + // (nonce|timestamp|trace_id+flags|encrypt(redirect_url),mac(nonce,ts)) + b := []byte(fmt.Sprintf("%s|%d||%s", nonce, tt.ts, tt.extraMac)) enc := cryptutil.Encrypt(a.state.Load().cookieCipher, []byte(tt.redirectURI), b) b = append(b, enc...) encodedState := base64.URLEncoding.EncodeToString(b) @@ -466,20 +462,19 @@ func TestAuthenticate_SessionValidatorMiddleware(t *testing.T) { if err != nil { t.Fatal(err) } - a := &Authenticate{ - cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ *config.Options, _ string) (identity.Authenticator, error) { - return tt.provider, nil - })), - state: atomicutil.NewValue(&authenticateState{ - cookieSecret: cryptutil.NewKey(), - redirectURL: uriParseHelper("https://authenticate.corp.beyondperimeter.com"), - sessionStore: tt.session, - cookieCipher: aead, - sharedEncoder: signer, - flow: new(stubFlow), - }), - options: config.NewAtomicOptions(), - } + a := testAuthenticate(t) + a.cfg = getAuthenticateConfig(WithGetIdentityProvider(func(_ context.Context, _ oteltrace.TracerProvider, _ *config.Options, _ string) (identity.Authenticator, error) { + return tt.provider, nil + })) + a.state = atomicutil.NewValue(&authenticateState{ + cookieSecret: cryptutil.NewKey(), + redirectURL: uriParseHelper("https://authenticate.corp.beyondperimeter.com"), + sessionStore: tt.session, + cookieCipher: aead, + sharedEncoder: signer, + flow: new(stubFlow), + }) + a.options = config.NewAtomicOptions() r := httptest.NewRequest(http.MethodGet, "/", nil) state, err := tt.session.LoadSession(r) if err != nil { @@ -512,7 +507,7 @@ func TestAuthenticate_userInfo(t *testing.T) { t.Run("cookie-redirect-uri", func(t *testing.T) { w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "https://authenticate.service.cluster.local/.pomerium/?pomerium_redirect_uri=https://www.example.com", nil) - var a Authenticate + a := testAuthenticate(t) a.state = atomicutil.NewValue(&authenticateState{ cookieSecret: cryptutil.NewKey(), flow: new(stubFlow), @@ -577,14 +572,13 @@ func TestAuthenticate_userInfo(t *testing.T) { if !tt.validSignature { f.verifySignatureErr = errors.New("bad signature") } - a := &Authenticate{ - options: o, - state: atomicutil.NewValue(&authenticateState{ - sessionStore: tt.sessionStore, - sharedEncoder: signer, - flow: f, - }), - } + a := testAuthenticate(t) + a.options = o + a.state = atomicutil.NewValue(&authenticateState{ + sessionStore: tt.sessionStore, + sharedEncoder: signer, + flow: f, + }) r := httptest.NewRequest(http.MethodGet, tt.url, nil) state, err := tt.sessionStore.LoadSession(r) if err != nil { @@ -606,7 +600,7 @@ func TestAuthenticate_userInfo(t *testing.T) { func TestAuthenticate_CORS(t *testing.T) { f := new(stubFlow) - auth := testAuthenticate() + auth := testAuthenticate(t) state := auth.state.Load() state.sessionLoader = &mstore.Store{Session: &sessions.State{}} state.sharedEncoder = mock.Encoder{} @@ -645,7 +639,7 @@ func TestAuthenticate_CORS(t *testing.T) { func TestSignOutBranding(t *testing.T) { t.Parallel() - auth := testAuthenticate() + auth := testAuthenticate(t) auth.state.Load().flow.(*stubFlow).verifySignatureErr = errors.New("unsigned URL") auth.options.Store(&config.Options{ BrandingOptions: &configproto.Settings{ diff --git a/authenticate/identity.go b/authenticate/identity.go index 8ea432e15..1ff0efe79 100644 --- a/authenticate/identity.go +++ b/authenticate/identity.go @@ -1,13 +1,16 @@ package authenticate import ( + "context" + "github.com/pomerium/pomerium/config" "github.com/pomerium/pomerium/internal/urlutil" "github.com/pomerium/pomerium/pkg/identity" "github.com/pomerium/pomerium/pkg/identity/oauth" + oteltrace "go.opentelemetry.io/otel/trace" ) -func defaultGetIdentityProvider(options *config.Options, idpID string) (identity.Authenticator, error) { +func defaultGetIdentityProvider(ctx context.Context, tracerProvider oteltrace.TracerProvider, options *config.Options, idpID string) (identity.Authenticator, error) { authenticateURL, err := options.GetAuthenticateURL() if err != nil { return nil, err @@ -23,7 +26,7 @@ func defaultGetIdentityProvider(options *config.Options, idpID string) (identity if err != nil { return nil, err } - return identity.NewAuthenticator(oauth.Options{ + return identity.NewAuthenticator(ctx, tracerProvider, oauth.Options{ RedirectURL: redirectURL, ProviderName: idp.GetType(), ProviderURL: idp.GetUrl(), diff --git a/authenticate/state.go b/authenticate/state.go index 3680f0f46..522a0e842 100644 --- a/authenticate/state.go +++ b/authenticate/state.go @@ -8,6 +8,7 @@ import ( "net/url" "github.com/go-jose/go-jose/v3" + oteltrace "go.opentelemetry.io/otel/trace" "golang.org/x/oauth2" "github.com/pomerium/pomerium/config" @@ -65,7 +66,9 @@ func newAuthenticateState() *authenticateState { func newAuthenticateStateFromConfig( ctx context.Context, - cfg *config.Config, authenticateConfig *authenticateConfig, + tracerProvider oteltrace.TracerProvider, + cfg *config.Config, + authenticateConfig *authenticateConfig, ) (*authenticateState, error) { err := ValidateOptions(cfg.Options) if err != nil { @@ -147,6 +150,7 @@ func newAuthenticateStateFromConfig( if cfg.Options.UseStatelessAuthenticateFlow() { state.flow, err = authenticateflow.NewStateless(ctx, + tracerProvider, cfg, cookieStore, authenticateConfig.getIdentityProvider, @@ -154,7 +158,7 @@ func newAuthenticateStateFromConfig( authenticateConfig.authEventFn, ) } else { - state.flow, err = authenticateflow.NewStateful(ctx, cfg, cookieStore) + state.flow, err = authenticateflow.NewStateful(ctx, tracerProvider, cfg, cookieStore) } if err != nil { return nil, err diff --git a/authorize/authorize.go b/authorize/authorize.go index 5c608b1cb..4cd3dd496 100644 --- a/authorize/authorize.go +++ b/authorize/authorize.go @@ -21,6 +21,7 @@ import ( "github.com/pomerium/pomerium/pkg/cryptutil" "github.com/pomerium/pomerium/pkg/grpc/databroker" "github.com/pomerium/pomerium/pkg/storage" + oteltrace "go.opentelemetry.io/otel/trace" ) // Authorize struct holds @@ -35,18 +36,25 @@ type Authorize struct { // This should provide a consistent view of the data at a given server/record version and // avoid partial updates. stateLock sync.RWMutex + + tracerProvider oteltrace.TracerProvider + tracer oteltrace.Tracer } // New validates and creates a new Authorize service from a set of config options. func New(ctx context.Context, cfg *config.Config) (*Authorize, error) { + tracerProvider := trace.NewTracerProvider(ctx, "Authorize") + tracer := tracerProvider.Tracer(trace.PomeriumCoreTracer) a := &Authorize{ currentOptions: config.NewAtomicOptions(), store: store.New(), globalCache: storage.NewGlobalCache(time.Minute), + tracerProvider: tracerProvider, + tracer: tracer, } a.accessTracker = NewAccessTracker(a, accessTrackerMaxSize, accessTrackerDebouncePeriod) - state, err := newAuthorizeStateFromConfig(ctx, cfg, a.store, nil) + state, err := newAuthorizeStateFromConfig(ctx, tracerProvider, cfg, a.store, nil) if err != nil { return nil, err } @@ -88,7 +96,7 @@ func newPolicyEvaluator( ctx = log.WithContext(ctx, func(c zerolog.Context) zerolog.Context { return c.Str("service", "authorize") }) - ctx, span := trace.StartSpan(ctx, "authorize.newPolicyEvaluator") + ctx, span := trace.Continue(ctx, "authorize.newPolicyEvaluator") defer span.End() clientCA, err := opts.DownstreamMTLS.GetCA() @@ -142,7 +150,7 @@ func newPolicyEvaluator( func (a *Authorize) OnConfigChange(ctx context.Context, cfg *config.Config) { currentState := a.state.Load() a.currentOptions.Store(cfg.Options) - if state, err := newAuthorizeStateFromConfig(ctx, cfg, a.store, currentState.evaluator); err != nil { + if state, err := newAuthorizeStateFromConfig(ctx, a.tracerProvider, cfg, a.store, currentState.evaluator); err != nil { log.Ctx(ctx).Error().Err(err).Msg("authorize: error updating state") } else { a.state.Store(state) diff --git a/authorize/check_response.go b/authorize/check_response.go index 6a0b8b25d..027402af4 100644 --- a/authorize/check_response.go +++ b/authorize/check_response.go @@ -228,16 +228,21 @@ func (a *Authorize) requireLoginResponse( // always assume https scheme checkRequestURL := getCheckRequestURL(in) checkRequestURL.Scheme = "https" + var signInURLQuery url.Values + headers := http.Header{} + if id := in.GetAttributes().GetRequest().GetHttp().GetHeaders()["traceparent"]; id != "" { + signInURLQuery = url.Values{} + signInURLQuery.Add("pomerium_traceparent", id) + } redirectTo, err := state.authenticateFlow.AuthenticateSignInURL( - ctx, nil, &checkRequestURL, idp.GetId()) + ctx, signInURLQuery, &checkRequestURL, idp.GetId()) if err != nil { return nil, err } + headers["Location"] = []string{redirectTo} - return a.deniedResponse(ctx, in, http.StatusFound, "Login", http.Header{ - "Location": {redirectTo}, - }) + return a.deniedResponse(ctx, in, http.StatusFound, "Login", headers) } func (a *Authorize) requireWebAuthnResponse( diff --git a/authorize/databroker.go b/authorize/databroker.go index de7535752..e82550522 100644 --- a/authorize/databroker.go +++ b/authorize/databroker.go @@ -3,7 +3,6 @@ package authorize import ( "context" - "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/pkg/grpc/databroker" "github.com/pomerium/pomerium/pkg/grpc/session" "github.com/pomerium/pomerium/pkg/grpc/user" @@ -63,7 +62,7 @@ func (a *Authorize) getDataBrokerSessionOrServiceAccount( sessionID string, dataBrokerRecordVersion uint64, ) (s sessionOrServiceAccount, err error) { - ctx, span := trace.StartSpan(ctx, "authorize.getDataBrokerSessionOrServiceAccount") + ctx, span := a.tracer.Start(ctx, "authorize.getDataBrokerSessionOrServiceAccount") defer span.End() record, err := getDataBrokerRecord(ctx, grpcutil.GetTypeURL(new(session.Session)), sessionID, dataBrokerRecordVersion) @@ -96,7 +95,7 @@ func (a *Authorize) getDataBrokerUser( ctx context.Context, userID string, ) (*user.User, error) { - ctx, span := trace.StartSpan(ctx, "authorize.getDataBrokerUser") + ctx, span := a.tracer.Start(ctx, "authorize.getDataBrokerUser") defer span.End() record, err := getDataBrokerRecord(ctx, grpcutil.GetTypeURL(new(user.User)), userID, 0) diff --git a/authorize/evaluator/evaluator.go b/authorize/evaluator/evaluator.go index 726bd8b8f..1523cf04c 100644 --- a/authorize/evaluator/evaluator.go +++ b/authorize/evaluator/evaluator.go @@ -199,7 +199,7 @@ func getOrCreatePolicyEvaluators( // Evaluate evaluates the rego for the given policy and generates the identity headers. func (e *Evaluator) Evaluate(ctx context.Context, req *Request) (*Result, error) { - ctx, span := trace.StartSpan(ctx, "authorize.Evaluator.Evaluate") + ctx, span := trace.Continue(ctx, "authorize.Evaluator.Evaluate") defer span.End() eg, ctx := errgroup.WithContext(ctx) diff --git a/authorize/evaluator/headers_evaluator.go b/authorize/evaluator/headers_evaluator.go index e5a706917..7ed1baeb6 100644 --- a/authorize/evaluator/headers_evaluator.go +++ b/authorize/evaluator/headers_evaluator.go @@ -30,7 +30,7 @@ func NewHeadersEvaluator(store *store.Store) *HeadersEvaluator { // Evaluate evaluates the headers.rego script. func (e *HeadersEvaluator) Evaluate(ctx context.Context, req *Request, options ...rego.EvalOption) (*HeadersResponse, error) { - ctx, span := trace.StartSpan(ctx, "authorize.HeadersEvaluator.Evaluate") + ctx, span := trace.Continue(ctx, "authorize.HeadersEvaluator.Evaluate") defer span.End() ectx := new(rego.EvalContext) diff --git a/authorize/evaluator/policy_evaluator.go b/authorize/evaluator/policy_evaluator.go index a0b7fef96..56a4f228d 100644 --- a/authorize/evaluator/policy_evaluator.go +++ b/authorize/evaluator/policy_evaluator.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/open-policy-agent/opa/rego" - octrace "go.opencensus.io/trace" + "go.opentelemetry.io/otel/attribute" "github.com/pomerium/pomerium/authorize/internal/store" "github.com/pomerium/pomerium/config" @@ -209,9 +209,9 @@ func (e *PolicyEvaluator) Evaluate(ctx context.Context, req *PolicyRequest) (*Po } func (e *PolicyEvaluator) evaluateQuery(ctx context.Context, req *PolicyRequest, query policyQuery) (*PolicyResponse, error) { - ctx, span := trace.StartSpan(ctx, "authorize.PolicyEvaluator.evaluateQuery") + ctx, span := trace.Continue(ctx, "authorize.PolicyEvaluator.evaluateQuery") defer span.End() - span.AddAttributes(octrace.StringAttribute("script_checksum", query.checksum())) + span.SetAttributes(attribute.String("script_checksum", query.checksum())) rs, err := safeEval(ctx, query.PreparedEvalQuery, rego.EvalInput(req), diff --git a/authorize/grpc.go b/authorize/grpc.go index 3324a2177..8172226f9 100644 --- a/authorize/grpc.go +++ b/authorize/grpc.go @@ -19,7 +19,6 @@ import ( "github.com/pomerium/pomerium/internal/httputil" "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/sessions" - "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/urlutil" "github.com/pomerium/pomerium/pkg/contextutil" "github.com/pomerium/pomerium/pkg/grpc/user" @@ -29,7 +28,7 @@ import ( // Check implements the envoy auth server gRPC endpoint. func (a *Authorize) Check(ctx context.Context, in *envoy_service_auth_v3.CheckRequest) (*envoy_service_auth_v3.CheckResponse, error) { - ctx, span := trace.StartSpan(ctx, "authorize.grpc.Check") + ctx, span := a.tracer.Start(ctx, "authorize.grpc.Check") defer span.End() querier := storage.NewTracingQuerier( diff --git a/authorize/internal/store/store.go b/authorize/internal/store/store.go index 5c32257a7..6f651e645 100644 --- a/authorize/internal/store/store.go +++ b/authorize/internal/store/store.go @@ -14,7 +14,7 @@ import ( opastorage "github.com/open-policy-agent/opa/storage" "github.com/open-policy-agent/opa/storage/inmem" "github.com/open-policy-agent/opa/types" - octrace "go.opencensus.io/trace" + "go.opentelemetry.io/otel/attribute" "google.golang.org/grpc" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" @@ -145,20 +145,20 @@ func (s *Store) GetDataBrokerRecordOption() func(*rego.Rego) { types.NewObject(nil, types.NewDynamicProperty(types.S, types.S)), ), }, func(bctx rego.BuiltinContext, op1 *ast.Term, op2 *ast.Term) (*ast.Term, error) { - ctx, span := trace.StartSpan(bctx.Context, "rego.get_databroker_record") + ctx, span := trace.Continue(bctx.Context, "rego.get_databroker_record") defer span.End() recordType, ok := op1.Value.(ast.String) if !ok { return nil, fmt.Errorf("invalid record type: %T", op1) } - span.AddAttributes(octrace.StringAttribute("record_type", recordType.String())) + span.SetAttributes(attribute.String("record_type", recordType.String())) recordIDOrIndex, ok := op2.Value.(ast.String) if !ok { return nil, fmt.Errorf("invalid record id: %T", op2) } - span.AddAttributes(octrace.StringAttribute("record_id", recordIDOrIndex.String())) + span.SetAttributes(attribute.String("record_id", recordIDOrIndex.String())) msg := s.GetDataBrokerRecord(ctx, string(recordType), string(recordIDOrIndex)) if msg == nil { diff --git a/authorize/log.go b/authorize/log.go index 594689245..3e5088262 100644 --- a/authorize/log.go +++ b/authorize/log.go @@ -7,10 +7,10 @@ import ( envoy_service_auth_v3 "github.com/envoyproxy/go-control-plane/envoy/service/auth/v3" "github.com/go-jose/go-jose/v3/jwt" "github.com/rs/zerolog" + "go.opentelemetry.io/otel/attribute" "github.com/pomerium/pomerium/authorize/evaluator" "github.com/pomerium/pomerium/internal/log" - "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/pkg/grpc/databroker" "github.com/pomerium/pomerium/pkg/grpc/session" "github.com/pomerium/pomerium/pkg/grpc/user" @@ -24,7 +24,7 @@ func (a *Authorize) logAuthorizeCheck( in *envoy_service_auth_v3.CheckRequest, res *evaluator.Result, s sessionOrServiceAccount, u *user.User, ) { - ctx, span := trace.StartSpan(ctx, "authorize.grpc.LogAuthorizeCheck") + ctx, span := a.tracer.Start(ctx, "authorize.grpc.LogAuthorizeCheck") defer span.End() hdrs := getCheckRequestHeaders(in) @@ -39,17 +39,24 @@ func (a *Authorize) logAuthorizeCheck( // result if res != nil { + span.SetAttributes(attribute.Bool("result.allow", res.Allow.Value)) evt = evt.Bool("allow", res.Allow.Value) + allowReasons := res.Allow.Reasons.Strings() if res.Allow.Value { - evt = evt.Strs("allow-why-true", res.Allow.Reasons.Strings()) + span.SetAttributes(attribute.StringSlice("result.allow-why-true", allowReasons)) + evt = evt.Strs("allow-why-true", allowReasons) } else { - evt = evt.Strs("allow-why-false", res.Allow.Reasons.Strings()) + span.SetAttributes(attribute.StringSlice("result.allow-why-false", allowReasons)) + evt = evt.Strs("allow-why-false", allowReasons) } evt = evt.Bool("deny", res.Deny.Value) + denyReasons := res.Deny.Reasons.Strings() if res.Deny.Value { - evt = evt.Strs("deny-why-true", res.Deny.Reasons.Strings()) + span.SetAttributes(attribute.StringSlice("result.deny-why-true", denyReasons)) + evt = evt.Strs("deny-why-true", denyReasons) } else { - evt = evt.Strs("deny-why-false", res.Deny.Reasons.Strings()) + span.SetAttributes(attribute.StringSlice("result.deny-why-false", denyReasons)) + evt = evt.Strs("deny-why-false", denyReasons) } } diff --git a/authorize/state.go b/authorize/state.go index ec82e8c57..10cfb4195 100644 --- a/authorize/state.go +++ b/authorize/state.go @@ -5,6 +5,8 @@ import ( "fmt" "net/url" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + oteltrace "go.opentelemetry.io/otel/trace" googlegrpc "google.golang.org/grpc" "github.com/pomerium/pomerium/authorize/evaluator" @@ -32,7 +34,10 @@ type authorizeState struct { func newAuthorizeStateFromConfig( ctx context.Context, - cfg *config.Config, store *store.Store, previousPolicyEvaluator *evaluator.Evaluator, + tracerProvider oteltrace.TracerProvider, + cfg *config.Config, + store *store.Store, + previousPolicyEvaluator *evaluator.Evaluator, ) (*authorizeState, error) { if err := validateOptions(cfg.Options); err != nil { return nil, fmt.Errorf("authorize: bad options: %w", err) @@ -62,7 +67,7 @@ func newAuthorizeStateFromConfig( InstallationID: cfg.Options.InstallationID, ServiceName: cfg.Options.Services, SignedJWTKey: sharedKey, - }) + }, googlegrpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(tracerProvider)))) if err != nil { return nil, fmt.Errorf("authorize: error creating databroker connection: %w", err) } @@ -75,9 +80,9 @@ func newAuthorizeStateFromConfig( } if cfg.Options.UseStatelessAuthenticateFlow() { - state.authenticateFlow, err = authenticateflow.NewStateless(ctx, cfg, nil, nil, nil, nil) + state.authenticateFlow, err = authenticateflow.NewStateless(ctx, tracerProvider, cfg, nil, nil, nil, nil) } else { - state.authenticateFlow, err = authenticateflow.NewStateful(ctx, cfg, nil) + state.authenticateFlow, err = authenticateflow.NewStateful(ctx, tracerProvider, cfg, nil) } if err != nil { return nil, err diff --git a/cmd/pomerium/main.go b/cmd/pomerium/main.go index 9c035e8fa..0d2869bea 100644 --- a/cmd/pomerium/main.go +++ b/cmd/pomerium/main.go @@ -12,6 +12,7 @@ import ( "github.com/pomerium/pomerium/config" "github.com/pomerium/pomerium/internal/log" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/version" _ "github.com/pomerium/pomerium/internal/zero/bootstrap/writers/filesystem" _ "github.com/pomerium/pomerium/internal/zero/bootstrap/writers/k8s" @@ -30,9 +31,15 @@ func main() { } root.AddCommand(zero_cmd.BuildRootCmd()) root.PersistentFlags().StringVar(&configFile, "config", "", "Specify configuration file location") - - ctx := context.Background() log.SetLevel(zerolog.InfoLevel) + ctx := trace.Options{ + RemoteClient: trace.NewSyncClient(trace.NewRemoteClientFromEnv()), + }.NewContext(context.Background()) + defer func() { + if err := trace.ShutdownContext(ctx); err != nil { + log.Error().Err(err).Send() + } + }() runFn := run if zero_cmd.IsManagedMode(configFile) { runFn = zero_cmd.Run diff --git a/config/envoyconfig/bootstrap.go b/config/envoyconfig/bootstrap.go index d1dbbd8f9..665f216fb 100644 --- a/config/envoyconfig/bootstrap.go +++ b/config/envoyconfig/bootstrap.go @@ -3,6 +3,7 @@ package envoyconfig import ( "context" "fmt" + "math" "os" "path/filepath" @@ -37,7 +38,7 @@ func (b *Builder) BuildBootstrap( cfg *config.Config, fullyStatic bool, ) (bootstrap *envoy_config_bootstrap_v3.Bootstrap, err error) { - ctx, span := trace.StartSpan(ctx, "envoyconfig.Builder.BuildBootstrap") + ctx, span := trace.Continue(ctx, "envoyconfig.Builder.BuildBootstrap") defer span.End() bootstrap = new(envoy_config_bootstrap_v3.Bootstrap) @@ -52,7 +53,7 @@ func (b *Builder) BuildBootstrap( return nil, fmt.Errorf("error building bootstrap dynamic resources: %w", err) } - bootstrap.LayeredRuntime, err = b.BuildBootstrapLayeredRuntime() + bootstrap.LayeredRuntime, err = b.BuildBootstrapLayeredRuntime(ctx) if err != nil { return nil, fmt.Errorf("error building bootstrap layered runtime: %w", err) } @@ -148,7 +149,13 @@ func (b *Builder) BuildBootstrapDynamicResources( } // BuildBootstrapLayeredRuntime builds the layered runtime for the envoy bootstrap. -func (b *Builder) BuildBootstrapLayeredRuntime() (*envoy_config_bootstrap_v3.LayeredRuntime, error) { +func (b *Builder) BuildBootstrapLayeredRuntime(ctx context.Context) (*envoy_config_bootstrap_v3.LayeredRuntime, error) { + flushIntervalMs := 5000 + minFlushSpans := 3 + if trace.DebugFlagsFromContext(ctx).Check(trace.EnvoyFlushEverySpan) { + minFlushSpans = 1 + flushIntervalMs = math.MaxInt32 + } layer, err := structpb.NewStruct(map[string]any{ "re2": map[string]any{ "max_program_size": map[string]any{ @@ -156,6 +163,21 @@ func (b *Builder) BuildBootstrapLayeredRuntime() (*envoy_config_bootstrap_v3.Lay "warn_level": 1024, }, }, + "tracing": map[string]any{ + "opentelemetry": map[string]any{ + "flush_interval_ms": flushIntervalMs, + // For most requests, envoy generates 3 spans: + // - ingress (downstream->envoy) + // - ext_authz check request (envoy->pomerium) + // - egress (envoy->upstream) + // The default value is 5, which usually leads to delayed exports. + // This can be set lower, e.g. 1 to have envoy export every span + // individually (useful for testing), but 3 is a reasonable default. + // If set to 1, also set flush_interval_ms to a very large number to + // effectively disable it. + "min_flush_spans": minFlushSpans, + }, + }, }) if err != nil { return nil, fmt.Errorf("envoyconfig: failed to create layered runtime layer: %w", err) @@ -180,7 +202,7 @@ func (b *Builder) BuildBootstrapStaticResources( cfg *config.Config, fullyStatic bool, ) (staticResources *envoy_config_bootstrap_v3.Bootstrap_StaticResources, err error) { - ctx, span := trace.StartSpan(ctx, "envoyconfig.Builder.BuildBootstrapStaticResources") + ctx, span := trace.Continue(ctx, "envoyconfig.Builder.BuildBootstrapStaticResources") defer span.End() staticResources = new(envoy_config_bootstrap_v3.Bootstrap_StaticResources) diff --git a/config/envoyconfig/bootstrap_test.go b/config/envoyconfig/bootstrap_test.go index bdceb7945..ec27b7fe7 100644 --- a/config/envoyconfig/bootstrap_test.go +++ b/config/envoyconfig/bootstrap_test.go @@ -36,7 +36,7 @@ func TestBuilder_BuildBootstrapAdmin(t *testing.T) { func TestBuilder_BuildBootstrapLayeredRuntime(t *testing.T) { b := New("localhost:1111", "localhost:2222", "localhost:3333", filemgr.NewManager(), nil) - staticCfg, err := b.BuildBootstrapLayeredRuntime() + staticCfg, err := b.BuildBootstrapLayeredRuntime(context.Background()) assert.NoError(t, err) testutil.AssertProtoJSONEqual(t, ` { "layers": [{ @@ -47,6 +47,12 @@ func TestBuilder_BuildBootstrapLayeredRuntime(t *testing.T) { "error_level": 1048576, "warn_level": 1024 } + }, + "tracing": { + "opentelemetry": { + "flush_interval_ms": 5000, + "min_flush_spans": 3 + } } } }] } diff --git a/config/envoyconfig/clusters.go b/config/envoyconfig/clusters.go index d0d1fd7f9..bfb083cc5 100644 --- a/config/envoyconfig/clusters.go +++ b/config/envoyconfig/clusters.go @@ -26,7 +26,7 @@ import ( // BuildClusters builds envoy clusters from the given config. func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*envoy_config_cluster_v3.Cluster, error) { - ctx, span := trace.StartSpan(ctx, "envoyconfig.Builder.BuildClusters") + ctx, span := trace.Continue(ctx, "envoyconfig.Builder.BuildClusters") defer span.End() grpcURLs := []*url.URL{{ @@ -104,13 +104,6 @@ func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*env envoyAdminCluster, } - tracingCluster, err := buildTracingCluster(cfg.Options) - if err != nil { - return nil, err - } else if tracingCluster != nil { - clusters = append(clusters, tracingCluster) - } - if config.IsProxy(cfg.Options.Services) { for policy := range cfg.Options.GetAllPolicies() { if len(policy.To) > 0 { diff --git a/config/envoyconfig/listeners.go b/config/envoyconfig/listeners.go index f017c0834..e02b39e3e 100644 --- a/config/envoyconfig/listeners.go +++ b/config/envoyconfig/listeners.go @@ -20,7 +20,7 @@ func (b *Builder) BuildListeners( cfg *config.Config, fullyStatic bool, ) ([]*envoy_config_listener_v3.Listener, error) { - ctx, span := trace.StartSpan(ctx, "envoyconfig.Builder.BuildListeners") + ctx, span := trace.Continue(ctx, "envoyconfig.Builder.BuildListeners") defer span.End() var listeners []*envoy_config_listener_v3.Listener diff --git a/config/envoyconfig/listeners_grpc.go b/config/envoyconfig/listeners_grpc.go index 802947cf1..2f92b3de1 100644 --- a/config/envoyconfig/listeners_grpc.go +++ b/config/envoyconfig/listeners_grpc.go @@ -73,6 +73,9 @@ func (b *Builder) buildGRPCHTTPConnectionManagerFilter() *envoy_config_listener_ PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: fmt.Sprintf("/%s/", svc)}, Grpc: &envoy_config_route_v3.RouteMatch_GrpcRouteMatchOptions{}, }, + Decorator: &envoy_config_route_v3.Decorator{ + Operation: fmt.Sprintf("pomerium-control-plane-grpc %s", svc), + }, Action: &envoy_config_route_v3.Route_Route{ Route: &envoy_config_route_v3.RouteAction{ ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{ diff --git a/config/envoyconfig/listeners_main.go b/config/envoyconfig/listeners_main.go index e21915011..8f4feca7a 100644 --- a/config/envoyconfig/listeners_main.go +++ b/config/envoyconfig/listeners_main.go @@ -10,7 +10,6 @@ import ( envoy_config_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" envoy_extensions_access_loggers_grpc_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/grpc/v3" envoy_extensions_filters_network_http_connection_manager "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" - envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/wrapperspb" @@ -177,20 +176,11 @@ func (b *Builder) buildMainHTTPConnectionManagerFilter( maxStreamDuration = durationpb.New(cfg.Options.WriteTimeout) } - tracingProvider, err := buildTracingHTTP(cfg.Options) - if err != nil { - return nil, err - } - localReply, err := b.buildLocalReplyConfig(cfg.Options) if err != nil { return nil, err } - sampleRate := 1.0 - if cfg.Options.TracingSampleRate != nil { - sampleRate = *cfg.Options.TracingSampleRate - } mgr := &envoy_extensions_filters_network_http_connection_manager.HttpConnectionManager{ AlwaysSetRequestIdInResponse: true, StatPrefix: "ingress", @@ -202,10 +192,6 @@ func (b *Builder) buildMainHTTPConnectionManagerFilter( }, HttpProtocolOptions: http1ProtocolOptions, RequestTimeout: durationpb.New(cfg.Options.ReadTimeout), - Tracing: &envoy_extensions_filters_network_http_connection_manager.HttpConnectionManager_Tracing{ - RandomSampling: &envoy_type_v3.Percent{Value: max(0.0, min(1.0, sampleRate)) * 100}, - Provider: tracingProvider, - }, // See https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for UseRemoteAddress: &wrapperspb.BoolValue{Value: true}, SkipXffAppend: cfg.Options.SkipXffAppend, @@ -228,6 +214,8 @@ func (b *Builder) buildMainHTTPConnectionManagerFilter( mgr.CodecType = cfg.Options.GetCodecType().ToEnvoy() } + applyTracingConfig(ctx, mgr, cfg.Options) + if fullyStatic { routeConfiguration, err := b.buildMainRouteConfiguration(ctx, cfg) if err != nil { diff --git a/config/envoyconfig/listeners_test.go b/config/envoyconfig/listeners_test.go index bd7d9baa9..d38c83f4e 100644 --- a/config/envoyconfig/listeners_test.go +++ b/config/envoyconfig/listeners_test.go @@ -149,7 +149,9 @@ func Test_buildMainHTTPConnectionManagerFilter(t *testing.T) { options.SkipXffAppend = true options.XffNumTrustedHops = 1 options.AuthenticateURLString = "https://authenticate.example.com" + options.TracingProvider = "otlp" filter, err := b.buildMainHTTPConnectionManagerFilter(context.Background(), &config.Config{Options: options}, false, false) require.NoError(t, err) + testutil.AssertProtoJSONEqual(t, testData(t, "main_http_connection_manager_filter.json", nil), filter) } diff --git a/config/envoyconfig/outbound.go b/config/envoyconfig/outbound.go index 541987746..fbab8c0d5 100644 --- a/config/envoyconfig/outbound.go +++ b/config/envoyconfig/outbound.go @@ -101,6 +101,9 @@ func (b *Builder) buildOutboundRoutes() []*envoy_config_route_v3.Route { PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: prefix}, Grpc: &envoy_config_route_v3.RouteMatch_GrpcRouteMatchOptions{}, }, + Decorator: &envoy_config_route_v3.Decorator{ + Operation: fmt.Sprintf("Outbound (grpc): %s %s", def.Cluster, prefix), + }, Action: &envoy_config_route_v3.Route_Route{ Route: &envoy_config_route_v3.RouteAction{ ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{ @@ -123,6 +126,9 @@ func (b *Builder) buildOutboundRoutes() []*envoy_config_route_v3.Route { Match: &envoy_config_route_v3.RouteMatch{ PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: "/envoy/stats/prometheus"}, }, + Decorator: &envoy_config_route_v3.Decorator{ + Operation: "Outbound: envoy-metrics /envoy/stats/prometheus/*", + }, Action: &envoy_config_route_v3.Route_Route{ Route: &envoy_config_route_v3.RouteAction{ ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{ diff --git a/config/envoyconfig/outbound_test.go b/config/envoyconfig/outbound_test.go index 2a12488c2..3c7233f7a 100644 --- a/config/envoyconfig/outbound_test.go +++ b/config/envoyconfig/outbound_test.go @@ -15,6 +15,9 @@ func Test_buildOutboundRoutes(t *testing.T) { "grpc": {}, "prefix": "/envoy.service.auth.v3.Authorization/" }, + "decorator": { + "operation": "Outbound (grpc): pomerium-authorize /envoy.service.auth.v3.Authorization/" + }, "name": "pomerium-authorize", "route": { "autoHostRewrite": true, @@ -28,6 +31,9 @@ func Test_buildOutboundRoutes(t *testing.T) { "grpc": {}, "prefix": "/databroker.DataBrokerService/" }, + "decorator": { + "operation": "Outbound (grpc): pomerium-databroker /databroker.DataBrokerService/" + }, "name": "pomerium-databroker", "route": { "autoHostRewrite": true, @@ -41,6 +47,9 @@ func Test_buildOutboundRoutes(t *testing.T) { "grpc": {}, "prefix": "/registry.Registry/" }, + "decorator": { + "operation": "Outbound (grpc): pomerium-databroker /registry.Registry/" + }, "name": "pomerium-databroker", "route": { "autoHostRewrite": true, @@ -54,6 +63,9 @@ func Test_buildOutboundRoutes(t *testing.T) { "grpc": {}, "prefix": "/" }, + "decorator": { + "operation": "Outbound (grpc): pomerium-control-plane-grpc /" + }, "name": "pomerium-control-plane-grpc", "route": { "autoHostRewrite": true, @@ -66,6 +78,9 @@ func Test_buildOutboundRoutes(t *testing.T) { "match": { "prefix": "/envoy/stats/prometheus" }, + "decorator": { + "operation": "Outbound: envoy-metrics /envoy/stats/prometheus/*" + }, "name": "envoy-metrics", "route": { "cluster": "pomerium-envoy-admin", diff --git a/config/envoyconfig/route_configurations.go b/config/envoyconfig/route_configurations.go index 4757b1a74..9881e7322 100644 --- a/config/envoyconfig/route_configurations.go +++ b/config/envoyconfig/route_configurations.go @@ -20,7 +20,7 @@ func (b *Builder) BuildRouteConfigurations( ctx context.Context, cfg *config.Config, ) ([]*envoy_config_route_v3.RouteConfiguration, error) { - ctx, span := trace.StartSpan(ctx, "envoyconfig.Builder.BuildRouteConfigurations") + ctx, span := trace.Continue(ctx, "envoyconfig.Builder.BuildRouteConfigurations") defer span.End() var routeConfigurations []*envoy_config_route_v3.RouteConfiguration diff --git a/config/envoyconfig/route_configurations_test.go b/config/envoyconfig/route_configurations_test.go index d7e97eb6a..06593b567 100644 --- a/config/envoyconfig/route_configurations_test.go +++ b/config/envoyconfig/route_configurations_test.go @@ -51,6 +51,10 @@ func TestBuilder_buildMainRouteConfiguration(t *testing.T) { `+protojson.Format(b.buildControlPlanePrefixRoute(cfg.Options, "/.well-known/pomerium/"))+`, { "name": "policy-0", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "headers": [ { "name": ":authority", "stringMatch": { "safeRegex": { "regex": "^(.*)\\.example\\.com$" } }} @@ -104,6 +108,10 @@ func TestBuilder_buildMainRouteConfiguration(t *testing.T) { }, { "name": "policy-0", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "headers": [ { "name": ":authority", "stringMatch": { "safeRegex": { "regex": "^(.*)\\.example\\.com:443$" } }} diff --git a/config/envoyconfig/routes.go b/config/envoyconfig/routes.go index 850976c1e..4bbb10263 100644 --- a/config/envoyconfig/routes.go +++ b/config/envoyconfig/routes.go @@ -114,6 +114,9 @@ func (b *Builder) buildControlPlanePathRoute( Match: &envoy_config_route_v3.RouteMatch{ PathSpecifier: &envoy_config_route_v3.RouteMatch_Path{Path: path}, }, + Decorator: &envoy_config_route_v3.Decorator{ + Operation: "internal: ${method} ${host}${path}", + }, Action: &envoy_config_route_v3.Route_Route{ Route: &envoy_config_route_v3.RouteAction{ ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{ @@ -138,6 +141,9 @@ func (b *Builder) buildControlPlanePrefixRoute( Match: &envoy_config_route_v3.RouteMatch{ PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: prefix}, }, + Decorator: &envoy_config_route_v3.Decorator{ + Operation: "internal: ${method} ${host}${path}", + }, Action: &envoy_config_route_v3.Route_Route{ Route: &envoy_config_route_v3.RouteAction{ ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{ @@ -269,8 +275,12 @@ func (b *Builder) buildRouteForPolicyAndMatch( } route := &envoy_config_route_v3.Route{ - Name: name, - Match: match, + Name: name, + Match: match, + Decorator: &envoy_config_route_v3.Decorator{ + Operation: "ingress: ${method} ${host}${path}", + Propagate: wrapperspb.Bool(false), + }, Metadata: &envoy_config_core_v3.Metadata{}, RequestHeadersToRemove: getRequestHeadersToRemove(cfg.Options, policy), ResponseHeadersToAdd: toEnvoyHeaders(cfg.Options.GetSetResponseHeadersForPolicy(policy)), diff --git a/config/envoyconfig/routes_test.go b/config/envoyconfig/routes_test.go index 86825413a..3c9ea98ce 100644 --- a/config/envoyconfig/routes_test.go +++ b/config/envoyconfig/routes_test.go @@ -59,6 +59,9 @@ func Test_buildPomeriumHTTPRoutes(t *testing.T) { routeString := func(typ, name string) string { str := `{ "name": "pomerium-` + typ + `-` + name + `", + "decorator": { + "operation": "internal: ${method} ${host}${path}" + }, "match": { "` + typ + `": "` + name + `" }, @@ -135,6 +138,9 @@ func Test_buildControlPlanePathRoute(t *testing.T) { testutil.AssertProtoJSONEqual(t, ` { "name": "pomerium-path-/hello/world", + "decorator": { + "operation": "internal: ${method} ${host}${path}" + }, "match": { "path": "/hello/world" }, @@ -179,6 +185,9 @@ func Test_buildControlPlanePrefixRoute(t *testing.T) { testutil.AssertProtoJSONEqual(t, ` { "name": "pomerium-prefix-/hello/world/", + "decorator": { + "operation": "internal: ${method} ${host}${path}" + }, "match": { "prefix": "/hello/world/" }, @@ -406,6 +415,10 @@ func Test_buildPolicyRoutes(t *testing.T) { [ { "name": "policy-1", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/" }, @@ -476,6 +489,10 @@ func Test_buildPolicyRoutes(t *testing.T) { }, { "name": "policy-2", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "path": "/some/path" }, @@ -547,6 +564,10 @@ func Test_buildPolicyRoutes(t *testing.T) { }, { "name": "policy-3", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/some/prefix/" }, @@ -617,6 +638,10 @@ func Test_buildPolicyRoutes(t *testing.T) { }, { "name": "policy-4", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "safeRegex": { "regex": "^/[a]+$" @@ -689,6 +714,10 @@ func Test_buildPolicyRoutes(t *testing.T) { }, { "name": "policy-5", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/some/prefix/" }, @@ -760,6 +789,10 @@ func Test_buildPolicyRoutes(t *testing.T) { }, { "name": "policy-6", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "path": "/some/path" }, @@ -830,6 +863,10 @@ func Test_buildPolicyRoutes(t *testing.T) { }, { "name": "policy-7", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "path": "/some/path" }, @@ -901,6 +938,10 @@ func Test_buildPolicyRoutes(t *testing.T) { }, { "name": "policy-8", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "path": "/websocket-timeout" }, @@ -994,6 +1035,10 @@ func Test_buildPolicyRoutes(t *testing.T) { [ { "name": "policy-0", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/" }, @@ -1083,6 +1128,10 @@ func Test_buildPolicyRoutes(t *testing.T) { [ { "name": "policy-0", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "connectMatcher": {} }, @@ -1155,6 +1204,10 @@ func Test_buildPolicyRoutes(t *testing.T) { }, { "name": "policy-1", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "connectMatcher": {} }, @@ -1248,6 +1301,10 @@ func Test_buildPolicyRoutes(t *testing.T) { [ { "name": "policy-0", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "connectMatcher": {} }, @@ -1345,6 +1402,10 @@ func Test_buildPolicyRoutes(t *testing.T) { [ { "name": "policy-0", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/" }, @@ -1440,6 +1501,10 @@ func Test_buildPolicyRoutes(t *testing.T) { [ { "name": "policy-0", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/" }, @@ -1588,6 +1653,10 @@ func Test_buildPolicyRoutesRewrite(t *testing.T) { [ { "name": "policy-0", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/" }, @@ -1659,6 +1728,10 @@ func Test_buildPolicyRoutesRewrite(t *testing.T) { }, { "name": "policy-1", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/" }, @@ -1730,6 +1803,10 @@ func Test_buildPolicyRoutesRewrite(t *testing.T) { }, { "name": "policy-2", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/" }, @@ -1806,6 +1883,10 @@ func Test_buildPolicyRoutesRewrite(t *testing.T) { }, { "name": "policy-3", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/" }, @@ -1877,6 +1958,10 @@ func Test_buildPolicyRoutesRewrite(t *testing.T) { }, { "name": "policy-4", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/" }, @@ -1948,6 +2033,10 @@ func Test_buildPolicyRoutesRewrite(t *testing.T) { }, { "name": "policy-5", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/" }, diff --git a/config/envoyconfig/testdata/main_http_connection_manager_filter.json b/config/envoyconfig/testdata/main_http_connection_manager_filter.json index 859453cd4..4b254dc5f 100644 --- a/config/envoyconfig/testdata/main_http_connection_manager_filter.json +++ b/config/envoyconfig/testdata/main_http_connection_manager_filter.json @@ -23,6 +23,14 @@ "commonHttpProtocolOptions": { "idleTimeout": "300s" }, + "earlyHeaderMutationExtensions": [ + { + "name": "envoy.http.early_header_mutation.trace_context", + "typedConfig": { + "@type": "type.googleapis.com/pomerium.extensions.TraceContext" + } + } + ], "httpFilters": [ { "name": "envoy.filters.http.lua", @@ -52,13 +60,13 @@ }, "timeout": "10s" }, - "metadataContextNamespaces": [ - "com.pomerium.client-certificate-info" - ], + "transportApiVersion": "V3", "statusOnError": { "code": "InternalServerError" }, - "transportApiVersion": "V3" + "metadataContextNamespaces": [ + "com.pomerium.client-certificate-info" + ] } }, { @@ -75,7 +83,7 @@ "typedConfig": { "@type": "type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua", "defaultSourceCode": { - "inlineString": "function has_prefix(str, prefix)\n return str ~= nil and str:sub(1, #prefix) == prefix\nend\n\nfunction remove_pomerium_cookie(cookie_name, cookie)\n local result = \"\"\n for c in cookie:gmatch(\"([^;]+)\") do\n c = c:gsub(\"^ +\",\"\")\n local name = c:match(\"^([^=]+)\")\n if name ~= cookie_name then\n if string.len(result) \u003e 0 then\n result = result .. \"; \" .. c\n else\n result = result .. c\n end\n end\n end\n return result\nend\n\nfunction envoy_on_request(request_handle)\n local headers = request_handle:headers()\n local metadata = request_handle:metadata()\n\n local remove_cookie_name = metadata:get(\"remove_pomerium_cookie\")\n if remove_cookie_name then\n local cookie = headers:get(\"cookie\")\n if cookie ~= nil then\n local newcookie = remove_pomerium_cookie(remove_cookie_name, cookie)\n headers:replace(\"cookie\", newcookie)\n end\n end\n\n local remove_authorization = metadata:get(\"remove_pomerium_authorization\")\n if remove_authorization then\n local authorization = headers:get(\"authorization\")\n local authorization_prefix = \"Pomerium \"\n if has_prefix(authorization, authorization_prefix) then\n headers:remove(\"authorization\")\n end\n\n headers:remove('x-pomerium-authorization')\n end\nend\n\nfunction envoy_on_response(response_handle) end\n" + "inlineString": "function has_prefix(str, prefix)\n return str ~= nil and str:sub(1, #prefix) == prefix\nend\n\nfunction remove_pomerium_cookie(cookie_name, cookie)\n local result = \"\"\n for c in cookie:gmatch(\"([^;]+)\") do\n c = c:gsub(\"^ +\",\"\")\n local name = c:match(\"^([^=]+)\")\n if name ~= cookie_name then\n if string.len(result) > 0 then\n result = result .. \"; \" .. c\n else\n result = result .. c\n end\n end\n end\n return result\nend\n\nfunction envoy_on_request(request_handle)\n local headers = request_handle:headers()\n local metadata = request_handle:metadata()\n\n local remove_cookie_name = metadata:get(\"remove_pomerium_cookie\")\n if remove_cookie_name then\n local cookie = headers:get(\"cookie\")\n if cookie ~= nil then\n local newcookie = remove_pomerium_cookie(remove_cookie_name, cookie)\n headers:replace(\"cookie\", newcookie)\n end\n end\n\n local remove_authorization = metadata:get(\"remove_pomerium_authorization\")\n if remove_authorization then\n local authorization = headers:get(\"authorization\")\n local authorization_prefix = \"Pomerium \"\n if has_prefix(authorization, authorization_prefix) then\n headers:remove(\"authorization\")\n end\n\n headers:remove('x-pomerium-authorization')\n end\nend\n\nfunction envoy_on_response(response_handle) end\n" } } }, @@ -169,6 +177,13 @@ } ] }, + "requestIdExtension": { + "typedConfig": { + "@type": "type.googleapis.com/pomerium.extensions.UuidxRequestIdConfig", + "packTraceReason": true, + "useRequestIdForTraceSampling": true + } + }, "requestTimeout": "30s", "normalizePath": true, "rds": { @@ -183,7 +198,33 @@ "tracing": { "randomSampling": { "value": 100 - } + }, + "verbose": true, + "maxPathTagLength": 1024, + "provider": { + "name": "envoy.tracers.pomerium_otel", + "typedConfig": { + "@type": "type.googleapis.com/pomerium.extensions.OpenTelemetryConfig", + "grpcService": { + "envoyGrpc": { + "clusterName": "pomerium-control-plane-grpc" + } + }, + "serviceName": "Envoy", + "resourceDetectors": [ + { + "name": "envoy.tracers.opentelemetry.resource_detectors.static_config", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.tracers.opentelemetry.resource_detectors.v3.StaticConfigResourceDetectorConfig", + "attributes": { + "pomerium.envoy": "true" + } + } + } + ] + } + }, + "spawnUpstreamSpan": true }, "useRemoteAddress": true, "xffNumTrustedHops": 1, diff --git a/config/envoyconfig/tracing.go b/config/envoyconfig/tracing.go index 60eb69f9d..7cfc8724a 100644 --- a/config/envoyconfig/tracing.go +++ b/config/envoyconfig/tracing.go @@ -1,134 +1,151 @@ package envoyconfig import ( - "fmt" - "net" - - envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" - envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" - envoy_config_trace_v3 "github.com/envoyproxy/go-control-plane/envoy/config/trace/v3" - "google.golang.org/protobuf/types/known/durationpb" + "context" + "os" + "strconv" + envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + tracev3 "github.com/envoyproxy/go-control-plane/envoy/config/trace/v3" + envoy_extensions_filters_http_header_to_metadata "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/header_to_metadata/v3" + envoy_extensions_filters_network_http_connection_manager "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + envoy_extensions_tracers_otel "github.com/envoyproxy/go-control-plane/envoy/extensions/tracers/opentelemetry/resource_detectors/v3" + metadatav3 "github.com/envoyproxy/go-control-plane/envoy/type/metadata/v3" + envoy_tracing_v3 "github.com/envoyproxy/go-control-plane/envoy/type/tracing/v3" + envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3" + extensions_trace_context "github.com/pomerium/envoy-custom/api/extensions/http/early_header_mutation/trace_context" + extensions_uuidx "github.com/pomerium/envoy-custom/api/extensions/request_id/uuidx" + extensions_pomerium_otel "github.com/pomerium/envoy-custom/api/extensions/tracers/pomerium_otel" "github.com/pomerium/pomerium/config" "github.com/pomerium/pomerium/internal/telemetry/trace" - "github.com/pomerium/pomerium/pkg/protoutil" + "google.golang.org/protobuf/types/known/wrapperspb" ) -func buildTracingCluster(options *config.Options) (*envoy_config_cluster_v3.Cluster, error) { - tracingOptions, err := config.NewTracingOptions(options) - if err != nil { - return nil, fmt.Errorf("envoyconfig: invalid tracing config: %w", err) +func isTracingEnabled(cfg *config.Options) bool { + if os.Getenv("OTEL_SDK_DISABLED") == "true" { + return false } - - switch tracingOptions.Provider { - case trace.DatadogTracingProviderName: - addr, _ := parseAddress("127.0.0.1:8126") - - if options.TracingDatadogAddress != "" { - addr, err = parseAddress(options.TracingDatadogAddress) - if err != nil { - return nil, fmt.Errorf("envoyconfig: invalid tracing datadog address: %w", err) - } - } - - endpoints := []*envoy_config_endpoint_v3.LbEndpoint{{ - HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{ - Endpoint: &envoy_config_endpoint_v3.Endpoint{ - Address: addr, - }, - }, - }} - - return &envoy_config_cluster_v3.Cluster{ - Name: "datadog-apm", - ConnectTimeout: &durationpb.Duration{ - Seconds: 5, - }, - ClusterDiscoveryType: getClusterDiscoveryType(endpoints), - LbPolicy: envoy_config_cluster_v3.Cluster_ROUND_ROBIN, - LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{ - ClusterName: "datadog-apm", - Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{{ - LbEndpoints: endpoints, - }}, - }, - }, nil - case trace.ZipkinTracingProviderName: - host := tracingOptions.ZipkinEndpoint.Host - if _, port, _ := net.SplitHostPort(host); port == "" { - if tracingOptions.ZipkinEndpoint.Scheme == "https" { - host = net.JoinHostPort(host, "443") - } else { - host = net.JoinHostPort(host, "80") - } - } - - addr, err := parseAddress(host) - if err != nil { - return nil, fmt.Errorf("envoyconfig: invalid tracing zipkin address: %w", err) - } - - endpoints := []*envoy_config_endpoint_v3.LbEndpoint{{ - HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{ - Endpoint: &envoy_config_endpoint_v3.Endpoint{ - Address: addr, - }, - }, - }} - return &envoy_config_cluster_v3.Cluster{ - Name: "zipkin", - ConnectTimeout: &durationpb.Duration{ - Seconds: 5, - }, - ClusterDiscoveryType: getClusterDiscoveryType(endpoints), - LbPolicy: envoy_config_cluster_v3.Cluster_ROUND_ROBIN, - LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{ - ClusterName: "zipkin", - Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{{ - LbEndpoints: endpoints, - }}, - }, - }, nil - default: - return nil, nil + switch cfg.TracingProvider { + case "none", "noop": // explicitly disabled from config + return false + case "": // unset + return trace.IsEnabledViaEnvironment() + default: // set to a non-empty value + return !trace.IsDisabledViaEnvironment() } } -func buildTracingHTTP(options *config.Options) (*envoy_config_trace_v3.Tracing_Http, error) { - tracingOptions, err := config.NewTracingOptions(options) - if err != nil { - return nil, fmt.Errorf("invalid tracing config: %w", err) +func applyTracingConfig( + ctx context.Context, + mgr *envoy_extensions_filters_network_http_connection_manager.HttpConnectionManager, + opts *config.Options, +) { + if !isTracingEnabled(opts) { + return + } + mgr.EarlyHeaderMutationExtensions = []*envoy_config_core_v3.TypedExtensionConfig{ + { + Name: "envoy.http.early_header_mutation.trace_context", + TypedConfig: marshalAny(&extensions_trace_context.TraceContext{}), + }, + } + mgr.RequestIdExtension = &envoy_extensions_filters_network_http_connection_manager.RequestIDExtension{ + TypedConfig: marshalAny(&extensions_uuidx.UuidxRequestIdConfig{ + PackTraceReason: wrapperspb.Bool(true), + UseRequestIdForTraceSampling: wrapperspb.Bool(true), + }), } - switch tracingOptions.Provider { - case trace.DatadogTracingProviderName: - tracingTC := protoutil.NewAny(&envoy_config_trace_v3.DatadogConfig{ - CollectorCluster: "datadog-apm", - ServiceName: tracingOptions.Service, - }) - return &envoy_config_trace_v3.Tracing_Http{ - Name: "envoy.tracers.datadog", - ConfigType: &envoy_config_trace_v3.Tracing_Http_TypedConfig{ - TypedConfig: tracingTC, - }, - }, nil - case trace.ZipkinTracingProviderName: - path := tracingOptions.ZipkinEndpoint.Path - if path == "" { - path = "/" + maxPathTagLength := uint32(1024) + if value, ok := os.LookupEnv("OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT"); ok { + if num, err := strconv.ParseUint(value, 10, 32); err == nil { + maxPathTagLength = max(64, uint32(num)) } - tracingTC := protoutil.NewAny(&envoy_config_trace_v3.ZipkinConfig{ - CollectorCluster: "zipkin", - CollectorEndpoint: path, - CollectorEndpointVersion: envoy_config_trace_v3.ZipkinConfig_HTTP_JSON, - }) - return &envoy_config_trace_v3.Tracing_Http{ - Name: "envoy.tracers.zipkin", - ConfigType: &envoy_config_trace_v3.Tracing_Http_TypedConfig{ - TypedConfig: tracingTC, + } + sampleRate := 1.0 + if value, ok := os.LookupEnv("OTEL_TRACES_SAMPLER_ARG"); ok { + if rate, err := strconv.ParseFloat(value, 64); err == nil { + sampleRate = rate + } + } + if opts.TracingSampleRate != nil { + sampleRate = *opts.TracingSampleRate + } + mgr.Tracing = &envoy_extensions_filters_network_http_connection_manager.HttpConnectionManager_Tracing{ + RandomSampling: &envoy_type_v3.Percent{Value: max(0.0, min(1.0, sampleRate)) * 100}, + Verbose: true, + SpawnUpstreamSpan: wrapperspb.Bool(true), + Provider: &tracev3.Tracing_Http{ + Name: "envoy.tracers.pomerium_otel", + ConfigType: &tracev3.Tracing_Http_TypedConfig{ + TypedConfig: marshalAny(&extensions_pomerium_otel.OpenTelemetryConfig{ + GrpcService: &envoy_config_core_v3.GrpcService{ + TargetSpecifier: &envoy_config_core_v3.GrpcService_EnvoyGrpc_{ + EnvoyGrpc: &envoy_config_core_v3.GrpcService_EnvoyGrpc{ + ClusterName: "pomerium-control-plane-grpc", + }, + }, + }, + ServiceName: "Envoy", + ResourceDetectors: []*envoy_config_core_v3.TypedExtensionConfig{ + { + Name: "envoy.tracers.opentelemetry.resource_detectors.static_config", + TypedConfig: marshalAny(&envoy_extensions_tracers_otel.StaticConfigResourceDetectorConfig{ + Attributes: map[string]string{ + "pomerium.envoy": "true", + }, + }), + }, + }, + }), }, - }, nil - default: - return nil, nil + }, + // this allows full URLs to be displayed in traces, they are otherwise truncated + MaxPathTagLength: wrapperspb.UInt32(maxPathTagLength), + } + + debugFlags := trace.DebugFlagsFromContext(ctx) + if debugFlags.Check(trace.TrackSpanReferences) { + mgr.HttpFilters = append([]*envoy_extensions_filters_network_http_connection_manager.HttpFilter{ + { + Name: "envoy.filters.http.header_to_metadata", + ConfigType: &envoy_extensions_filters_network_http_connection_manager.HttpFilter_TypedConfig{ + TypedConfig: marshalAny(&envoy_extensions_filters_http_header_to_metadata.Config{ + RequestRules: []*envoy_extensions_filters_http_header_to_metadata.Config_Rule{ + { + Header: "x-pomerium-external-parent-span", + OnHeaderPresent: &envoy_extensions_filters_http_header_to_metadata.Config_KeyValuePair{ + MetadataNamespace: "pomerium.internal", + Key: "external-parent-span", + }, + Remove: true, + }, + }, + }), + }, + }, + }, mgr.HttpFilters...) + mgr.Tracing.CustomTags = append(mgr.Tracing.CustomTags, &envoy_tracing_v3.CustomTag{ + Tag: "pomerium.external-parent-span", + Type: &envoy_tracing_v3.CustomTag_Metadata_{ + Metadata: &envoy_tracing_v3.CustomTag_Metadata{ + Kind: &metadatav3.MetadataKind{ + Kind: &metadatav3.MetadataKind_Request_{ + Request: &metadatav3.MetadataKind_Request{}, + }, + }, + MetadataKey: &metadatav3.MetadataKey{ + Key: "pomerium.internal", + Path: []*metadatav3.MetadataKey_PathSegment{ + { + Segment: &metadatav3.MetadataKey_PathSegment_Key{ + Key: "external-parent-span", + }, + }, + }, + }, + }, + }, + }) } } diff --git a/config/envoyconfig/tracing_test.go b/config/envoyconfig/tracing_test.go deleted file mode 100644 index d9eadec4a..000000000 --- a/config/envoyconfig/tracing_test.go +++ /dev/null @@ -1,135 +0,0 @@ -package envoyconfig - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/pomerium/pomerium/config" - "github.com/pomerium/pomerium/internal/testutil" -) - -func TestBuildTracingCluster(t *testing.T) { - t.Run("datadog", func(t *testing.T) { - c, err := buildTracingCluster(&config.Options{ - TracingProvider: "datadog", - }) - require.NoError(t, err) - testutil.AssertProtoJSONEqual(t, ` - { - "name": "datadog-apm", - "type": "STATIC", - "connectTimeout": "5s", - "loadAssignment": { - "clusterName": "datadog-apm", - "endpoints": [{ - "lbEndpoints": [{ - "endpoint": { - "address": { - "socketAddress": { - "address": "127.0.0.1", - "portValue": 8126 - } - } - } - }] - }] - } - } - `, c) - - c, err = buildTracingCluster(&config.Options{ - TracingProvider: "datadog", - TracingDatadogAddress: "example.com:8126", - }) - require.NoError(t, err) - testutil.AssertProtoJSONEqual(t, ` - { - "name": "datadog-apm", - "type": "STRICT_DNS", - "connectTimeout": "5s", - "loadAssignment": { - "clusterName": "datadog-apm", - "endpoints": [{ - "lbEndpoints": [{ - "endpoint": { - "address": { - "socketAddress": { - "address": "example.com", - "portValue": 8126 - } - } - } - }] - }] - } - } - `, c) - }) - t.Run("zipkin", func(t *testing.T) { - c, err := buildTracingCluster(&config.Options{ - TracingProvider: "zipkin", - ZipkinEndpoint: "https://example.com/api/v2/spans", - }) - require.NoError(t, err) - testutil.AssertProtoJSONEqual(t, ` - { - "name": "zipkin", - "type": "STRICT_DNS", - "connectTimeout": "5s", - "loadAssignment": { - "clusterName": "zipkin", - "endpoints": [{ - "lbEndpoints": [{ - "endpoint": { - "address": { - "socketAddress": { - "address": "example.com", - "portValue": 443 - } - } - } - }] - }] - } - } - `, c) - }) -} - -func TestBuildTracingHTTP(t *testing.T) { - t.Run("datadog", func(t *testing.T) { - h, err := buildTracingHTTP(&config.Options{ - TracingProvider: "datadog", - }) - require.NoError(t, err) - testutil.AssertProtoJSONEqual(t, ` - { - "name": "envoy.tracers.datadog", - "typedConfig": { - "@type": "type.googleapis.com/envoy.config.trace.v3.DatadogConfig", - "collectorCluster": "datadog-apm", - "serviceName": "pomerium" - } - } - `, h) - }) - t.Run("zipkin", func(t *testing.T) { - h, err := buildTracingHTTP(&config.Options{ - TracingProvider: "zipkin", - ZipkinEndpoint: "https://example.com/api/v2/spans", - }) - require.NoError(t, err) - testutil.AssertProtoJSONEqual(t, ` - { - "name": "envoy.tracers.zipkin", - "typedConfig": { - "@type": "type.googleapis.com/envoy.config.trace.v3.ZipkinConfig", - "collectorCluster": "zipkin", - "collectorEndpoint": "/api/v2/spans", - "collectorEndpointVersion": "HTTP_JSON" - } - } - `, h) - }) -} diff --git a/config/log.go b/config/log.go index 0b86e6ca4..545a5e69a 100644 --- a/config/log.go +++ b/config/log.go @@ -20,11 +20,6 @@ func NewLogManager(ctx context.Context, src Source) *LogManager { return mgr } -// Close closes the log manager. -func (mgr *LogManager) Close() error { - return nil -} - // OnConfigChange is called whenever configuration changes. func (mgr *LogManager) OnConfigChange(_ context.Context, cfg *Config) { if cfg == nil || cfg.Options == nil { diff --git a/config/metrics.go b/config/metrics.go index 695f9598b..39746dfb3 100644 --- a/config/metrics.go +++ b/config/metrics.go @@ -45,11 +45,6 @@ func NewMetricsManager(ctx context.Context, src Source) *MetricsManager { return mgr } -// Close closes any underlying http server. -func (mgr *MetricsManager) Close() error { - return nil -} - // OnConfigChange updates the metrics manager when configuration is changed. func (mgr *MetricsManager) OnConfigChange(ctx context.Context, cfg *Config) { mgr.mu.Lock() diff --git a/config/options.go b/config/options.go index 4100e142e..d4914933f 100644 --- a/config/options.go +++ b/config/options.go @@ -210,30 +210,10 @@ type Options struct { MetricsClientCA string `mapstructure:"metrics_client_ca" yaml:"metrics_client_ca,omitempty"` MetricsClientCAFile string `mapstructure:"metrics_client_ca_file" yaml:"metrics_client_ca_file,omitempty"` - // Tracing shared settings - TracingProvider string `mapstructure:"tracing_provider" yaml:"tracing_provider,omitempty"` - TracingSampleRate *float64 `mapstructure:"tracing_sample_rate" yaml:"tracing_sample_rate,omitempty"` - - TracingOTLPEndpoint string `mapstructure:"tracing_otlp_endpoint" yaml:"tracing_otlp_endpoint,omitempty"` - TracingOTLPProtocol string `mapstructure:"tracing_otlp_protocol" yaml:"tracing_otlp_protocol,omitempty"` - - // Datadog tracing address - TracingDatadogAddress string `mapstructure:"tracing_datadog_address" yaml:"tracing_datadog_address,omitempty"` - - // Jaeger - // - // CollectorEndpoint is the full url to the Jaeger HTTP Thrift collector. - // For example, http://localhost:14268/api/traces - TracingJaegerCollectorEndpoint string `mapstructure:"tracing_jaeger_collector_endpoint" yaml:"tracing_jaeger_collector_endpoint,omitempty"` - // AgentEndpoint instructs exporter to send spans to jaeger-agent at this address. - // For example, localhost:6831. - TracingJaegerAgentEndpoint string `mapstructure:"tracing_jaeger_agent_endpoint" yaml:"tracing_jaeger_agent_endpoint,omitempty"` - - // Zipkin - // - // ZipkinEndpoint configures the zipkin collector URI - // Example: http://zipkin:9411/api/v2/spans - ZipkinEndpoint string `mapstructure:"tracing_zipkin_endpoint" yaml:"tracing_zipkin_endpoint"` + TracingSampleRate *float64 `mapstructure:"tracing_sample_rate" yaml:"tracing_sample_rate,omitempty"` + TracingProvider string `mapstructure:"tracing_provider" yaml:"tracing_provider,omitempty"` + TracingOTLPEndpoint string `mapstructure:"tracing_otlp_endpoint" yaml:"tracing_otlp_endpoint,omitempty"` + TracingOTLPProtocol string `mapstructure:"tracing_otlp_protocol" yaml:"tracing_otlp_protocol,omitempty"` // GRPC Service Settings @@ -1527,10 +1507,6 @@ func (o *Options) ApplySettings(ctx context.Context, certsIndex *cryptutil.Certi set(&o.TracingOTLPEndpoint, settings.TracingOtlpEndpoint) set(&o.TracingOTLPProtocol, settings.TracingOtlpProtocol) setOptional(&o.TracingSampleRate, settings.TracingSampleRate) - set(&o.TracingDatadogAddress, settings.TracingDatadogAddress) - set(&o.TracingJaegerCollectorEndpoint, settings.TracingJaegerCollectorEndpoint) - set(&o.TracingJaegerAgentEndpoint, settings.TracingJaegerAgentEndpoint) - set(&o.ZipkinEndpoint, settings.TracingZipkinEndpoint) set(&o.GRPCAddr, settings.GrpcAddress) setOptional(&o.GRPCInsecure, settings.GrpcInsecure) setDuration(&o.GRPCClientTimeout, settings.GrpcClientTimeout) @@ -1619,10 +1595,6 @@ func (o *Options) ToProto() *config.Config { settings.TracingSampleRate = o.TracingSampleRate copySrcToOptionalDest(&settings.TracingOtlpEndpoint, &o.TracingOTLPEndpoint) copySrcToOptionalDest(&settings.TracingOtlpProtocol, &o.TracingOTLPProtocol) - copySrcToOptionalDest(&settings.TracingDatadogAddress, &o.TracingDatadogAddress) - copySrcToOptionalDest(&settings.TracingJaegerCollectorEndpoint, &o.TracingJaegerCollectorEndpoint) - copySrcToOptionalDest(&settings.TracingJaegerAgentEndpoint, &o.TracingJaegerAgentEndpoint) - copySrcToOptionalDest(&settings.TracingZipkinEndpoint, &o.ZipkinEndpoint) copySrcToOptionalDest(&settings.GrpcAddress, &o.GRPCAddr) settings.GrpcInsecure = o.GRPCInsecure copyOptionalDuration(&settings.GrpcClientTimeout, o.GRPCClientTimeout) diff --git a/config/options_check.go b/config/options_check.go index 261122d45..0d8ed6861 100644 --- a/config/options_check.go +++ b/config/options_check.go @@ -31,14 +31,18 @@ var reKeyPath = regexp.MustCompile(`\[\d+\]`) var ( // options that were deprecated in the config removedConfigFields = map[string]string{ - "client_ca": "https://www.pomerium.com/docs/deploy/core/upgrading#new-downstream-mtls-settings", - "client_ca_file": "https://www.pomerium.com/docs/deploy/core/upgrading#new-downstream-mtls-settings", - "idp_service_account": "https://docs.pomerium.com/docs/overview/upgrading#idp-directory-sync", - "idp_refresh_directory_timeout": "https://docs.pomerium.com/docs/overview/upgrading#idp-directory-sync", - "idp_refresh_directory_interval": "https://docs.pomerium.com/docs/overview/upgrading#idp-directory-sync", - "idp_qps": "https://docs.pomerium.com/docs/overview/upgrading#idp-directory-sync", - "routes.allowed_groups": "https://docs.pomerium.com/docs/overview/upgrading#idp-groups-policy", - "routes.set_authorization_header": "https://www.pomerium.com/docs/deploy/core/upgrading#set-authorization-header", + "client_ca": "https://www.pomerium.com/docs/deploy/core/upgrading#new-downstream-mtls-settings", + "client_ca_file": "https://www.pomerium.com/docs/deploy/core/upgrading#new-downstream-mtls-settings", + "idp_service_account": "https://docs.pomerium.com/docs/overview/upgrading#idp-directory-sync", + "idp_refresh_directory_timeout": "https://docs.pomerium.com/docs/overview/upgrading#idp-directory-sync", + "idp_refresh_directory_interval": "https://docs.pomerium.com/docs/overview/upgrading#idp-directory-sync", + "idp_qps": "https://docs.pomerium.com/docs/overview/upgrading#idp-directory-sync", + "routes.allowed_groups": "https://docs.pomerium.com/docs/overview/upgrading#idp-groups-policy", + "routes.set_authorization_header": "https://www.pomerium.com/docs/deploy/core/upgrading#set-authorization-header", + "tracing_datadog_address": "https://docs.pomerium.com/docs/overview/upgrading#removed-tracing-options", + "tracing_jaeger_collector_endpoint": "https://docs.pomerium.com/docs/overview/upgrading#removed-tracing-options", + "tracing_jaeger_agent_endpoint": "https://docs.pomerium.com/docs/overview/upgrading#removed-tracing-options", + "tracing_zipkin_endpoint": "https://docs.pomerium.com/docs/overview/upgrading#removed-tracing-options", } ignoreConfigFields = map[string]struct{}{ diff --git a/config/trace.go b/config/trace.go deleted file mode 100644 index a0cd82b71..000000000 --- a/config/trace.go +++ /dev/null @@ -1,129 +0,0 @@ -package config - -import ( - "context" - "fmt" - "reflect" - "sync" - - "github.com/rs/zerolog" - - "github.com/pomerium/pomerium/internal/log" - "github.com/pomerium/pomerium/internal/telemetry" - "github.com/pomerium/pomerium/internal/telemetry/trace" - "github.com/pomerium/pomerium/internal/urlutil" -) - -// TracingOptions are the options for tracing. -type TracingOptions = trace.TracingOptions - -// NewTracingOptions builds a new TracingOptions from core Options -func NewTracingOptions(o *Options) (*TracingOptions, error) { - sampleRate := 1.0 - if o.TracingSampleRate != nil { - sampleRate = *o.TracingSampleRate - } - tracingOpts := TracingOptions{ - Provider: o.TracingProvider, - Service: telemetry.ServiceName(o.Services), - JaegerAgentEndpoint: o.TracingJaegerAgentEndpoint, - SampleRate: sampleRate, - } - - switch o.TracingProvider { - case trace.DatadogTracingProviderName: - tracingOpts.DatadogAddress = o.TracingDatadogAddress - case trace.JaegerTracingProviderName: - if o.TracingJaegerCollectorEndpoint != "" { - jaegerCollectorEndpoint, err := urlutil.ParseAndValidateURL(o.TracingJaegerCollectorEndpoint) - if err != nil { - return nil, fmt.Errorf("config: invalid jaeger endpoint url: %w", err) - } - tracingOpts.JaegerCollectorEndpoint = jaegerCollectorEndpoint - tracingOpts.JaegerAgentEndpoint = o.TracingJaegerAgentEndpoint - } - case trace.ZipkinTracingProviderName: - zipkinEndpoint, err := urlutil.ParseAndValidateURL(o.ZipkinEndpoint) - if err != nil { - return nil, fmt.Errorf("config: invalid zipkin endpoint url: %w", err) - } - tracingOpts.ZipkinEndpoint = zipkinEndpoint - case "": - return &TracingOptions{}, nil - default: - return nil, fmt.Errorf("config: provider %s unknown", o.TracingProvider) - } - - return &tracingOpts, nil -} - -// A TraceManager manages setting up a trace exporter based on configuration options. -type TraceManager struct { - mu sync.Mutex - traceOpts *TracingOptions - provider trace.Provider -} - -// NewTraceManager creates a new TraceManager. -func NewTraceManager(ctx context.Context, src Source) *TraceManager { - ctx = log.WithContext(ctx, func(c zerolog.Context) zerolog.Context { - return c.Str("service", "trace_manager") - }) - mgr := &TraceManager{} - src.OnConfigChange(ctx, mgr.OnConfigChange) - mgr.OnConfigChange(ctx, src.GetConfig()) - return mgr -} - -// Close closes any underlying trace exporter. -func (mgr *TraceManager) Close() error { - mgr.mu.Lock() - defer mgr.mu.Unlock() - - var err error - if mgr.provider != nil { - err = mgr.provider.Unregister() - } - return err -} - -// OnConfigChange updates the manager whenever the configuration is changed. -func (mgr *TraceManager) OnConfigChange(ctx context.Context, cfg *Config) { - mgr.mu.Lock() - defer mgr.mu.Unlock() - - traceOpts, err := NewTracingOptions(cfg.Options) - if err != nil { - log.Ctx(ctx).Error().Err(err).Msg("trace: failed to build tracing options") - return - } - - if reflect.DeepEqual(traceOpts, mgr.traceOpts) { - log.Ctx(ctx).Debug().Msg("no change detected in trace options") - return - } - mgr.traceOpts = traceOpts - - if mgr.provider != nil { - _ = mgr.provider.Unregister() - mgr.provider = nil - } - - if !traceOpts.Enabled() { - return - } - - log.Ctx(ctx).Info().Interface("options", traceOpts).Msg("trace: starting exporter") - - mgr.provider, err = trace.GetProvider(traceOpts) - if err != nil { - log.Ctx(ctx).Error().Err(err).Msg("trace: failed to register exporter") - return - } - - err = mgr.provider.Register(traceOpts) - if err != nil { - log.Ctx(ctx).Error().Err(err).Msg("trace: failed to register exporter") - return - } -} diff --git a/config/trace_client.go b/config/trace_client.go new file mode 100644 index 000000000..9a8dfa858 --- /dev/null +++ b/config/trace_client.go @@ -0,0 +1,44 @@ +package config + +import ( + "errors" + "fmt" + "strings" + + "github.com/pomerium/pomerium/internal/telemetry/trace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" +) + +var ErrNoTracingConfig = errors.New("no tracing config") + +func NewTraceClientFromOptions(opts *Options) (otlptrace.Client, error) { + switch opts.TracingProvider { + case "otlp": + endpoint := opts.TracingOTLPEndpoint + protocol := opts.TracingOTLPProtocol + if protocol == "" && endpoint != "" { + // treat this field as equivalent to OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + protocol = trace.BestEffortProtocolFromOTLPEndpoint(opts.TracingOTLPEndpoint, true) + } + switch strings.ToLower(strings.TrimSpace(protocol)) { + case "grpc": + return otlptracegrpc.NewClient( + otlptracegrpc.WithEndpointURL(endpoint), + ), nil + case "http/protobuf", "": + return otlptracehttp.NewClient( + otlptracehttp.WithEndpointURL(endpoint), + ), nil + default: + return nil, fmt.Errorf(`unknown otlp trace exporter protocol %q, expected "grpc" or "http/protobuf"\n`, protocol) + } + case "none", "noop": + return trace.NoopClient{}, nil + case "": + return nil, ErrNoTracingConfig + default: + return nil, fmt.Errorf(`unknown tracing provider %q, expected one of ["otlp"]`, opts.TracingProvider) + } +} diff --git a/config/trace_test.go b/config/trace_test.go deleted file mode 100644 index 2ae2a7896..000000000 --- a/config/trace_test.go +++ /dev/null @@ -1,159 +0,0 @@ -package config - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "net/url" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "github.com/stretchr/testify/assert" - - "github.com/pomerium/pomerium/internal/telemetry/trace" -) - -func Test_NewTracingOptions(t *testing.T) { - tests := []struct { - name string - opts *Options - want *TracingOptions - wantErr bool - }{ - { - "datadog_good", - &Options{TracingProvider: "datadog"}, - &TracingOptions{Provider: "datadog", Service: "pomerium", SampleRate: 1}, - false, - }, - { - "jaeger_good", - &Options{TracingProvider: "jaeger", TracingJaegerAgentEndpoint: "foo", TracingJaegerCollectorEndpoint: "http://foo", Services: ServiceAll}, - &TracingOptions{Provider: "jaeger", JaegerAgentEndpoint: "foo", JaegerCollectorEndpoint: &url.URL{Scheme: "http", Host: "foo"}, Service: "pomerium", SampleRate: 1}, - false, - }, - { - "jaeger_bad", - &Options{TracingProvider: "jaeger", TracingJaegerAgentEndpoint: "foo", TracingJaegerCollectorEndpoint: "badurl"}, - nil, - true, - }, - { - "zipkin_good", - &Options{TracingProvider: "zipkin", ZipkinEndpoint: "https://foo/api/v1/spans", Services: ServiceAuthorize}, - &TracingOptions{Provider: "zipkin", ZipkinEndpoint: &url.URL{Scheme: "https", Host: "foo", Path: "/api/v1/spans"}, Service: "pomerium-authorize", SampleRate: 1}, - false, - }, - { - "zipkin_bad", - &Options{TracingProvider: "zipkin", ZipkinEndpoint: "notaurl"}, - nil, - true, - }, - { - "noprovider", - &Options{}, - &TracingOptions{}, - false, - }, - { - "fakeprovider", - &Options{TracingProvider: "fake"}, - nil, - true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := NewTracingOptions(tt.opts) - assert.NotEqual(t, err == nil, tt.wantErr, "unexpected error value") - assert.Empty(t, cmp.Diff(tt.want, got)) - }) - } -} - -func Test_TracingEnabled(t *testing.T) { - tests := []struct { - name string - opts *TracingOptions - want bool - }{ - {"enabled", &TracingOptions{Provider: "zipkin"}, true}, - {"not enabled", &TracingOptions{}, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, tt.opts.Enabled(), "unexpected tracing state") - }) - } -} - -func TestTraceManager(t *testing.T) { - ctx, clearTimeout := context.WithTimeout(context.Background(), time.Second*30) - defer clearTimeout() - - type Request struct { - URL string - Name string - } - - incoming := make(chan Request, 100) - - h := http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { - var objs []struct { - Name string - } - json.NewDecoder(r.Body).Decode(&objs) - for _, obj := range objs { - incoming <- Request{Name: obj.Name, URL: r.Host} - } - }) - - srv1 := httptest.NewServer(h) - defer srv1.Close() - srv2 := httptest.NewServer(h) - defer srv2.Close() - - src := NewStaticSource(&Config{Options: &Options{ - TracingProvider: "zipkin", - ZipkinEndpoint: srv1.URL, - }}) - - _ = NewTraceManager(ctx, src) - - _, span := trace.StartSpan(ctx, "Example") - span.End() - - src.SetConfig(ctx, &Config{Options: &Options{ - TracingProvider: "zipkin", - ZipkinEndpoint: srv2.URL, - }}) - - _, span = trace.StartSpan(ctx, "Example") - span.End() - - expect := map[Request]struct{}{ - {Name: "example", URL: srv1.Listener.Addr().String()}: {}, - {Name: "example", URL: srv2.Listener.Addr().String()}: {}, - } - - for len(expect) > 0 { - var req Request - select { - case <-ctx.Done(): - t.Error("timeout waiting for requests") - return - case req = <-incoming: - } - - if _, ok := expect[req]; ok { - delete(expect, req) - } else { - t.Error("unexpected request", req) - return - } - } -} diff --git a/databroker/cache.go b/databroker/cache.go index 8105ed932..f3110ad87 100644 --- a/databroker/cache.go +++ b/databroker/cache.go @@ -10,6 +10,7 @@ import ( "time" "github.com/rs/zerolog" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/metadata" @@ -18,7 +19,7 @@ import ( "github.com/pomerium/pomerium/internal/atomicutil" "github.com/pomerium/pomerium/internal/events" "github.com/pomerium/pomerium/internal/log" - "github.com/pomerium/pomerium/internal/telemetry" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/version" "github.com/pomerium/pomerium/pkg/cryptutil" "github.com/pomerium/pomerium/pkg/envoy/files" @@ -28,6 +29,7 @@ import ( "github.com/pomerium/pomerium/pkg/identity" "github.com/pomerium/pomerium/pkg/identity/legacymanager" "github.com/pomerium/pomerium/pkg/identity/manager" + oteltrace "go.opentelemetry.io/otel/trace" ) // DataBroker represents the databroker service. The databroker service is a simple interface @@ -43,6 +45,8 @@ type DataBroker struct { localGRPCServer *grpc.Server localGRPCConnection *grpc.ClientConn sharedKey *atomicutil.Value[[]byte] + tracerProvider oteltrace.TracerProvider + tracer oteltrace.Tracer } type Options struct { @@ -87,9 +91,12 @@ func New(ctx context.Context, cfg *config.Config, eventsMgr *events.Manager, opt ), ) + tracerProvider := trace.NewTracerProvider(ctx, "Data Broker") + tracer := tracerProvider.Tracer(trace.PomeriumCoreTracer) // No metrics handler because we have one in the control plane. Add one // if we no longer register with that grpc Server localGRPCServer := grpc.NewServer( + grpc.StatsHandler(otelgrpc.NewServerHandler(otelgrpc.WithTracerProvider(tracerProvider))), grpc.ChainStreamInterceptor(log.StreamServerInterceptor(log.Ctx(ctx)), si), grpc.ChainUnaryInterceptor(log.UnaryServerInterceptor(log.Ctx(ctx)), ui), ) @@ -100,12 +107,11 @@ func New(ctx context.Context, cfg *config.Config, eventsMgr *events.Manager, opt } sharedKeyValue := atomicutil.NewValue(sharedKey) - clientStatsHandler := telemetry.NewGRPCClientStatsHandler(cfg.Options.Services) clientDialOptions := []grpc.DialOption{ grpc.WithInsecure(), - grpc.WithChainUnaryInterceptor(clientStatsHandler.UnaryInterceptor, grpcutil.WithUnarySignedJWT(sharedKeyValue.Load)), + grpc.WithChainUnaryInterceptor(grpcutil.WithUnarySignedJWT(sharedKeyValue.Load)), grpc.WithChainStreamInterceptor(grpcutil.WithStreamSignedJWT(sharedKeyValue.Load)), - grpc.WithStatsHandler(clientStatsHandler.Handler), + grpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(tracerProvider))), } ctx = log.WithContext(ctx, func(c zerolog.Context) zerolog.Context { @@ -120,7 +126,7 @@ func New(ctx context.Context, cfg *config.Config, eventsMgr *events.Manager, opt return nil, err } - dataBrokerServer, err := newDataBrokerServer(ctx, cfg) + dataBrokerServer, err := newDataBrokerServer(ctx, tracerProvider, cfg) if err != nil { return nil, err } @@ -133,6 +139,8 @@ func New(ctx context.Context, cfg *config.Config, eventsMgr *events.Manager, opt localGRPCConnection: localGRPCConnection, sharedKey: sharedKeyValue, eventsMgr: eventsMgr, + tracerProvider: tracerProvider, + tracer: tracer, } c.Register(c.localGRPCServer) @@ -202,7 +210,7 @@ func (c *DataBroker) update(ctx context.Context, cfg *config.Config) error { }, c.legacyManagerOptions...) if cfg.Options.SupportsUserRefresh() { - authenticator, err := identity.NewAuthenticator(oauthOptions) + authenticator, err := identity.NewAuthenticator(ctx, c.tracerProvider, oauthOptions) if err != nil { log.Ctx(ctx).Error().Err(err).Msg("databroker: failed to create authenticator") } else { diff --git a/databroker/databroker.go b/databroker/databroker.go index 7817f403d..94087a8a4 100644 --- a/databroker/databroker.go +++ b/databroker/databroker.go @@ -5,6 +5,7 @@ import ( "context" "fmt" + oteltrace "go.opentelemetry.io/otel/trace" "google.golang.org/protobuf/types/known/emptypb" "github.com/pomerium/pomerium/config" @@ -23,7 +24,7 @@ type dataBrokerServer struct { } // newDataBrokerServer creates a new databroker service server. -func newDataBrokerServer(ctx context.Context, cfg *config.Config) (*dataBrokerServer, error) { +func newDataBrokerServer(ctx context.Context, tracerProvider oteltrace.TracerProvider, cfg *config.Config) (*dataBrokerServer, error) { srv := &dataBrokerServer{ sharedKey: atomicutil.NewValue([]byte{}), } @@ -33,7 +34,7 @@ func newDataBrokerServer(ctx context.Context, cfg *config.Config) (*dataBrokerSe return nil, err } - srv.server = databroker.New(ctx, opts...) + srv.server = databroker.New(ctx, tracerProvider, opts...) srv.setKey(cfg) return srv, nil } diff --git a/databroker/databroker_test.go b/databroker/databroker_test.go index 1d1fa46bf..5e379aafb 100644 --- a/databroker/databroker_test.go +++ b/databroker/databroker_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -29,7 +30,7 @@ var lis *bufconn.Listener func init() { lis = bufconn.Listen(bufSize) s := grpc.NewServer() - internalSrv := internal_databroker.New(context.Background()) + internalSrv := internal_databroker.New(context.Background(), trace.NewNoopTracerProvider()) srv := &dataBrokerServer{server: internalSrv, sharedKey: atomicutil.NewValue([]byte{})} databroker.RegisterDataBrokerServiceServer(s, srv) diff --git a/go.mod b/go.mod index 66406d8a2..f29aaf075 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,8 @@ go 1.23.0 require ( cloud.google.com/go/storage v1.49.0 - contrib.go.opencensus.io/exporter/jaeger v0.2.1 contrib.go.opencensus.io/exporter/prometheus v0.4.2 - contrib.go.opencensus.io/exporter/zipkin v0.1.2 github.com/CAFxX/httpcompression v0.0.9 - github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20200406135749-5c268882acf0 github.com/VictoriaMetrics/fastcache v1.12.2 github.com/aws/aws-sdk-go-v2 v1.32.7 github.com/aws/aws-sdk-go-v2/config v1.28.7 @@ -20,7 +17,7 @@ require ( github.com/cloudflare/circl v1.5.0 github.com/coreos/go-oidc/v3 v3.11.0 github.com/docker/docker v27.4.1+incompatible - github.com/envoyproxy/go-control-plane/envoy v1.32.2 + github.com/envoyproxy/go-control-plane/envoy v1.32.3 github.com/envoyproxy/protoc-gen-validate v1.1.0 github.com/go-chi/chi/v5 v5.2.0 github.com/go-jose/go-jose/v3 v3.0.3 @@ -46,11 +43,11 @@ require ( github.com/natefinch/atomic v1.0.1 github.com/oapi-codegen/runtime v1.1.1 github.com/open-policy-agent/opa v1.0.0 - github.com/openzipkin/zipkin-go v0.4.3 github.com/peterbourgon/ff/v3 v3.4.0 github.com/pires/go-proxyproto v0.8.0 github.com/pomerium/csrf v1.7.0 github.com/pomerium/datasource v0.18.2-0.20221108160055-c6134b5ed524 + github.com/pomerium/envoy-custom v1.32.4-0.20250114182541-6f6d2147bea6 github.com/pomerium/protoutil v0.0.0-20240813175624-47b7ac43ff46 github.com/pomerium/webauthn v0.0.0-20240603205124-0428df511172 github.com/prometheus/client_golang v1.20.5 @@ -70,15 +67,20 @@ require ( github.com/volatiletech/null/v9 v9.0.0 github.com/yuin/gopher-lua v1.1.1 go.opencensus.io v0.24.0 + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.57.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 + go.opentelemetry.io/contrib/propagators/autoprop v0.57.0 go.opentelemetry.io/otel v1.33.0 - go.opentelemetry.io/otel/bridge/opencensus v1.33.0 - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.33.0 + go.opentelemetry.io/otel/bridge/opencensus v1.32.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0 go.opentelemetry.io/otel/metric v1.33.0 go.opentelemetry.io/otel/sdk v1.33.0 - go.opentelemetry.io/otel/sdk/metric v1.33.0 + go.opentelemetry.io/otel/sdk/metric v1.32.0 go.opentelemetry.io/otel/trace v1.33.0 + go.opentelemetry.io/proto/otlp v1.4.0 go.uber.org/automaxprocs v1.6.0 go.uber.org/mock v0.5.0 go.uber.org/zap v1.27.0 @@ -91,7 +93,7 @@ require ( google.golang.org/api v0.214.0 google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 google.golang.org/grpc v1.69.2 - google.golang.org/protobuf v1.36.1 + google.golang.org/protobuf v1.36.2 gopkg.in/yaml.v3 v3.0.1 namespacelabs.dev/go-filenotify v0.0.0-20220511192020-53ea11be7eaa sigs.k8s.io/yaml v1.4.0 @@ -107,7 +109,6 @@ require ( cloud.google.com/go/monitoring v1.21.2 // indirect dario.cat/mergo v1.0.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect - github.com/DataDog/datadog-go v3.5.0+incompatible // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect @@ -133,7 +134,7 @@ require ( github.com/aws/smithy-go v1.22.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/caddyserver/zerossl v0.1.3 // indirect - github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect + github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect @@ -142,7 +143,7 @@ require ( github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/envoyproxy/go-control-plane v0.13.2 // indirect + github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/fxamacker/cbor/v2 v2.6.0 // indirect @@ -195,7 +196,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect - github.com/philhofer/fwd v1.1.2 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -218,10 +219,8 @@ require ( github.com/tchap/go-patricia/v2 v2.3.1 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect - github.com/tinylib/msgp v1.1.8 // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect - github.com/uber/jaeger-client-go v2.25.0+incompatible // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect @@ -232,10 +231,10 @@ require ( github.com/zeebo/xxh3 v1.0.2 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.31.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 // indirect - go.opentelemetry.io/proto/otlp v1.4.0 // indirect + go.opentelemetry.io/contrib/propagators/aws v1.32.0 // indirect + go.opentelemetry.io/contrib/propagators/b3 v1.32.0 // indirect + go.opentelemetry.io/contrib/propagators/jaeger v1.32.0 // indirect + go.opentelemetry.io/contrib/propagators/ot v1.32.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect golang.org/x/mod v0.20.0 // indirect @@ -243,7 +242,6 @@ require ( golang.org/x/tools v0.24.0 // indirect google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect - gopkg.in/DataDog/dd-trace-go.v1 v1.22.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 962193b06..e77fe687b 100644 --- a/go.sum +++ b/go.sum @@ -52,12 +52,8 @@ cloud.google.com/go/storage v1.49.0 h1:zenOPBOWHCnojRd9aJZAyQXBYqkJkdQS42dxL55CI cloud.google.com/go/storage v1.49.0/go.mod h1:k1eHhhpLvrPjVGfo0mOUPEJ4Y2+a/Hv5PiwehZI9qGU= cloud.google.com/go/trace v1.11.2 h1:4ZmaBdL8Ng/ajrgKqY5jfvzqMXbrDcBsUGXOT9aqTtI= cloud.google.com/go/trace v1.11.2/go.mod h1:bn7OwXd4pd5rFuAnTrzBuoZ4ax2XQeG3qNgYmfCy0Io= -contrib.go.opencensus.io/exporter/jaeger v0.2.1 h1:yGBYzYMewVL0yO9qqJv3Z5+IRhPdU7e9o/2oKpX4YvI= -contrib.go.opencensus.io/exporter/jaeger v0.2.1/go.mod h1:Y8IsLgdxqh1QxYxPC5IgXVmBaeLUeQFfBeBi9PbeZd0= contrib.go.opencensus.io/exporter/prometheus v0.4.2 h1:sqfsYl5GIY/L570iT+l93ehxaWJs2/OwXtiWwew3oAg= contrib.go.opencensus.io/exporter/prometheus v0.4.2/go.mod h1:dvEHbiKmgvbr5pjaF9fpw1KeYcjrnC1J8B+JKjsZyRQ= -contrib.go.opencensus.io/exporter/zipkin v0.1.2 h1:YqE293IZrKtqPnpwDPH/lOqTWD/s3Iwabycam74JV3g= -contrib.go.opencensus.io/exporter/zipkin v0.1.2/go.mod h1:mP5xM3rrgOjpn79MM8fZbj3gsxcuytSqtH0dxSWW1RE= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -69,10 +65,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CAFxX/httpcompression v0.0.9 h1:0ue2X8dOLEpxTm8tt+OdHcgA+gbDge0OqFQWGKSqgrg= github.com/CAFxX/httpcompression v0.0.9/go.mod h1:XX8oPZA+4IDcfZ0A71Hz0mZsv/YJOgYygkFhizVPilM= -github.com/DataDog/datadog-go v3.5.0+incompatible h1:AShr9cqkF+taHjyQgcBcQUt/ZNK+iPq4ROaZwSX5c/U= -github.com/DataDog/datadog-go v3.5.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20200406135749-5c268882acf0 h1:Y6HFfo8UuntPOpfmUmLb0o3MNYKfUuH2aNmvypsDbY4= -github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20200406135749-5c268882acf0/go.mod h1:/VV3EFO/hTNQZHAqaj+CPGy2+ioFrP4EX3iRwozubhQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 h1:3c8yed4lgqTt+oTQ+JNMDo+F4xprBf+O/il4ZC0nRLw= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0/go.mod h1:obipzmGjfSjam60XLwGfqUkJsfiheAl+TUjG+4yzyPM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 h1:UQ0AhxogsIRZDkElkblfnwjc3IaltCm2HUMvezQaL7s= @@ -86,8 +78,6 @@ github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= github.com/agnivade/levenshtein v1.2.0 h1:U9L4IOT0Y3i0TIlUIDJ7rVUziKi/zPbrJGaFrtYH3SY= @@ -172,8 +162,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys= github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 h1:QVw89YDxXxEe+l8gU8ETbOasdwEV+avkR75ZzsVV9WI= -github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3 h1:boJj011Hh+874zpIySeApCX4GeOjPl9qhRF3QuIZq+Q= +github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= @@ -206,17 +196,13 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.13.2 h1:HoRou/hxH5VAnylep8UAh0jFAllMY0UlP0TwPmIEgYI= -github.com/envoyproxy/go-control-plane v0.13.2/go.mod h1:mcYj6+AKxG86c/jKeZsCIWv8oLzhR+SJynG0TB94Xw8= -github.com/envoyproxy/go-control-plane/envoy v1.32.2 h1:zidqwmijfcbyKqVxjQDFx042PgX+p9U+/fu/f9VtSk8= -github.com/envoyproxy/go-control-plane/envoy v1.32.2/go.mod h1:eR2SOX2IedqlPvmiKjUH7Wu//S602JKI7HPC/L3SRq8= +github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= +github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= +github.com/envoyproxy/go-control-plane/envoy v1.32.3 h1:hVEaommgvzTjTd4xCaFd+kEQ2iYBtGxP6luyLrx6uOk= +github.com/envoyproxy/go-control-plane/envoy v1.32.3/go.mod h1:F6hWupPfh75TBXGKA++MCT/CZHFq5r9/uwt/kQYkZfE= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= @@ -277,9 +263,7 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -316,7 +300,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/brotli/go/cbrotli v0.0.0-20230829110029-ed738e842d2f h1:jopqB+UTSdJGEJT8tEqYyE29zN91fi2827oLET8tl7k= @@ -372,8 +355,6 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.14.0 h1:f+jMrjBPl+DL9nI4IQzLUxMq7XrAqFYB7hBPqMNIe8o= github.com/googleapis/gax-go/v2 v2.14.0/go.mod h1:lhBCnjdLrWRaPvLWhmc8IS24m9mr07qSYnHncrgo+zk= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= @@ -450,7 +431,6 @@ github.com/libdns/libdns v0.2.2 h1:O6ws7bAfRPaBsgAYt8MDe2HcNBGC29hkZ9MX2eUSX3s= github.com/libdns/libdns v0.2.2/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ= github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae h1:dIZY4ULFcto4tAFlj1FYZl8ztUZ13bdq+PLY+NOfbyI= github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= -github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/martinlindhe/base36 v1.1.1 h1:1F1MZ5MGghBXDZ2KJ3QfxmiydlWOGB8HCEtkap5NkVg= @@ -505,13 +485,11 @@ github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+ github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.19.1 h1:QXgq3Z8Crl5EL1WBAC98A5sEBHARrAJNzAmMxzLcRF0= github.com/onsi/ginkgo/v2 v2.19.1/go.mod h1:O3DtEWQkPa/F7fBMgmZQKKsluAy8pd3rEQdrjkPb9zA= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= @@ -522,18 +500,10 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/openzipkin/zipkin-go v0.4.3 h1:9EGwpqkgnwdEIJ+Od7QVSEIH+ocmm5nPat0G7sjsSdg= -github.com/openzipkin/zipkin-go v0.4.3/go.mod h1:M9wCJZFWCo2RiY+o1eBCEMe0Dp2S5LDHcMZmk3RmK7c= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/peterbourgon/ff/v3 v3.4.0 h1:QBvM/rizZM1cB0p0lGMdmR7HxZeI/ZrBWB4DqLkMUBc= github.com/peterbourgon/ff/v3 v3.4.0/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ= -github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= -github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= -github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= -github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1 h1:VGcrWe3yk6o+t7BdVNy5UDPWa4OZuDWtE1W1ZbS7Kyw= -github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -543,7 +513,6 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -553,6 +522,8 @@ github.com/pomerium/csrf v1.7.0 h1:Qp4t6oyEod3svQtKfJZs589mdUTWKVf7q0PgCKYCshY= github.com/pomerium/csrf v1.7.0/go.mod h1:hAPZV47mEj2T9xFs+ysbum4l7SF1IdrryYaY6PdoIqw= github.com/pomerium/datasource v0.18.2-0.20221108160055-c6134b5ed524 h1:3YQY1sb54tEEbr0L73rjHkpLB0IB6qh3zl1+XQbMLis= github.com/pomerium/datasource v0.18.2-0.20221108160055-c6134b5ed524/go.mod h1:7fGbUYJnU8RcxZJvUvhukOIBv1G7LWDAHMfDxAf5+Y0= +github.com/pomerium/envoy-custom v1.32.4-0.20250114182541-6f6d2147bea6 h1:QLVgpx23jcbgR9qJzIicJ+uXGjQXO0GAy55SCo0Jd9o= +github.com/pomerium/envoy-custom v1.32.4-0.20250114182541-6f6d2147bea6/go.mod h1:afbaKE6YfshVUOrYc6XWUWfZcXencWmi1jTc00ki0Oo= github.com/pomerium/protoutil v0.0.0-20240813175624-47b7ac43ff46 h1:NRTg8JOXCxcIA1lAgD74iYud0rbshbWOB3Ou4+Huil8= github.com/pomerium/protoutil v0.0.0-20240813175624-47b7ac43ff46/go.mod h1:QqZmx6ZgPxz18va7kqoT4t/0yJtP7YFIDiT/W2n2fZ4= github.com/pomerium/webauthn v0.0.0-20240603205124-0428df511172 h1:TqoPqRgXSHpn+tEJq6H72iCS5pv66j3rPprThUEZg0E= @@ -598,7 +569,6 @@ github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= github.com/quic-go/quic-go v0.48.2 h1:wsKXZPeGWpMpCGSWqOcqpW2wZYic/8T3aqiOID0/KWE= github.com/quic-go/quic-go v0.48.2/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -644,7 +614,6 @@ github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+ github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/sryoya/protorand v0.0.0-20240429201223-e7440656b2a4 h1:/jKH9ivHOUkahZs3zPfJfOmkXDFB6OdsHZ4W8gyDb/c= github.com/sryoya/protorand v0.0.0-20240429201223-e7440656b2a4/go.mod h1:9a23nlv6vzBeVlQq6JQCjljZ6sfzsB6aha1m5Ly1W2Y= -github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -677,17 +646,12 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= -github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= -github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= github.com/tniswong/go.rfcx v0.0.0-20181019234604-07783c52761f h1:C43EMGXFtvYf/zunHR6ivZV7Z6ytg73t0GXwYyicXMQ= github.com/tniswong/go.rfcx v0.0.0-20181019234604-07783c52761f/go.mod h1:N+sR0vLSCTtI6o06PMWsjMB4TVqqDttKNq4iC9wvxVY= -github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U= -github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/valyala/gozstd v1.20.1 h1:xPnnnvjmaDDitMFfDxmQ4vpx0+3CdTg2o3lALvXTU/g= github.com/valyala/gozstd v1.20.1/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= @@ -730,30 +694,40 @@ go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJyS go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/contrib/detectors/gcp v1.31.0 h1:G1JQOreVrfhRkner+l4mrGxmfqYCAuy76asTDAo0xsA= go.opentelemetry.io/contrib/detectors/gcp v1.31.0/go.mod h1:tzQL6E1l+iV44YFTkcAeNQqzXUiekSYP9jjJjXwEd00= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.57.0 h1:qtFISDHKolvIxzSs0gIaiPUPR0Cucb0F2coHC7ZLdps= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.57.0/go.mod h1:Y+Pop1Q6hCOnETWTW4NROK/q1hv50hM7yDaUTjG8lp8= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= +go.opentelemetry.io/contrib/propagators/autoprop v0.57.0 h1:bNPJOdT5154XxzeFmrh8R+PXnV4t3TZEczy8gHEpcpg= +go.opentelemetry.io/contrib/propagators/autoprop v0.57.0/go.mod h1:Tb0j0mK+QatKdCxCKPN7CSzc7kx/q34/KaohJx/N96s= +go.opentelemetry.io/contrib/propagators/aws v1.32.0 h1:NELzr8bW7a7aHVZj5gaep1PfkvoSCGx+1qNGZx/uhhU= +go.opentelemetry.io/contrib/propagators/aws v1.32.0/go.mod h1:XKMrzHNka3eOA+nGEcNKYVL9s77TAhkwQEynYuaRFnQ= +go.opentelemetry.io/contrib/propagators/b3 v1.32.0 h1:MazJBz2Zf6HTN/nK/s3Ru1qme+VhWU5hm83QxEP+dvw= +go.opentelemetry.io/contrib/propagators/b3 v1.32.0/go.mod h1:B0s70QHYPrJwPOwD1o3V/R8vETNOG9N3qZf4LDYvA30= +go.opentelemetry.io/contrib/propagators/jaeger v1.32.0 h1:K/fOyTMD6GELKTIJBaJ9k3ppF2Njt8MeUGBOwfaWXXA= +go.opentelemetry.io/contrib/propagators/jaeger v1.32.0/go.mod h1:ISE6hda//MTWvtngG7p4et3OCngsrTVfl7c6DjN17f8= +go.opentelemetry.io/contrib/propagators/ot v1.32.0 h1:Poy02A4wOZubHyd2hpHPDgZW+rn6EIq0vCwTZJ6Lmu8= +go.opentelemetry.io/contrib/propagators/ot v1.32.0/go.mod h1:cbhaURV+VR3NIMarzDYZU1RDEkXG1fNd1WMP1XCcGkY= go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= -go.opentelemetry.io/otel/bridge/opencensus v1.33.0 h1:sGcK0Wif7sPG3GZG9z8b+tpRZiUHwv27WgmsaZ1wgzM= -go.opentelemetry.io/otel/bridge/opencensus v1.33.0/go.mod h1:LXJy68HiJRu+2yJmVnbDn/F9JS9Kxfsj5WpA5t5NfRY= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.33.0 h1:7F29RDmnlqk6B5d+sUqemt8TBfDqxryYW5gX6L74RFA= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.33.0/go.mod h1:ZiGDq7xwDMKmWDrN1XsXAj0iC7hns+2DhxBFSncNHSE= +go.opentelemetry.io/otel/bridge/opencensus v1.32.0 h1:OVbbFgPG60UolI8ZUs+Z75NnKiO0C9QltXBrqUDImS0= +go.opentelemetry.io/otel/bridge/opencensus v1.32.0/go.mod h1:J5SEiJNu6zzqpcA6+AVpxUKzxNocUMsefgHRpS8zdW8= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0 h1:j7ZSD+5yn+lo3sGV69nW04rRR0jhYnBwjuX3r0HvnK0= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0/go.mod h1:WXbYJTUaZXAbYd8lbgGuvih0yuCfOFC5RJoYnoLcGz8= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 h1:j9+03ymgYhPKmeXGk5Zu+cIZOlVzd9Zv7QIiyItjFBU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0/go.mod h1:Y5+XiUG4Emn1hTfciPzGPJaSI+RpDts6BnCIir0SLqk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0 h1:cMyu9O88joYEaI47CnQkxO1XZdpoTF9fEnW2duIddhw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0/go.mod h1:6Am3rn7P9TVVeXYG+wtcGE7IE1tsQ+bP3AuWcKt/gOI= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0 h1:WDdP9acbMYjbKIyJUhTvtzj601sVJOqgWdUxSdR/Ysc= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I= go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= -go.opentelemetry.io/otel/sdk/metric v1.33.0 h1:Gs5VK9/WUJhNXZgn8MR6ITatvAmKeIuCtNbsP3JkNqU= -go.opentelemetry.io/otel/sdk/metric v1.33.0/go.mod h1:dL5ykHZmm1B1nVRk9dDjChwDmt81MjVp3gLkQRwKf/Q= +go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= +go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= @@ -811,7 +785,6 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= @@ -852,7 +825,6 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= @@ -935,7 +907,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -946,7 +917,6 @@ golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= @@ -959,7 +929,6 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= @@ -1014,7 +983,6 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= @@ -1082,7 +1050,6 @@ google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY= google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= @@ -1111,10 +1078,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= -google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= -gopkg.in/DataDog/dd-trace-go.v1 v1.22.0 h1:gpWsqqkwUldNZXGJqT69NU9MdEDhLboK1C4nMgR0MWw= -gopkg.in/DataDog/dd-trace-go.v1 v1.22.0/go.mod h1:DVp8HmDh8PuTu2Z0fVVlBsyWaC++fzwVCaGWylTe3tg= +google.golang.org/protobuf v1.36.2 h1:R8FeyR1/eLmkutZOM5CWghmo5itiG9z0ktFlTVLuTmU= +google.golang.org/protobuf v1.36.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/atomicutil/value.go b/internal/atomicutil/value.go index 665ff3241..279b7ad32 100644 --- a/internal/atomicutil/value.go +++ b/internal/atomicutil/value.go @@ -33,3 +33,14 @@ func (v *Value[T]) Load() T { func (v *Value[T]) Store(val T) { v.value.Store(val) } + +// Swap swaps the value atomically. +func (v *Value[T]) Swap(val T) T { + old, _ := v.value.Swap(val).(T) + return old +} + +// Swap swaps the value atomically. +func (v *Value[T]) CompareAndSwap(old, new T) bool { + return v.value.CompareAndSwap(old, new) +} diff --git a/internal/authenticateflow/authenticateflow.go b/internal/authenticateflow/authenticateflow.go index bdd7f8be8..2fd7edafe 100644 --- a/internal/authenticateflow/authenticateflow.go +++ b/internal/authenticateflow/authenticateflow.go @@ -4,11 +4,17 @@ package authenticateflow import ( + "context" "fmt" "time" + oteltrace "go.opentelemetry.io/otel/trace" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/structpb" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/pkg/grpc" "github.com/pomerium/pomerium/pkg/grpc/user" "github.com/pomerium/pomerium/pkg/identity" @@ -33,3 +39,23 @@ func populateUserFromClaims(u *user.User, claims map[string]any) { u.Claims[k] = vs } } + +var outboundDatabrokerTraceClientOpts = []trace.ClientStatsHandlerOption{ + trace.WithStatsInterceptor(ignoreNotFoundErrors), +} + +func ignoreNotFoundErrors(ctx context.Context, rs stats.RPCStats) stats.RPCStats { + if end, ok := rs.(*stats.End); ok && end.IsClient() { + if status.Code(end.Error) == codes.NotFound { + oteltrace.SpanFromContext(ctx).AddEvent("status code: NotFound") + return &stats.End{ + Client: end.Client, + BeginTime: end.BeginTime, + EndTime: end.EndTime, + Trailer: end.Trailer, + Error: nil, + } + } + } + return rs +} diff --git a/internal/authenticateflow/stateful.go b/internal/authenticateflow/stateful.go index 44050325d..58d62d516 100644 --- a/internal/authenticateflow/stateful.go +++ b/internal/authenticateflow/stateful.go @@ -9,7 +9,11 @@ import ( "net/url" "time" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "go.opentelemetry.io/otel" + oteltrace "go.opentelemetry.io/otel/trace" "golang.org/x/oauth2" + googlegrpc "google.golang.org/grpc" "google.golang.org/protobuf/types/known/timestamppb" "github.com/pomerium/pomerium/config" @@ -19,6 +23,7 @@ import ( "github.com/pomerium/pomerium/internal/httputil" "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/sessions" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/urlutil" "github.com/pomerium/pomerium/pkg/cryptutil" "github.com/pomerium/pomerium/pkg/grpc" @@ -56,7 +61,7 @@ type Stateful struct { // NewStateful initializes the authentication flow for the given configuration // and session store. -func NewStateful(ctx context.Context, cfg *config.Config, sessionStore sessions.SessionStore) (*Stateful, error) { +func NewStateful(ctx context.Context, tracerProvider oteltrace.TracerProvider, cfg *config.Config, sessionStore sessions.SessionStore) (*Stateful, error) { s := &Stateful{ sessionDuration: cfg.Options.CookieExpire, sessionStore: sessionStore, @@ -94,7 +99,10 @@ func NewStateful(ctx context.Context, cfg *config.Config, sessionStore sessions. InstallationID: cfg.Options.InstallationID, ServiceName: cfg.Options.Services, SignedJWTKey: s.sharedKey, - }) + }, googlegrpc.WithStatsHandler(trace.NewClientStatsHandler( + otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(tracerProvider)), + outboundDatabrokerTraceClientOpts..., + ))) if err != nil { return nil, err } @@ -316,7 +324,7 @@ func (s *Stateful) LogAuthenticateEvent(*http.Request) {} // AuthenticateSignInURL returns a URL to redirect the user to the authenticate // domain. func (s *Stateful) AuthenticateSignInURL( - _ context.Context, queryParams url.Values, redirectURL *url.URL, idpID string, + ctx context.Context, queryParams url.Values, redirectURL *url.URL, idpID string, ) (string, error) { signinURL := s.authenticateURL.ResolveReference(&url.URL{ Path: "/.pomerium/sign_in", @@ -327,6 +335,7 @@ func (s *Stateful) AuthenticateSignInURL( } queryParams.Set(urlutil.QueryRedirectURI, redirectURL.String()) queryParams.Set(urlutil.QueryIdentityProviderID, idpID) + otel.GetTextMapPropagator().Inject(ctx, trace.PomeriumURLQueryCarrier(queryParams)) signinURL.RawQuery = queryParams.Encode() redirectTo := urlutil.NewSignedURL(s.sharedKey, signinURL).String() diff --git a/internal/authenticateflow/stateful_test.go b/internal/authenticateflow/stateful_test.go index 173374410..aa6e85841 100644 --- a/internal/authenticateflow/stateful_test.go +++ b/internal/authenticateflow/stateful_test.go @@ -15,6 +15,7 @@ import ( "github.com/go-jose/go-jose/v3/jwt" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace" "go.uber.org/mock/gomock" "golang.org/x/oauth2" "google.golang.org/grpc" @@ -69,7 +70,7 @@ func TestStatefulSignIn(t *testing.T) { tt := tt t.Run(tt.name, func(t *testing.T) { sessionStore := &mstore.Store{SaveError: tt.saveError} - flow, err := NewStateful(context.Background(), &config.Config{Options: opts}, sessionStore) + flow, err := NewStateful(context.Background(), trace.NewNoopTracerProvider(), &config.Config{Options: opts}, sessionStore) if err != nil { t.Fatal(err) } @@ -123,12 +124,12 @@ func TestStatefulAuthenticateSignInURL(t *testing.T) { opts.AuthenticateURLString = "https://authenticate.example.com" key := cryptutil.NewKey() opts.SharedKey = base64.StdEncoding.EncodeToString(key) - flow, err := NewStateful(context.Background(), &config.Config{Options: opts}, nil) + flow, err := NewStateful(context.Background(), trace.NewNoopTracerProvider(), &config.Config{Options: opts}, nil) require.NoError(t, err) t.Run("NilQueryParams", func(t *testing.T) { redirectURL := &url.URL{Scheme: "https", Host: "example.com"} - u, err := flow.AuthenticateSignInURL(nil, nil, redirectURL, "fake-idp-id") + u, err := flow.AuthenticateSignInURL(context.Background(), nil, redirectURL, "fake-idp-id") assert.NoError(t, err) parsed, _ := url.Parse(u) assert.NoError(t, urlutil.NewSignedURL(key, parsed).Validate()) @@ -143,7 +144,7 @@ func TestStatefulAuthenticateSignInURL(t *testing.T) { redirectURL := &url.URL{Scheme: "https", Host: "example.com"} q := url.Values{} q.Set("foo", "bar") - u, err := flow.AuthenticateSignInURL(nil, q, redirectURL, "fake-idp-id") + u, err := flow.AuthenticateSignInURL(context.Background(), q, redirectURL, "fake-idp-id") assert.NoError(t, err) parsed, _ := url.Parse(u) assert.NoError(t, urlutil.NewSignedURL(key, parsed).Validate()) @@ -238,7 +239,7 @@ func TestStatefulCallback(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - flow, err := NewStateful(context.Background(), &config.Config{Options: opts}, tt.sessionStore) + flow, err := NewStateful(context.Background(), trace.NewNoopTracerProvider(), &config.Config{Options: opts}, tt.sessionStore) if err != nil { t.Fatal(err) } @@ -289,7 +290,7 @@ func TestStatefulCallback(t *testing.T) { func TestStatefulRevokeSession(t *testing.T) { opts := config.NewDefaultOptions() - flow, err := NewStateful(context.Background(), &config.Config{Options: opts}, nil) + flow, err := NewStateful(context.Background(), trace.NewNoopTracerProvider(), &config.Config{Options: opts}, nil) require.NoError(t, err) ctrl := gomock.NewController(t) @@ -367,7 +368,7 @@ func TestPersistSession(t *testing.T) { opts := config.NewDefaultOptions() opts.CookieExpire = 4 * time.Hour - flow, err := NewStateful(context.Background(), &config.Config{Options: opts}, nil) + flow, err := NewStateful(context.Background(), trace.NewNoopTracerProvider(), &config.Config{Options: opts}, nil) require.NoError(t, err) ctrl := gomock.NewController(t) diff --git a/internal/authenticateflow/stateless.go b/internal/authenticateflow/stateless.go index e89e8199a..ef188eae9 100644 --- a/internal/authenticateflow/stateless.go +++ b/internal/authenticateflow/stateless.go @@ -10,6 +10,7 @@ import ( "github.com/go-jose/go-jose/v3" "golang.org/x/oauth2" + googlegrpc "google.golang.org/grpc" "google.golang.org/protobuf/encoding/protojson" "github.com/pomerium/pomerium/authenticate/events" @@ -20,6 +21,7 @@ import ( "github.com/pomerium/pomerium/internal/httputil" "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/sessions" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/urlutil" "github.com/pomerium/pomerium/pkg/cryptutil" "github.com/pomerium/pomerium/pkg/grpc" @@ -29,6 +31,9 @@ import ( "github.com/pomerium/pomerium/pkg/grpc/user" "github.com/pomerium/pomerium/pkg/hpke" "github.com/pomerium/pomerium/pkg/identity" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "go.opentelemetry.io/otel" + oteltrace "go.opentelemetry.io/otel/trace" ) // Stateless implements the stateless authentication flow. In this flow, the @@ -56,18 +61,21 @@ type Stateless struct { dataBrokerClient databroker.DataBrokerServiceClient - getIdentityProvider func(options *config.Options, idpID string) (identity.Authenticator, error) + getIdentityProvider func(ctx context.Context, tracerProvider oteltrace.TracerProvider, options *config.Options, idpID string) (identity.Authenticator, error) profileTrimFn func(*identitypb.Profile) authEventFn events.AuthEventFn + + tracerProvider oteltrace.TracerProvider } // NewStateless initializes the authentication flow for the given // configuration, session store, and additional options. func NewStateless( ctx context.Context, + tracerProvider oteltrace.TracerProvider, cfg *config.Config, sessionStore sessions.SessionStore, - getIdentityProvider func(options *config.Options, idpID string) (identity.Authenticator, error), + getIdentityProvider func(ctx context.Context, tracerProvider oteltrace.TracerProvider, options *config.Options, idpID string) (identity.Authenticator, error), profileTrimFn func(*identitypb.Profile), authEventFn events.AuthEventFn, ) (*Stateless, error) { @@ -77,6 +85,7 @@ func NewStateless( getIdentityProvider: getIdentityProvider, profileTrimFn: profileTrimFn, authEventFn: authEventFn, + tracerProvider: tracerProvider, } var err error @@ -137,7 +146,10 @@ func NewStateless( InstallationID: cfg.Options.InstallationID, ServiceName: cfg.Options.Services, SignedJWTKey: sharedKey, - }) + }, googlegrpc.WithStatsHandler(trace.NewClientStatsHandler( + otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(tracerProvider)), + outboundDatabrokerTraceClientOpts..., + ))) if err != nil { return nil, err } @@ -154,7 +166,7 @@ func (s *Stateless) VerifySession(ctx context.Context, r *http.Request, _ *sessi return fmt.Errorf("identity profile load error: %w", err) } - authenticator, err := s.getIdentityProvider(s.options, profile.GetProviderId()) + authenticator, err := s.getIdentityProvider(ctx, s.tracerProvider, s.options, profile.GetProviderId()) if err != nil { return fmt.Errorf("couldn't get identity provider: %w", err) } @@ -355,6 +367,7 @@ func (s *Stateless) AuthenticateSignInURL( for k, v := range queryParams { q[k] = v } + otel.GetTextMapPropagator().Inject(ctx, trace.PomeriumURLQueryCarrier(q)) authenticateURLWithParams.RawQuery = q.Encode() return urlutil.SignInURL( diff --git a/internal/benchmarks/latency_bench_test.go b/internal/benchmarks/latency_bench_test.go index e3de8d94c..d25185942 100644 --- a/internal/benchmarks/latency_bench_test.go +++ b/internal/benchmarks/latency_bench_test.go @@ -48,6 +48,7 @@ func TestRequestLatency(t *testing.T) { for i := range numRoutes { routes[i] = up.Route(). From(env.SubdomainURL(fmt.Sprintf("from-%d", i))). + // Policy(func(p *config.Policy) { p.AllowPublicUnauthenticatedAccess = true }) PPL(fmt.Sprintf(`{"allow":{"and":["email":{"is":"user%d@example.com"}]}}`, i)) } env.AddUpstream(up) diff --git a/internal/controlplane/events.go b/internal/controlplane/events.go index 90bc086d2..dabbc8cc5 100644 --- a/internal/controlplane/events.go +++ b/internal/controlplane/events.go @@ -7,6 +7,8 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/google/uuid" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + googlegrpc "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" @@ -76,7 +78,7 @@ func (srv *Server) getDataBrokerClient(ctx context.Context) (databrokerpb.DataBr InstallationID: cfg.Options.InstallationID, ServiceName: cfg.Options.Services, SignedJWTKey: sharedKey, - }) + }, googlegrpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(srv.tracerProvider)))) if err != nil { return nil, fmt.Errorf("controlplane: error creating databroker connection: %w", err) } diff --git a/internal/controlplane/grpc_accesslog.go b/internal/controlplane/grpc_accesslog.go index b3f078cfc..ae5f11960 100644 --- a/internal/controlplane/grpc_accesslog.go +++ b/internal/controlplane/grpc_accesslog.go @@ -7,6 +7,8 @@ import ( envoy_data_accesslog_v3 "github.com/envoyproxy/go-control-plane/envoy/data/accesslog/v3" envoy_service_accesslog_v3 "github.com/envoyproxy/go-control-plane/envoy/service/accesslog/v3" "github.com/rs/zerolog" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/pomerium/pomerium/internal/log" ) @@ -21,7 +23,11 @@ func (srv *Server) StreamAccessLogs(stream envoy_service_accesslog_v3.AccessLogS for { msg, err := stream.Recv() if err != nil { - log.Ctx(stream.Context()).Error().Err(err).Msg("access log stream error, disconnecting") + if status.Code(err) == codes.Canceled { + log.Ctx(stream.Context()).Debug().Err(err).Msg("access log stream canceled") + } else { + log.Ctx(stream.Context()).Error().Err(err).Msg("access log stream error, disconnecting") + } return err } diff --git a/internal/controlplane/http.go b/internal/controlplane/http.go index d648fefbf..8f01e5881 100644 --- a/internal/controlplane/http.go +++ b/internal/controlplane/http.go @@ -2,6 +2,7 @@ package controlplane import ( + "context" "fmt" "net/http" "time" @@ -9,18 +10,21 @@ import ( "github.com/CAFxX/httpcompression" "github.com/gorilla/mux" "github.com/rs/zerolog" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "github.com/pomerium/pomerium/config" "github.com/pomerium/pomerium/internal/handlers" "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/middleware" "github.com/pomerium/pomerium/internal/telemetry" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/urlutil" hpke_handlers "github.com/pomerium/pomerium/pkg/hpke/handlers" "github.com/pomerium/pomerium/pkg/telemetry/requestid" ) -func (srv *Server) addHTTPMiddleware(root *mux.Router, logger *zerolog.Logger, _ *config.Config) { +func (srv *Server) addHTTPMiddleware(ctx context.Context, root *mux.Router, _ *config.Config) { + logger := log.Ctx(ctx) compressor, err := httpcompression.DefaultAdapter() if err != nil { panic(err) @@ -69,9 +73,11 @@ func (srv *Server) mountCommonEndpoints(root *mux.Router, cfg *config.Config) er root.HandleFunc("/healthz", handlers.HealthCheck) root.HandleFunc("/ping", handlers.HealthCheck) - root.Handle("/.well-known/pomerium", handlers.WellKnownPomerium(authenticateURL)) - root.Handle("/.well-known/pomerium/", handlers.WellKnownPomerium(authenticateURL)) - root.Path("/.well-known/pomerium/jwks.json").Methods(http.MethodGet).Handler(handlers.JWKSHandler(signingKey)) - root.Path(urlutil.HPKEPublicKeyPath).Methods(http.MethodGet).Handler(hpke_handlers.HPKEPublicKeyHandler(hpkePublicKey)) + + traceHandler := trace.NewHTTPMiddleware(otelhttp.WithTracerProvider(srv.tracerProvider)) + root.Handle("/.well-known/pomerium", traceHandler(handlers.WellKnownPomerium(authenticateURL))) + root.Handle("/.well-known/pomerium/", traceHandler(handlers.WellKnownPomerium(authenticateURL))) + root.Path("/.well-known/pomerium/jwks.json").Methods(http.MethodGet).Handler(traceHandler(handlers.JWKSHandler(signingKey))) + root.Path(urlutil.HPKEPublicKeyPath).Methods(http.MethodGet).Handler(traceHandler(hpke_handlers.HPKEPublicKeyHandler(hpkePublicKey))) return nil } diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go index 587d19318..96bf1f3d4 100644 --- a/internal/controlplane/server.go +++ b/internal/controlplane/server.go @@ -11,6 +11,8 @@ import ( envoy_service_discovery_v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" "github.com/gorilla/mux" "github.com/rs/zerolog" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/health/grpc_health_v1" @@ -25,7 +27,6 @@ import ( "github.com/pomerium/pomerium/internal/events" "github.com/pomerium/pomerium/internal/httputil/reproxy" "github.com/pomerium/pomerium/internal/log" - "github.com/pomerium/pomerium/internal/telemetry" "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/urlutil" "github.com/pomerium/pomerium/internal/version" @@ -34,6 +35,7 @@ import ( "github.com/pomerium/pomerium/pkg/grpcutil" "github.com/pomerium/pomerium/pkg/httputil" "github.com/pomerium/pomerium/pkg/telemetry/requestid" + oteltrace "go.opentelemetry.io/otel/trace" ) // A Service can be mounted on the control plane. @@ -43,6 +45,7 @@ type Service interface { // A Server is the control-plane gRPC and HTTP servers. type Server struct { + coltracepb.UnimplementedTraceServiceServer GRPCListener net.Listener GRPCServer *grpc.Server HTTPListener net.Listener @@ -66,6 +69,9 @@ type Server struct { proxySvc Service haveSetCapacity map[string]bool + + tracerProvider oteltrace.TracerProvider + tracer oteltrace.Tracer } // NewServer creates a new Server. Listener ports are chosen by the OS. @@ -76,7 +82,10 @@ func NewServer( eventsMgr *events.Manager, fileMgr *filemgr.Manager, ) (*Server, error) { + tracerProvider := trace.NewTracerProvider(ctx, "Control Plane") srv := &Server{ + tracerProvider: tracerProvider, + tracer: tracerProvider.Tracer(trace.PomeriumCoreTracer), metricsMgr: metricsMgr, EventsMgr: eventsMgr, filemgr: fileMgr, @@ -105,7 +114,7 @@ func NewServer( ), ) srv.GRPCServer = grpc.NewServer( - grpc.StatsHandler(telemetry.NewGRPCServerStatsHandler(cfg.Options.Services)), + grpc.StatsHandler(otelgrpc.NewServerHandler(otelgrpc.WithTracerProvider(tracerProvider))), grpc.ChainUnaryInterceptor( log.UnaryServerInterceptor(log.Ctx(ctx)), requestid.UnaryServerInterceptor(), @@ -177,7 +186,9 @@ func NewServer( srv.xdsmgr = xdsmgr.NewManager(res) envoy_service_discovery_v3.RegisterAggregatedDiscoveryServiceServer(srv.GRPCServer, srv.xdsmgr) - + if exp := trace.ExporterServerFromContext(ctx); exp != nil { + coltracepb.RegisterTraceServiceServer(srv.GRPCServer, exp) + } return srv, nil } @@ -241,7 +252,7 @@ func (srv *Server) Run(ctx context.Context) error { // OnConfigChange updates the pomerium config options. func (srv *Server) OnConfigChange(ctx context.Context, cfg *config.Config) error { - ctx, span := trace.StartSpan(ctx, "controlplane.Server.OnConfigChange") + ctx, span := srv.tracer.Start(ctx, "controlplane.Server.OnConfigChange") defer span.End() select { @@ -265,7 +276,7 @@ func (srv *Server) EnableProxy(ctx context.Context, svc Service) error { } func (srv *Server) update(ctx context.Context, cfg *config.Config) error { - ctx, span := trace.StartSpan(ctx, "controlplane.Server.update") + ctx, span := srv.tracer.Start(ctx, "controlplane.Server.update") defer span.End() if err := srv.updateRouter(ctx, cfg); err != nil { @@ -283,7 +294,7 @@ func (srv *Server) update(ctx context.Context, cfg *config.Config) error { func (srv *Server) updateRouter(ctx context.Context, cfg *config.Config) error { httpRouter := mux.NewRouter() - srv.addHTTPMiddleware(httpRouter, log.Ctx(ctx), cfg) + srv.addHTTPMiddleware(ctx, httpRouter, cfg) if err := srv.mountCommonEndpoints(httpRouter, cfg); err != nil { return err } diff --git a/internal/controlplane/xds.go b/internal/controlplane/xds.go index 6e8343526..aa47ab6ac 100644 --- a/internal/controlplane/xds.go +++ b/internal/controlplane/xds.go @@ -9,7 +9,6 @@ import ( "golang.org/x/sync/errgroup" "github.com/pomerium/pomerium/internal/log" - "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/pkg/cryptutil" "github.com/pomerium/pomerium/pkg/protoutil" ) @@ -21,7 +20,7 @@ const ( ) func (srv *Server) buildDiscoveryResources(ctx context.Context) (map[string][]*envoy_service_discovery_v3.Resource, error) { - ctx, span := trace.StartSpan(ctx, "controlplane.Server.buildDiscoveryResources") + ctx, span := srv.tracer.Start(ctx, "controlplane.Server.buildDiscoveryResources") defer span.End() cfg := srv.currentConfig.Load() diff --git a/internal/databroker/config_source.go b/internal/databroker/config_source.go index b76986ab3..403bcf481 100644 --- a/internal/databroker/config_source.go +++ b/internal/databroker/config_source.go @@ -22,6 +22,9 @@ import ( "github.com/pomerium/pomerium/pkg/grpc/databroker" "github.com/pomerium/pomerium/pkg/grpcutil" "github.com/pomerium/pomerium/pkg/health" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + oteltrace "go.opentelemetry.io/otel/trace" + googlegrpc "google.golang.org/grpc" ) // ConfigSource provides a new Config source that decorates an underlying config with @@ -35,6 +38,7 @@ type ConfigSource struct { updaterHash uint64 cancel func() enableValidation bool + tracerProvider oteltrace.TracerProvider config.ChangeDispatcher } @@ -50,11 +54,13 @@ type EnableConfigValidation bool // NewConfigSource creates a new ConfigSource. func NewConfigSource( ctx context.Context, + tracerProvider oteltrace.TracerProvider, underlying config.Source, enableValidation EnableConfigValidation, listeners ...config.ChangeListener, ) *ConfigSource { src := &ConfigSource{ + tracerProvider: tracerProvider, enableValidation: bool(enableValidation), dbConfigs: map[string]dbConfig{}, outboundGRPCConnection: new(grpc.CachedOutboundGRPClientConn), @@ -85,7 +91,7 @@ func (src *ConfigSource) GetConfig() *config.Config { type firstTime bool func (src *ConfigSource) rebuild(ctx context.Context, firstTime firstTime) { - _, span := trace.StartSpan(ctx, "databroker.config_source.rebuild") + _, span := trace.Continue(ctx, "databroker.config_source.rebuild") defer span.End() now := time.Now() @@ -259,7 +265,8 @@ func (src *ConfigSource) runUpdater(ctx context.Context, cfg *config.Config) { ctx, src.cancel = context.WithCancel(ctx) - cc, err := src.outboundGRPCConnection.Get(ctx, connectionOptions) + cc, err := src.outboundGRPCConnection.Get(ctx, connectionOptions, + googlegrpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(src.tracerProvider)))) if err != nil { log.Ctx(ctx).Error().Err(err).Msg("databroker: failed to create gRPC connection to data broker") return diff --git a/internal/databroker/config_source_test.go b/internal/databroker/config_source_test.go index 4116b5b1e..bafa9d24a 100644 --- a/internal/databroker/config_source_test.go +++ b/internal/databroker/config_source_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" "google.golang.org/protobuf/proto" @@ -41,7 +42,7 @@ func TestConfigSource(t *testing.T) { defer func() { _ = li.Close() }() _, outboundPort, _ := net.SplitHostPort(li.Addr().String()) - dataBrokerServer := New(ctx) + dataBrokerServer := New(ctx, trace.NewNoopTracerProvider()) srv := grpc.NewServer() databroker.RegisterDataBrokerServiceServer(srv, dataBrokerServer) go func() { _ = srv.Serve(li) }() @@ -65,7 +66,7 @@ func TestConfigSource(t *testing.T) { OutboundPort: outboundPort, Options: base, }) - src := NewConfigSource(ctx, baseSource, EnableConfigValidation(true), func(_ context.Context, cfg *config.Config) { + src := NewConfigSource(ctx, trace.NewNoopTracerProvider(), baseSource, EnableConfigValidation(true), func(_ context.Context, cfg *config.Config) { cfgs <- cfg }) cfgs <- src.GetConfig() diff --git a/internal/databroker/registry.go b/internal/databroker/registry.go index fee359e4a..605b7bf4c 100644 --- a/internal/databroker/registry.go +++ b/internal/databroker/registry.go @@ -9,7 +9,6 @@ import ( "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/registry" "github.com/pomerium/pomerium/internal/registry/inmemory" - "github.com/pomerium/pomerium/internal/telemetry/trace" registrypb "github.com/pomerium/pomerium/pkg/grpc/registry" "github.com/pomerium/pomerium/pkg/storage" ) @@ -25,7 +24,7 @@ func (stream registryWatchServer) Context() context.Context { // Report calls the registry Report method. func (srv *Server) Report(ctx context.Context, req *registrypb.RegisterRequest) (*registrypb.RegisterResponse, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.Report") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Report") defer span.End() r, err := srv.getRegistry(ctx) @@ -38,7 +37,7 @@ func (srv *Server) Report(ctx context.Context, req *registrypb.RegisterRequest) // List calls the registry List method. func (srv *Server) List(ctx context.Context, req *registrypb.ListRequest) (*registrypb.ServiceList, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.List") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.List") defer span.End() r, err := srv.getRegistry(ctx) @@ -52,7 +51,7 @@ func (srv *Server) List(ctx context.Context, req *registrypb.ListRequest) (*regi // Watch calls the registry Watch method. func (srv *Server) Watch(req *registrypb.ListRequest, stream registrypb.Registry_WatchServer) error { ctx := stream.Context() - ctx, span := trace.StartSpan(ctx, "databroker.grpc.Watch") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Watch") defer span.End() r, err := srv.getRegistry(ctx) diff --git a/internal/databroker/server.go b/internal/databroker/server.go index 4a6f207b3..2671bf0cc 100644 --- a/internal/databroker/server.go +++ b/internal/databroker/server.go @@ -22,22 +22,28 @@ import ( "github.com/pomerium/pomerium/pkg/storage" "github.com/pomerium/pomerium/pkg/storage/inmemory" "github.com/pomerium/pomerium/pkg/storage/postgres" + oteltrace "go.opentelemetry.io/otel/trace" ) // Server implements the databroker service using an in memory database. type Server struct { cfg *serverConfig - mu sync.RWMutex - backend storage.Backend - backendCtx context.Context - registry registry.Interface + mu sync.RWMutex + backend storage.Backend + backendCtx context.Context + registry registry.Interface + tracerProvider oteltrace.TracerProvider + tracer oteltrace.Tracer } // New creates a new server. -func New(ctx context.Context, options ...ServerOption) *Server { +func New(ctx context.Context, tracerProvider oteltrace.TracerProvider, options ...ServerOption) *Server { + tracer := tracerProvider.Tracer(trace.PomeriumCoreTracer) srv := &Server{ - backendCtx: ctx, + backendCtx: ctx, + tracerProvider: tracerProvider, + tracer: tracer, } srv.UpdateConfig(ctx, options...) return srv @@ -74,7 +80,7 @@ func (srv *Server) UpdateConfig(ctx context.Context, options ...ServerOption) { // AcquireLease acquires a lease. func (srv *Server) AcquireLease(ctx context.Context, req *databroker.AcquireLeaseRequest) (*databroker.AcquireLeaseResponse, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.AcquireLease") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.AcquireLease") defer span.End() log.Ctx(ctx).Debug(). Str("name", req.GetName()). @@ -101,7 +107,7 @@ func (srv *Server) AcquireLease(ctx context.Context, req *databroker.AcquireLeas // Get gets a record from the in-memory list. func (srv *Server) Get(ctx context.Context, req *databroker.GetRequest) (*databroker.GetResponse, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.Get") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Get") defer span.End() log.Ctx(ctx).Debug(). Str("type", req.GetType()). @@ -128,7 +134,7 @@ func (srv *Server) Get(ctx context.Context, req *databroker.GetRequest) (*databr // ListTypes lists all the record types. func (srv *Server) ListTypes(ctx context.Context, _ *emptypb.Empty) (*databroker.ListTypesResponse, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.ListTypes") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.ListTypes") defer span.End() log.Ctx(ctx).Debug().Msg("list types") @@ -145,7 +151,7 @@ func (srv *Server) ListTypes(ctx context.Context, _ *emptypb.Empty) (*databroker // Query queries for records. func (srv *Server) Query(ctx context.Context, req *databroker.QueryRequest) (*databroker.QueryResponse, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.Query") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Query") defer span.End() log.Ctx(ctx).Debug(). Str("type", req.GetType()). @@ -198,7 +204,7 @@ func (srv *Server) Query(ctx context.Context, req *databroker.QueryRequest) (*da // Put updates an existing record or adds a new one. func (srv *Server) Put(ctx context.Context, req *databroker.PutRequest) (*databroker.PutResponse, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.Put") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Put") defer span.End() records := req.GetRecords() @@ -237,7 +243,7 @@ func (srv *Server) Put(ctx context.Context, req *databroker.PutRequest) (*databr // Patch updates specific fields of an existing record. func (srv *Server) Patch(ctx context.Context, req *databroker.PatchRequest) (*databroker.PatchResponse, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.Patch") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Patch") defer span.End() records := req.GetRecords() @@ -276,9 +282,9 @@ func (srv *Server) Patch(ctx context.Context, req *databroker.PatchRequest) (*da // ReleaseLease releases a lease. func (srv *Server) ReleaseLease(ctx context.Context, req *databroker.ReleaseLeaseRequest) (*emptypb.Empty, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.ReleaseLease") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.ReleaseLease") defer span.End() - log.Ctx(ctx).Debug(). + log.Ctx(ctx).Trace(). Str("name", req.GetName()). Str("id", req.GetId()). Msg("release lease") @@ -298,9 +304,9 @@ func (srv *Server) ReleaseLease(ctx context.Context, req *databroker.ReleaseLeas // RenewLease releases a lease. func (srv *Server) RenewLease(ctx context.Context, req *databroker.RenewLeaseRequest) (*emptypb.Empty, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.RenewLease") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.RenewLease") defer span.End() - log.Ctx(ctx).Debug(). + log.Ctx(ctx).Trace(). Str("name", req.GetName()). Str("id", req.GetId()). Dur("duration", req.GetDuration().AsDuration()). @@ -323,7 +329,7 @@ func (srv *Server) RenewLease(ctx context.Context, req *databroker.RenewLeaseReq // SetOptions sets options for a type in the databroker. func (srv *Server) SetOptions(ctx context.Context, req *databroker.SetOptionsRequest) (*databroker.SetOptionsResponse, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.SetOptions") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.SetOptions") defer span.End() backend, err := srv.getBackend(ctx) @@ -346,7 +352,7 @@ func (srv *Server) SetOptions(ctx context.Context, req *databroker.SetOptionsReq // Sync streams updates for the given record type. func (srv *Server) Sync(req *databroker.SyncRequest, stream databroker.DataBrokerService_SyncServer) error { ctx := stream.Context() - ctx, span := trace.StartSpan(ctx, "databroker.grpc.Sync") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Sync") defer span.End() ctx, cancel := context.WithCancel(ctx) @@ -384,7 +390,7 @@ func (srv *Server) Sync(req *databroker.SyncRequest, stream databroker.DataBroke // SyncLatest returns the latest value of every record in the databroker as a stream of records. func (srv *Server) SyncLatest(req *databroker.SyncLatestRequest, stream databroker.DataBrokerService_SyncLatestServer) error { ctx := stream.Context() - ctx, span := trace.StartSpan(ctx, "databroker.grpc.SyncLatest") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.SyncLatest") defer span.End() ctx, cancel := context.WithCancel(ctx) diff --git a/internal/databroker/server_test.go b/internal/databroker/server_test.go index aa1c7c5c8..f837e127f 100644 --- a/internal/databroker/server_test.go +++ b/internal/databroker/server_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace/noop" "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -49,10 +50,9 @@ func (h testSyncerHandler) UpdateRecords(ctx context.Context, serverVersion uint } func newServer(cfg *serverConfig) *Server { - return &Server{ - cfg: cfg, - backendCtx: context.Background(), - } + srv := New(context.Background(), noop.NewTracerProvider()) + srv.cfg = cfg + return srv } func TestServer_Get(t *testing.T) { diff --git a/internal/middleware/middleware.go b/internal/middleware/middleware.go index db00f0488..f6a261d4e 100644 --- a/internal/middleware/middleware.go +++ b/internal/middleware/middleware.go @@ -6,7 +6,6 @@ import ( "net/http" "github.com/pomerium/pomerium/internal/httputil" - "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/urlutil" ) @@ -14,12 +13,10 @@ import ( func SetHeaders(headers map[string]string) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ctx, span := trace.StartSpan(r.Context(), "middleware.SetHeaders") - defer span.End() for key, val := range headers { w.Header().Set(key, val) } - next.ServeHTTP(w, r.WithContext(ctx)) + next.ServeHTTP(w, r) }) } } @@ -29,12 +26,10 @@ func SetHeaders(headers map[string]string) func(next http.Handler) http.Handler func ValidateSignature(sharedKey []byte) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return httputil.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { - ctx, span := trace.StartSpan(r.Context(), "middleware.ValidateSignature") - defer span.End() if err := ValidateRequestURL(r, sharedKey); err != nil { return httputil.NewError(http.StatusBadRequest, err) } - next.ServeHTTP(w, r.WithContext(ctx)) + next.ServeHTTP(w, r) return nil }) } diff --git a/internal/registry/reporter.go b/internal/registry/reporter.go index f34b87e91..964287210 100644 --- a/internal/registry/reporter.go +++ b/internal/registry/reporter.go @@ -15,18 +15,23 @@ import ( "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/pkg/grpc" pb "github.com/pomerium/pomerium/pkg/grpc/registry" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + oteltrace "go.opentelemetry.io/otel/trace" + googlegrpc "google.golang.org/grpc" ) // Reporter periodically submits a list of services available on this instance to the service registry type Reporter struct { cancel func() outboundGRPCConnection *grpc.CachedOutboundGRPClientConn + tracerProvider oteltrace.TracerProvider } // NewReporter creates a new Reporter. -func NewReporter() *Reporter { +func NewReporter(tracerProvider oteltrace.TracerProvider) *Reporter { return &Reporter{ outboundGRPCConnection: new(grpc.CachedOutboundGRPClientConn), + tracerProvider: tracerProvider, } } @@ -52,7 +57,7 @@ func (r *Reporter) OnConfigChange(ctx context.Context, cfg *config.Config) { InstallationID: cfg.Options.InstallationID, ServiceName: cfg.Options.Services, SignedJWTKey: sharedKey, - }) + }, googlegrpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(r.tracerProvider)))) if err != nil { log.Ctx(ctx).Error().Err(err).Msg("connecting to registry") return diff --git a/internal/retry/retry.go b/internal/retry/retry.go index ed3d6f8ac..612623191 100644 --- a/internal/retry/retry.go +++ b/internal/retry/retry.go @@ -9,6 +9,7 @@ import ( "time" "github.com/rs/zerolog" + "go.opentelemetry.io/otel/trace" "github.com/pomerium/pomerium/internal/log" ) @@ -47,7 +48,10 @@ restart: backoff: for { interval := backoff.NextBackOff() - log.Ctx(ctx).Info().Msgf("backing off for %s...", interval.String()) + span := trace.SpanFromContext(ctx) + msg := fmt.Sprintf("backing off for %s...", interval.String()) + span.AddEvent(msg) + log.Ctx(ctx).Info().Msg(msg) timer := time.NewTimer(interval) s := makeSelect(ctx, watches, name, timer.C, fn) next, err := s.Exec(ctx) diff --git a/internal/telemetry/grpc.go b/internal/telemetry/grpc.go deleted file mode 100644 index 54863838d..000000000 --- a/internal/telemetry/grpc.go +++ /dev/null @@ -1,100 +0,0 @@ -package telemetry - -import ( - "context" - "strings" - - "go.opencensus.io/plugin/ocgrpc" - "go.opencensus.io/plugin/ochttp/propagation/b3" - "go.opencensus.io/trace" - "go.opencensus.io/trace/propagation" - "google.golang.org/grpc" - "google.golang.org/grpc/metadata" - grpcstats "google.golang.org/grpc/stats" - - "github.com/pomerium/pomerium/internal/telemetry/metrics" -) - -const ( - grpcTraceBinHeader = "grpc-trace-bin" - b3TraceIDHeader = "x-b3-traceid" - b3SpanIDHeader = "x-b3-spanid" -) - -type tagRPCHandler interface { - TagRPC(context.Context, *grpcstats.RPCTagInfo) context.Context -} - -// GRPCServerStatsHandler provides a grpc stats.Handler for metrics and tracing for a pomerium service -type GRPCServerStatsHandler struct { - service string - metricsHandler tagRPCHandler - grpcstats.Handler -} - -// TagRPC implements grpc.stats.Handler and adds metrics and tracing metadata to the context of a given RPC -func (h *GRPCServerStatsHandler) TagRPC(ctx context.Context, tagInfo *grpcstats.RPCTagInfo) context.Context { - // the opencensus trace handler only supports grpc-trace-bin, so we use that code and support b3 too - - md, _ := metadata.FromIncomingContext(ctx) - name := strings.TrimPrefix(tagInfo.FullMethodName, "/") - name = strings.Replace(name, "/", ".", -1) - - var parent trace.SpanContext - hasParent := false - if traceBin := md[grpcTraceBinHeader]; len(traceBin) > 0 { - parent, hasParent = propagation.FromBinary([]byte(traceBin[0])) - } - - if hdr := md[b3TraceIDHeader]; len(hdr) > 0 { - if tid, ok := b3.ParseTraceID(hdr[0]); ok { - parent.TraceID = tid - hasParent = true - } - } - if hdr := md[b3SpanIDHeader]; len(hdr) > 0 { - if sid, ok := b3.ParseSpanID(hdr[0]); ok { - parent.SpanID = sid - hasParent = true - } - } - - if hasParent { - ctx, _ = trace.StartSpanWithRemoteParent(ctx, name, parent, - trace.WithSpanKind(trace.SpanKindServer)) - } else { - ctx, _ = trace.StartSpan(ctx, name, - trace.WithSpanKind(trace.SpanKindServer)) - } - - // ocgrpc's TagRPC must be called to attach the context rpcDataKey correctly - // https://github.com/census-instrumentation/opencensus-go/blob/bf52d9df8bb2d44cad934587ab946794456cf3c8/plugin/ocgrpc/server_stats_handler.go#L45 - metricCtx := h.metricsHandler.TagRPC(h.Handler.TagRPC(ctx, tagInfo), tagInfo) - return metricCtx -} - -// NewGRPCServerStatsHandler creates a new GRPCServerStatsHandler for a pomerium service -func NewGRPCServerStatsHandler(service string) grpcstats.Handler { - return &GRPCServerStatsHandler{ - service: ServiceName(service), - Handler: &ocgrpc.ServerHandler{}, - metricsHandler: metrics.NewGRPCServerMetricsHandler(ServiceName(service)), - } -} - -// GRPCClientStatsHandler provides DialOptions for grpc clients to instrument network calls with -// both metrics and tracing -type GRPCClientStatsHandler struct { - UnaryInterceptor grpc.UnaryClientInterceptor - // TODO: we should have a streaming interceptor too - grpcstats.Handler -} - -// NewGRPCClientStatsHandler returns a new GRPCClientStatsHandler used to create -// telemetry related client DialOptions -func NewGRPCClientStatsHandler(service string) *GRPCClientStatsHandler { - return &GRPCClientStatsHandler{ - Handler: &ocgrpc.ClientHandler{}, - UnaryInterceptor: metrics.GRPCClientInterceptor(ServiceName(service)), - } -} diff --git a/internal/telemetry/grpc_test.go b/internal/telemetry/grpc_test.go deleted file mode 100644 index 038f58ada..000000000 --- a/internal/telemetry/grpc_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package telemetry - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "go.opencensus.io/plugin/ocgrpc" - "go.opencensus.io/plugin/ochttp/propagation/b3" - "go.opencensus.io/trace" - "google.golang.org/grpc/metadata" - grpcstats "google.golang.org/grpc/stats" -) - -type mockTagHandler struct { - called bool -} - -type mockCtxTag string - -func (m *mockTagHandler) TagRPC(ctx context.Context, _ *grpcstats.RPCTagInfo) context.Context { - m.called = true - return context.WithValue(ctx, mockCtxTag("added"), "true") -} - -func Test_GRPCServerStatsHandler(t *testing.T) { - metricsHandler := &mockTagHandler{} - h := &GRPCServerStatsHandler{ - metricsHandler: metricsHandler, - Handler: &ocgrpc.ServerHandler{}, - } - - ctx := context.WithValue(context.Background(), mockCtxTag("original"), "true") - ctx = metadata.NewIncomingContext(ctx, metadata.MD{ - b3TraceIDHeader: {"9de3f6756f315fef"}, - b3SpanIDHeader: {"b4f83d3096b6bf9c"}, - }) - ctx = h.TagRPC(ctx, &grpcstats.RPCTagInfo{}) - - assert.True(t, metricsHandler.called) - assert.Equal(t, ctx.Value(mockCtxTag("added")), "true") - assert.Equal(t, ctx.Value(mockCtxTag("original")), "true") - - span := trace.FromContext(ctx) - expectedTraceID, _ := b3.ParseTraceID("9de3f6756f315fef") - assert.Equal(t, expectedTraceID, span.SpanContext().TraceID) -} diff --git a/internal/telemetry/trace/carriers.go b/internal/telemetry/trace/carriers.go new file mode 100644 index 000000000..ac1a1bf05 --- /dev/null +++ b/internal/telemetry/trace/carriers.go @@ -0,0 +1,28 @@ +package trace + +import ( + "net/url" + + "go.opentelemetry.io/otel/propagation" +) + +type PomeriumURLQueryCarrier url.Values + +// Get implements propagation.TextMapCarrier. +func (q PomeriumURLQueryCarrier) Get(key string) string { + return url.Values(q).Get("pomerium_" + key) +} + +// Set implements propagation.TextMapCarrier. +func (q PomeriumURLQueryCarrier) Set(key string, value string) { + url.Values(q).Set("pomerium_"+key, value) +} + +// Keys implements propagation.TextMapCarrier. +func (q PomeriumURLQueryCarrier) Keys() []string { + // this function is never called in otel, and the way it would be + // implemented in this instance is unclear. + panic("unimplemented") +} + +var _ propagation.TextMapCarrier = PomeriumURLQueryCarrier{} diff --git a/internal/telemetry/trace/carriers_test.go b/internal/telemetry/trace/carriers_test.go new file mode 100644 index 000000000..1e63df4b6 --- /dev/null +++ b/internal/telemetry/trace/carriers_test.go @@ -0,0 +1,30 @@ +package trace_test + +import ( + "net/url" + "testing" + + "github.com/pomerium/pomerium/internal/telemetry/trace" + "github.com/stretchr/testify/assert" +) + +func TestPomeriumURLQueryCarrier(t *testing.T) { + t.Parallel() + values := url.Values{} + carrier := trace.PomeriumURLQueryCarrier(values) + assert.Empty(t, carrier.Get("foo")) + carrier.Set("foo", "bar") + assert.Equal(t, url.Values{ + "pomerium_foo": []string{"bar"}, + }, values) + assert.Equal(t, "bar", carrier.Get("foo")) + carrier.Set("foo", "bar2") + assert.Equal(t, url.Values{ + "pomerium_foo": []string{"bar2"}, + }, values) + assert.Equal(t, "bar2", carrier.Get("foo")) + + assert.Panics(t, func() { + carrier.Keys() + }) +} diff --git a/internal/telemetry/trace/client.go b/internal/telemetry/trace/client.go new file mode 100644 index 000000000..0dcf332f9 --- /dev/null +++ b/internal/telemetry/trace/client.go @@ -0,0 +1,259 @@ +package trace + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "strings" + "sync" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + v1 "go.opentelemetry.io/proto/otlp/trace/v1" +) + +var ( + ErrNoClient = errors.New("no client") + ErrClientStopped = errors.New("client is stopped") +) + +type SyncClient interface { + otlptrace.Client + + Update(ctx context.Context, newClient otlptrace.Client) error +} + +func NewSyncClient(client otlptrace.Client) SyncClient { + return &syncClient{ + client: client, + } +} + +type syncClient struct { + mu sync.Mutex + client otlptrace.Client + waitForNewClient chan struct{} +} + +var _ SyncClient = (*syncClient)(nil) + +// Start implements otlptrace.Client. +func (ac *syncClient) Start(ctx context.Context) error { + ac.mu.Lock() + defer ac.mu.Unlock() + if ac.waitForNewClient != nil { + panic("bug: Start called during Stop or Update") + } + if ac.client == nil { + return ErrNoClient + } + return ac.client.Start(ctx) +} + +// Stop implements otlptrace.Client. +func (ac *syncClient) Stop(ctx context.Context) error { + ac.mu.Lock() + defer ac.mu.Unlock() + if ac.waitForNewClient != nil { + panic("bug: Stop called concurrently") + } + return ac.resetLocked(ctx, nil) +} + +func (ac *syncClient) resetLocked(ctx context.Context, newClient otlptrace.Client) error { + if ac.client == nil { + return ErrNoClient + } + ac.waitForNewClient = make(chan struct{}) + ac.mu.Unlock() + + err := ac.client.Stop(ctx) + + ac.mu.Lock() + close(ac.waitForNewClient) + ac.waitForNewClient = nil + ac.client = newClient + return err +} + +// UploadTraces implements otlptrace.Client. +func (ac *syncClient) UploadTraces(ctx context.Context, protoSpans []*v1.ResourceSpans) error { + ac.mu.Lock() + if ac.waitForNewClient != nil { + wait := ac.waitForNewClient + ac.mu.Unlock() + select { + case <-wait: + ac.mu.Lock() + case <-ctx.Done(): + return context.Cause(ctx) + } + } else if ac.client == nil { + ac.mu.Unlock() + return ErrNoClient + } + client := ac.client + ac.mu.Unlock() + if client == nil { + return ErrClientStopped + } + return client.UploadTraces(ctx, protoSpans) +} + +func (ac *syncClient) Update(ctx context.Context, newClient otlptrace.Client) error { + if newClient != nil { + if err := newClient.Start(ctx); err != nil { + return fmt.Errorf("error starting new client: %w", err) + } + } + ac.mu.Lock() + defer ac.mu.Unlock() + if ac.waitForNewClient != nil { + panic("bug: Update called during Stop") + } + if newClient == ac.client { + return nil + } + return ac.resetLocked(ctx, newClient) +} + +// NewRemoteClientFromEnv creates an otlp trace client using the well-known +// environment variables defined in the [OpenTelemetry documentation]. +// +// [OpenTelemetry documentation]: https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/ +func NewRemoteClientFromEnv() otlptrace.Client { + if os.Getenv("OTEL_SDK_DISABLED") == "true" { + return NoopClient{} + } + + exporter, ok := os.LookupEnv("OTEL_TRACES_EXPORTER") + if !ok { + exporter = "none" + } + + switch strings.ToLower(strings.TrimSpace(exporter)) { + case "none", "noop", "": + return NoopClient{} + case "otlp": + var protocol string + if v, ok := os.LookupEnv("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"); ok { + protocol = v + } else if v, ok := os.LookupEnv("OTEL_EXPORTER_OTLP_PROTOCOL"); ok { + protocol = v + } else { + // try to guess the expected protocol from the port number + var endpoint string + var specific bool + if v, ok := os.LookupEnv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"); ok { + endpoint = v + specific = true + } else if v, ok := os.LookupEnv("OTEL_EXPORTER_OTLP_ENDPOINT"); ok { + endpoint = v + } + protocol = BestEffortProtocolFromOTLPEndpoint(endpoint, specific) + } + switch strings.ToLower(strings.TrimSpace(protocol)) { + case "grpc": + return otlptracegrpc.NewClient() + case "http/protobuf", "": + return otlptracehttp.NewClient() + default: + otel.Handle(fmt.Errorf(`unknown otlp trace exporter protocol %q, expected "grpc" or "http/protobuf"`, protocol)) + return NoopClient{} + } + default: + otel.Handle(fmt.Errorf(`unknown otlp trace exporter %q, expected "otlp" or "none"`, exporter)) + return NoopClient{} + } +} + +func BestEffortProtocolFromOTLPEndpoint(endpoint string, specificEnv bool) string { + if endpoint == "" { + return "" + } + u, err := url.Parse(endpoint) + if err != nil { + return "" + } + switch u.Port() { + case "4318": + return "http/protobuf" + case "4317": + return "grpc" + default: + // For http, if the signal-specific form of the endpoint env variable + // (e.g. $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT) is used, the /v1/ + // ^^^^^^ + // path must be present. Otherwise, the path must _not_ be present, + // because the sdk will add it. + // This doesn't apply to grpc endpoints, so assume grpc if there is a + // conflict here. + hasPath := len(strings.Trim(u.Path, "/")) > 0 + switch { + case hasPath && specificEnv: + return "http/protobuf" + case !hasPath && specificEnv: + return "grpc" + case hasPath && !specificEnv: + // would be invalid for http, so assume it's grpc on a subpath + return "grpc" + case !hasPath && !specificEnv: + // could be either, but default to http + return "http/protobuf" + } + panic("unreachable") + } +} + +type NoopClient struct{} + +// Start implements otlptrace.Client. +func (n NoopClient) Start(context.Context) error { + return nil +} + +// Stop implements otlptrace.Client. +func (n NoopClient) Stop(context.Context) error { + return nil +} + +// UploadTraces implements otlptrace.Client. +func (n NoopClient) UploadTraces(context.Context, []*v1.ResourceSpans) error { + return nil +} + +func IsDisabledViaEnvironment() bool { + if os.Getenv("OTEL_SDK_DISABLED") == "true" { + return true + } + exporter, ok := os.LookupEnv("OTEL_TRACES_EXPORTER") + if !ok { + return false + } + switch strings.ToLower(strings.TrimSpace(exporter)) { + case "none, noop": + return true + default: + return false + } +} + +func IsEnabledViaEnvironment() bool { + if os.Getenv("OTEL_SDK_DISABLED") == "true" { + return false + } + exporter, ok := os.LookupEnv("OTEL_TRACES_EXPORTER") + if !ok { + return false + } + switch strings.ToLower(strings.TrimSpace(exporter)) { + case "none, noop", "": + return false + default: + return true + } +} diff --git a/internal/telemetry/trace/client_test.go b/internal/telemetry/trace/client_test.go new file mode 100644 index 000000000..4ae462553 --- /dev/null +++ b/internal/telemetry/trace/client_test.go @@ -0,0 +1,454 @@ +package trace_test + +import ( + "context" + "fmt" + "runtime" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/pomerium/pomerium/internal/log" + "github.com/pomerium/pomerium/internal/telemetry/trace" + "github.com/pomerium/pomerium/internal/testenv" + "github.com/pomerium/pomerium/internal/testenv/scenarios" + "github.com/pomerium/pomerium/internal/testenv/snippets" + . "github.com/pomerium/pomerium/internal/testutil/tracetest" //nolint:revive + "github.com/pomerium/pomerium/internal/testutil/tracetest/mock_otlptrace" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" + "go.uber.org/mock/gomock" +) + +func TestSyncClient(t *testing.T) { + t.Run("No client", func(t *testing.T) { + sc := trace.NewSyncClient(nil) + assert.ErrorIs(t, sc.Start(context.Background()), trace.ErrNoClient) + assert.ErrorIs(t, sc.UploadTraces(context.Background(), nil), trace.ErrNoClient) + assert.ErrorIs(t, sc.Stop(context.Background()), trace.ErrNoClient) + }) + + t.Run("Valid client", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockClient := mock_otlptrace.NewMockClient(ctrl) + start := mockClient.EXPECT(). + Start(gomock.Any()). + Return(nil) + upload := mockClient.EXPECT(). + UploadTraces(gomock.Any(), gomock.Any()). + Return(nil). + After(start) + mockClient.EXPECT(). + Stop(gomock.Any()). + Return(nil). + After(upload) + sc := trace.NewSyncClient(mockClient) + assert.NoError(t, sc.Start(context.Background())) + assert.NoError(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{})) + assert.NoError(t, sc.Stop(context.Background())) + }) + t.Run("Update", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockClient1 := mock_otlptrace.NewMockClient(ctrl) + mockClient2 := mock_otlptrace.NewMockClient(ctrl) + + start1 := mockClient1.EXPECT(). + Start(gomock.Any()). + Return(nil) + upload1 := mockClient1.EXPECT(). + UploadTraces(gomock.Any(), gomock.Any()). + Return(nil). + After(start1) + start2 := mockClient2.EXPECT(). + Start(gomock.Any()). + Return(nil). + After(upload1) + stop1 := mockClient1.EXPECT(). + Stop(gomock.Any()). + Return(nil). + After(start2) + upload2 := mockClient2.EXPECT(). + UploadTraces(gomock.Any(), gomock.Any()). + Return(nil). + After(stop1) + mockClient2.EXPECT(). + Stop(gomock.Any()). + Return(nil). + After(upload2) + sc := trace.NewSyncClient(mockClient1) + assert.NoError(t, sc.Start(context.Background())) + assert.NoError(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{})) + assert.NoError(t, sc.Update(context.Background(), mockClient2)) + assert.NoError(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{})) + assert.NoError(t, sc.Stop(context.Background())) + }) + + spinWait := func(counter *atomic.Int32, until int32) error { + startTime := time.Now() + for counter.Load() != until { + if time.Since(startTime) > 1*time.Second { + return fmt.Errorf("timed out waiting for counter to equal %d", until) + } + } + return nil + } + + t.Run("Concurrent UploadTraces", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockClient1 := mock_otlptrace.NewMockClient(ctrl) + count := atomic.Int32{} + unlock := make(chan struct{}) + concurrency := min(runtime.NumCPU(), 4) + mockClient1.EXPECT(). + UploadTraces(gomock.Any(), gomock.Any()). + DoAndReturn(func(context.Context, []*tracev1.ResourceSpans) error { + count.Add(1) + defer count.Add(-1) + <-unlock + return nil + }). + Times(concurrency) + sc := trace.NewSyncClient(mockClient1) + start := make(chan struct{}) + for range concurrency { + go func() { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + <-start + require.NoError(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{})) + }() + } + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + close(start) + assert.NoError(t, spinWait(&count, int32(concurrency))) + }) + + t.Run("Concurrent Update/UploadTraces", func(t *testing.T) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + ctrl := gomock.NewController(t) + mockClient1 := mock_otlptrace.NewMockClient(ctrl) + mockClient2 := mock_otlptrace.NewMockClient(ctrl) + uploadTracesCount1 := atomic.Int32{} + uploadTracesCount2 := atomic.Int32{} + unlock1 := make(chan struct{}) + unlock2 := make(chan struct{}) + waitForStop := make(chan struct{}) + concurrency := min(runtime.NumCPU(), 4) + + // start 1 -> upload 1 -> start 2 -> stop 1 -> upload 2 -> stop 2 + fStart1 := mockClient1.EXPECT(). + Start(gomock.Any()). + Return(nil) + fUpload1 := mockClient1.EXPECT(). + UploadTraces(gomock.Any(), gomock.Any()). + DoAndReturn(func(context.Context, []*tracev1.ResourceSpans) error { + // called from non-test threads + uploadTracesCount1.Add(1) + defer uploadTracesCount1.Add(-1) + <-unlock1 + return nil + }). + Times(concurrency). + After(fStart1) + fStart2 := mockClient2.EXPECT(). + Start(gomock.Any()). + Return(nil). + After(fUpload1) + fStop1 := mockClient1.EXPECT(). + Stop(gomock.Any()). + DoAndReturn(func(context.Context) error { + // called from test thread + close(unlock1) + assert.NoError(t, spinWait(&uploadTracesCount1, 0)) + return nil + }). + After(fStart2) + fUpload2 := mockClient2.EXPECT(). + UploadTraces(gomock.Any(), gomock.Any()). + DoAndReturn(func(context.Context, []*tracev1.ResourceSpans) error { + // called from non-test threads + uploadTracesCount2.Add(1) + defer uploadTracesCount2.Add(-1) + <-unlock2 + return nil + }). + Times(concurrency). + After(fStop1) + mockClient2.EXPECT(). + Stop(gomock.Any()). + DoAndReturn(func(context.Context) error { + // called from test thread + close(unlock2) + assert.NoError(t, spinWait(&uploadTracesCount2, 0)) + close(waitForStop) + // no way around sleeping here - we have to give the other threads time + // to call UploadTraces and block waiting on waitForNewClient to be + // closed, which happens after this function returns + time.Sleep(10 * time.Millisecond) + return nil + }). + After(fUpload2) + sc := trace.NewSyncClient(mockClient1) + require.NoError(t, sc.Start(context.Background())) + + for range concurrency { + go func() { + require.NoError(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{})) + }() + } + require.NoError(t, spinWait(&uploadTracesCount1, int32(concurrency))) + // at this point, all calls to UploadTraces for client1 are blocked + + for range concurrency { + go func() { + <-unlock1 // wait for client1.Stop + // after this, calls to UploadTraces will block waiting for the + // new client, instead of using the old one we're about to close + require.NoError(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{})) + }() + } + require.NoError(t, sc.Update(context.Background(), mockClient2)) + require.NoError(t, spinWait(&uploadTracesCount2, int32(concurrency))) + // at this point, all calls to UploadTraces for client2 are blocked. + + // while SyncClient is waiting for the underlying client to stop during + // sc.Stop(), *new* calls to sc.UploadTraces will wait for it to stop, then + // error with trace.ErrClientStopped, but the previous calls blocked in + // client2 will complete without error. + for range concurrency { + go func() { + <-waitForStop + assert.ErrorIs(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{}), trace.ErrClientStopped) + }() + } + assert.NoError(t, sc.Stop(context.Background())) + + // sanity checks + assert.ErrorIs(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{}), trace.ErrNoClient) + assert.ErrorIs(t, sc.Start(context.Background()), trace.ErrNoClient) + assert.ErrorIs(t, sc.Stop(context.Background()), trace.ErrNoClient) + assert.NoError(t, sc.Update(context.Background(), nil)) + }) +} + +type errHandler struct { + err error +} + +var _ otel.ErrorHandler = (*errHandler)(nil) + +func (h *errHandler) Handle(err error) { + h.err = err +} + +func TestNewRemoteClientFromEnv(t *testing.T) { + env := testenv.New(t, testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags)) + + receiver := scenarios.NewOTLPTraceReceiver() + env.Add(receiver) + + grpcEndpoint := receiver.GRPCEndpointURL() + httpEndpoint := receiver.HTTPEndpointURL() + + env.Start() + snippets.WaitStartupComplete(env) + + for _, tc := range []struct { + name string + env map[string]string + newClientErr string + uploadErr bool + expectNoSpans bool + }{ + { + name: "GRPC endpoint, auto protocol", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": grpcEndpoint.Value(), + }, + }, + { + name: "GRPC endpoint, alternate env, auto protocol", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_ENDPOINT": grpcEndpoint.Value(), + }, + uploadErr: true, + }, + { + name: "HTTP endpoint, auto protocol", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": httpEndpoint.Value(), + }, + }, + { + name: "HTTP endpoint, alternate env, auto protocol", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_ENDPOINT": strings.TrimSuffix(httpEndpoint.Value(), "/v1/traces"), // path is added automatically by the sdk here + }, + }, + { + name: "GRPC endpoint, explicit protocol", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": grpcEndpoint.Value(), + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "grpc", + }, + }, + { + name: "HTTP endpoint, explicit protocol", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": httpEndpoint.Value(), + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "http/protobuf", + }, + }, + { + name: "exporter unset", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": httpEndpoint.Value(), + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "http/protobuf", + }, + expectNoSpans: true, + }, + { + name: "exporter noop", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "noop", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": httpEndpoint.Value(), + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "http/protobuf", + }, + expectNoSpans: true, + }, + { + name: "exporter none", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "none", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": httpEndpoint.Value(), + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "http/protobuf", + }, + expectNoSpans: true, + }, + { + name: "invalid exporter", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "invalid", + }, + newClientErr: `unknown otlp trace exporter "invalid", expected "otlp" or "none"`, + }, + { + name: "invalid protocol", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": grpcEndpoint.Value(), + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "invalid", + }, + newClientErr: `unknown otlp trace exporter protocol "invalid", expected "grpc" or "http/protobuf"`, + }, + { + name: "valid configuration, but sdk disabled", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": grpcEndpoint.Value(), + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "grpc", + "OTEL_SDK_DISABLED": "true", + }, + expectNoSpans: true, + }, + { + name: "valid configuration, wrong value for sdk disabled env", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": grpcEndpoint.Value(), + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "grpc", + "OTEL_SDK_DISABLED": "1", // only "true" works according to the spec + }, + }, + { + name: "endpoint variable precedence", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_ENDPOINT": "invalid", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": grpcEndpoint.Value(), // should take precedence + "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc", + }, + }, + { + name: "protocol variable precedence", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_PROTOCOL": "invalid", + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "grpc", // should take precedence + "OTEL_EXPORTER_OTLP_ENDPOINT": grpcEndpoint.Value(), + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.env { + t.Setenv(k, v) + } + handler := &errHandler{} + oldErrHandler := otel.GetErrorHandler() + otel.SetErrorHandler(handler) + t.Cleanup(func() { otel.SetErrorHandler(oldErrHandler) }) + + remoteClient := trace.NewRemoteClientFromEnv() + ctx := trace.Options{ + RemoteClient: remoteClient, + }.NewContext(log.Ctx(env.Context()).WithContext(context.Background())) + + if tc.newClientErr != "" { + assert.ErrorContains(t, handler.err, tc.newClientErr) + return + } + + tp := trace.NewTracerProvider(ctx, t.Name()) + + _, span := tp.Tracer(trace.PomeriumCoreTracer).Start(ctx, "test span") + span.End() + + if tc.uploadErr { + assert.Error(t, trace.ForceFlush(ctx)) + assert.NoError(t, trace.ShutdownContext(ctx)) + return + } + assert.NoError(t, trace.ShutdownContext(ctx)) + + results := NewTraceResults(receiver.FlushResourceSpans()) + if tc.expectNoSpans { + results.MatchTraces(t, MatchOptions{Exact: true}) + } else { + results.MatchTraces(t, MatchOptions{ + Exact: true, + }, Match{Name: t.Name() + ": test span", TraceCount: 1, Services: []string{t.Name()}}) + } + }) + } +} + +func TestBestEffortProtocolFromOTLPEndpoint(t *testing.T) { + t.Run("Well-known port numbers", func(t *testing.T) { + assert.Equal(t, "grpc", trace.BestEffortProtocolFromOTLPEndpoint("http://127.0.0.1:4317", true)) + assert.Equal(t, "http/protobuf", trace.BestEffortProtocolFromOTLPEndpoint("http://127.0.0.1:4318", true)) + }) + t.Run("path presence", func(t *testing.T) { + assert.Equal(t, "http/protobuf", trace.BestEffortProtocolFromOTLPEndpoint("http://127.0.0.1:12345", false)) + assert.Equal(t, "grpc", trace.BestEffortProtocolFromOTLPEndpoint("http://127.0.0.1:12345", true)) + assert.Equal(t, "grpc", trace.BestEffortProtocolFromOTLPEndpoint("http://127.0.0.1:12345/v1/traces", false)) + assert.Equal(t, "http/protobuf", trace.BestEffortProtocolFromOTLPEndpoint("http://127.0.0.1:12345/v1/traces", true)) + }) + t.Run("invalid inputs", func(t *testing.T) { + assert.Equal(t, "", trace.BestEffortProtocolFromOTLPEndpoint("", false)) + assert.Equal(t, "", trace.BestEffortProtocolFromOTLPEndpoint("http://\x7f", false)) + }) +} diff --git a/internal/telemetry/trace/datadog.go b/internal/telemetry/trace/datadog.go deleted file mode 100644 index 10b346ca1..000000000 --- a/internal/telemetry/trace/datadog.go +++ /dev/null @@ -1,34 +0,0 @@ -package trace - -import ( - datadog "github.com/DataDog/opencensus-go-exporter-datadog" - octrace "go.opencensus.io/trace" -) - -type datadogProvider struct { - exporter *datadog.Exporter -} - -func (provider *datadogProvider) Register(opts *TracingOptions) error { - dOpts := datadog.Options{ - Service: opts.Service, - TraceAddr: opts.DatadogAddress, - } - dex, err := datadog.NewExporter(dOpts) - if err != nil { - return err - } - octrace.RegisterExporter(dex) - provider.exporter = dex - return nil -} - -func (provider *datadogProvider) Unregister() error { - if provider.exporter == nil { - return nil - } - octrace.UnregisterExporter(provider.exporter) - provider.exporter.Stop() - provider.exporter = nil - return nil -} diff --git a/internal/telemetry/trace/debug.go b/internal/telemetry/trace/debug.go new file mode 100644 index 000000000..ab6fb517c --- /dev/null +++ b/internal/telemetry/trace/debug.go @@ -0,0 +1,524 @@ +package trace + +import ( + "context" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "runtime" + "slices" + "strings" + "sync" + "time" + + "go.opentelemetry.io/otel/attribute" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + oteltrace "go.opentelemetry.io/otel/trace" + coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" + "google.golang.org/grpc/metadata" + "google.golang.org/protobuf/encoding/protojson" +) + +type DebugFlags uint32 + +const ( + // If set, adds the "caller" attribute to each trace with the source location + // where the trace was started. + TrackSpanCallers = (1 << iota) + + // If set, keeps track of all span references and will attempt to wait for + // all traces to complete when shutting down a trace context. + // Use with caution, this will cause increasing memory usage over time. + TrackSpanReferences = (1 << iota) + + // If set, keeps track of all observed spans, including span context and + // all attributes. + // Use with caution, this will cause significantly increasing memory usage + // over time. + TrackAllSpans = (1 << iota) | TrackSpanCallers + + // If set, will log all trace IDs and their span counts on close. + // + // Enables [TrackAllSpans] + LogTraceIDs = (1 << iota) | TrackAllSpans + + // If set, will log all spans observed by the exporter on close. These spans + // may belong to incomplete traces. + // + // Enables [TrackAllSpans] + LogAllSpans = (1 << iota) | TrackAllSpans + + // If set, will log all exported spans when a warning is issued on close + // (requires warning flags to also be set) + // + // Enables [TrackAllSpans] + LogAllSpansOnWarn = (1 << iota) | TrackAllSpans + + // If set, will log all trace ID mappings when a warning is issued on close. + // (requires warning flags to also be set) + LogTraceIDsOnWarn = (1 << iota) + + // If set, will print a warning to stderr on close if there are any incomplete + // traces (traces with no observed root spans) + WarnOnIncompleteTraces = (1 << iota) + + // If set, will print a warning to stderr on close if there are any incomplete + // spans (spans started, but not ended) + WarnOnIncompleteSpans = (1 << iota) + + // If set, will print a warning to stderr on close if there are any spans + // which reference unknown parent spans. + // + // Enables [TrackSpanReferences] + WarnOnUnresolvedReferences = (1 << iota) | TrackSpanReferences + + // If set, configures Envoy to flush every span individually, disabling its + // internal buffer. + EnvoyFlushEverySpan = (1 << iota) +) + +func (df DebugFlags) Check(flags DebugFlags) bool { + return (df & flags) == flags +} + +var ( + ErrIncompleteSpans = errors.New("exporter shut down with incomplete spans") + ErrMissingParentSpans = errors.New("exporter shut down with missing parent spans") +) + +// WaitForSpans will block up to the given max duration and wait for all +// in-flight spans from tracers created with the given context to end. This +// function can be called more than once, and is safe to call from multiple +// goroutines in parallel. +// +// This requires the [TrackSpanReferences] debug flag to have been set with +// [Options.NewContext]. Otherwise, this function is a no-op and will return +// immediately. +// +// If this function blocks for more than 10 seconds, it will print a warning +// to stderr containing a list of span IDs it is waiting for, and the IDs of +// their parents (if known). Additionally, if the [TrackAllSpans] debug flag +// is set, details about parent spans will be displayed, including call site +// and trace ID. +func WaitForSpans(ctx context.Context, maxDuration time.Duration) error { + if sys := systemContextFromContext(ctx); sys != nil && sys.observer != nil { + done := make(chan struct{}) + go func() { + defer close(done) + sys.observer.wait(10 * time.Second) + }() + select { + case <-done: + return nil + case <-time.After(maxDuration): + return ErrMissingParentSpans + } + } + return nil +} + +func DebugFlagsFromContext(ctx context.Context) DebugFlags { + if sys := systemContextFromContext(ctx); sys != nil { + return sys.options.DebugFlags + } + return 0 +} + +type stackTraceProcessor struct{} + +// ForceFlush implements trace.SpanProcessor. +func (s *stackTraceProcessor) ForceFlush(context.Context) error { + return nil +} + +// OnEnd implements trace.SpanProcessor. +func (*stackTraceProcessor) OnEnd(sdktrace.ReadOnlySpan) { +} + +// OnStart implements trace.SpanProcessor. +func (*stackTraceProcessor) OnStart(_ context.Context, s sdktrace.ReadWriteSpan) { + _, file, line, _ := runtime.Caller(2) + s.SetAttributes(attribute.String("caller", fmt.Sprintf("%s:%d", file, line))) +} + +// Shutdown implements trace.SpanProcessor. +func (s *stackTraceProcessor) Shutdown(context.Context) error { + return nil +} + +var debugMessageWriter io.Writer + +func startMsg(title string) *strings.Builder { + msg := &strings.Builder{} + msg.WriteString("\n==================================================\n") + msg.WriteString(title) + return msg +} + +func endMsg(msg *strings.Builder) { + msg.WriteString("==================================================\n") + w := debugMessageWriter + if w == nil { + w = os.Stderr + } + fmt.Fprint(w, msg.String()) +} + +type DebugEvent struct { + Timestamp time.Time `json:"timestamp"` + Request *coltracepb.ExportTraceServiceRequest `json:"request"` +} + +func (e DebugEvent) MarshalJSON() ([]byte, error) { + type debugEvent struct { + Timestamp time.Time `json:"timestamp"` + Request json.RawMessage `json:"request"` + } + reqData, _ := protojson.Marshal(e.Request) + return json.Marshal(debugEvent{ + Timestamp: e.Timestamp, + Request: reqData, + }) +} + +func (e *DebugEvent) UnmarshalJSON(b []byte) error { + type debugEvent struct { + Timestamp time.Time `json:"timestamp"` + Request json.RawMessage `json:"request"` + } + var ev debugEvent + if err := json.Unmarshal(b, &ev); err != nil { + return err + } + e.Timestamp = ev.Timestamp + var msg coltracepb.ExportTraceServiceRequest + if err := protojson.Unmarshal(ev.Request, &msg); err != nil { + return err + } + e.Request = &msg + return nil +} + +const shardCount = 64 + +type ( + shardedSet [shardCount]map[oteltrace.SpanID]struct{} + shardedLocks [shardCount]sync.Mutex +) + +func (s *shardedSet) Range(f func(key oteltrace.SpanID)) { + for i := range shardCount { + for k := range s[i] { + f(k) + } + } +} + +func (s *shardedLocks) LockAll() { + for i := range shardCount { + s[i].Lock() + } +} + +func (s *shardedLocks) UnlockAll() { + for i := range shardCount { + s[i].Unlock() + } +} + +type spanTracker struct { + inflightSpansMu shardedLocks + inflightSpans shardedSet + allSpans sync.Map + debugFlags DebugFlags + observer *spanObserver + shutdownOnce sync.Once +} + +func newSpanTracker(observer *spanObserver, debugFlags DebugFlags) *spanTracker { + st := &spanTracker{ + observer: observer, + debugFlags: debugFlags, + } + for i := range len(st.inflightSpans) { + st.inflightSpans[i] = make(map[oteltrace.SpanID]struct{}) + } + return st +} + +type spanInfo struct { + Name string + SpanContext oteltrace.SpanContext + Parent oteltrace.SpanContext + caller string + startTime time.Time +} + +// ForceFlush implements trace.SpanProcessor. +func (t *spanTracker) ForceFlush(context.Context) error { + return nil +} + +// OnEnd implements trace.SpanProcessor. +func (t *spanTracker) OnEnd(s sdktrace.ReadOnlySpan) { + id := s.SpanContext().SpanID() + bucket := binary.BigEndian.Uint64(id[:]) % shardCount + t.inflightSpansMu[bucket].Lock() + defer t.inflightSpansMu[bucket].Unlock() + delete(t.inflightSpans[bucket], id) +} + +// OnStart implements trace.SpanProcessor. +func (t *spanTracker) OnStart(_ context.Context, s sdktrace.ReadWriteSpan) { + id := s.SpanContext().SpanID() + bucket := binary.BigEndian.Uint64(id[:]) % shardCount + t.inflightSpansMu[bucket].Lock() + defer t.inflightSpansMu[bucket].Unlock() + t.inflightSpans[bucket][id] = struct{}{} + + if t.debugFlags.Check(TrackSpanReferences) { + if s.Parent().IsValid() { + t.observer.ObserveReference(s.Parent().SpanID(), id) + } + t.observer.Observe(id) + } + + if t.debugFlags.Check(TrackAllSpans) { + var caller string + for _, attr := range s.Attributes() { + if attr.Key == "caller" { + caller = attr.Value.AsString() + break + } + } + t.allSpans.Store(id, &spanInfo{ + Name: s.Name(), + SpanContext: s.SpanContext(), + Parent: s.Parent(), + caller: caller, + startTime: s.StartTime(), + }) + } +} + +// Shutdown implements trace.SpanProcessor. +func (t *spanTracker) Shutdown(_ context.Context) error { + if t.debugFlags == 0 { + return nil + } + didWarn := false + t.shutdownOnce.Do(func() { + if t.debugFlags.Check(WarnOnUnresolvedReferences) { + var unknownParentIDs []string + for id, via := range t.observer.referencedIDs { + if via.IsValid() { + if t.debugFlags.Check(TrackAllSpans) { + if viaSpan, ok := t.allSpans.Load(via); ok { + unknownParentIDs = append(unknownParentIDs, fmt.Sprintf("%s via %s (%s)", id, via, viaSpan.(*spanInfo).Name)) + } else { + unknownParentIDs = append(unknownParentIDs, fmt.Sprintf("%s via %s", id, via)) + } + } + } + } + if len(unknownParentIDs) > 0 { + didWarn = true + msg := startMsg("WARNING: parent spans referenced but never seen:\n") + for _, str := range unknownParentIDs { + msg.WriteString(str) + msg.WriteString("\n") + } + endMsg(msg) + } + } + if t.debugFlags.Check(WarnOnIncompleteSpans) { + if t.debugFlags.Check(TrackAllSpans) { + incompleteSpans := []*spanInfo{} + t.inflightSpansMu.LockAll() + t.inflightSpans.Range(func(key oteltrace.SpanID) { + if info, ok := t.allSpans.Load(key); ok { + incompleteSpans = append(incompleteSpans, info.(*spanInfo)) + } + }) + t.inflightSpansMu.UnlockAll() + if len(incompleteSpans) > 0 { + didWarn = true + msg := startMsg("WARNING: spans not ended:\n") + longestName := 0 + for _, span := range incompleteSpans { + longestName = max(longestName, len(span.Name)+2) + } + for _, span := range incompleteSpans { + var startedAt string + if span.caller != "" { + startedAt = " | started at: " + span.caller + } + fmt.Fprintf(msg, "%-*s (trace: %s | span: %s | parent: %s%s)\n", longestName, "'"+span.Name+"'", + span.SpanContext.TraceID(), span.SpanContext.SpanID(), span.Parent.SpanID(), startedAt) + } + endMsg(msg) + } + } else { + incompleteSpans := []oteltrace.SpanID{} + t.inflightSpansMu.LockAll() + t.inflightSpans.Range(func(key oteltrace.SpanID) { + incompleteSpans = append(incompleteSpans, key) + }) + t.inflightSpansMu.UnlockAll() + if len(incompleteSpans) > 0 { + didWarn = true + msg := startMsg("WARNING: spans not ended:\n") + for _, span := range incompleteSpans { + fmt.Fprintf(msg, "%s\n", span) + } + msg.WriteString("Note: set TrackAllSpans flag for more info\n") + endMsg(msg) + } + } + } + + if t.debugFlags.Check(LogAllSpans) || (t.debugFlags.Check(LogAllSpansOnWarn) && didWarn) { + allSpans := []*spanInfo{} + t.allSpans.Range(func(_, value any) bool { + allSpans = append(allSpans, value.(*spanInfo)) + return true + }) + slices.SortFunc(allSpans, func(a, b *spanInfo) int { + return a.startTime.Compare(b.startTime) + }) + msg := startMsg("All observed spans:\n") + longestName := 0 + for _, span := range allSpans { + longestName = max(longestName, len(span.Name)+2) + } + for _, span := range allSpans { + var startedAt string + if span.caller != "" { + startedAt = " | started at: " + span.caller + } + fmt.Fprintf(msg, "%-*s (trace: %s | span: %s | parent: %s%s)\n", longestName, "'"+span.Name+"'", + span.SpanContext.TraceID(), span.SpanContext.SpanID(), span.Parent.SpanID(), startedAt) + } + endMsg(msg) + } + + if t.debugFlags.Check(LogTraceIDs) || (didWarn && t.debugFlags.Check(LogTraceIDsOnWarn)) { + msg := startMsg("Known trace ids:\n") + traceIDs := map[oteltrace.TraceID]int{} + t.allSpans.Range(func(_, value any) bool { + v := value.(*spanInfo) + traceIDs[v.SpanContext.TraceID()]++ + return true + }) + for id, n := range traceIDs { + fmt.Fprintf(msg, "%s (%d spans)\n", id.String(), n) + } + endMsg(msg) + } + }) + if didWarn { + return ErrIncompleteSpans + } + return nil +} + +func newSpanObserver() *spanObserver { + return &spanObserver{ + referencedIDs: map[oteltrace.SpanID]oteltrace.SpanID{}, + cond: sync.NewCond(&sync.Mutex{}), + } +} + +type spanObserver struct { + cond *sync.Cond + referencedIDs map[oteltrace.SpanID]oteltrace.SpanID + unobservedIDs int +} + +func (obs *spanObserver) ObserveReference(id oteltrace.SpanID, via oteltrace.SpanID) { + obs.cond.L.Lock() + defer obs.cond.L.Unlock() + if _, referenced := obs.referencedIDs[id]; !referenced { + obs.referencedIDs[id] = via // referenced, but not observed + // It is possible for new unobserved references to come in while waiting, + // but incrementing the counter wouldn't satisfy the condition so we don't + // need to signal the waiters + obs.unobservedIDs++ + } +} + +func (obs *spanObserver) Observe(id oteltrace.SpanID) { + obs.cond.L.Lock() + defer obs.cond.L.Unlock() + if observed, referenced := obs.referencedIDs[id]; !referenced || observed.IsValid() { // NB: subtle condition + obs.referencedIDs[id] = zeroSpanID + if referenced { + obs.unobservedIDs-- + obs.cond.Broadcast() + } + } +} + +func (obs *spanObserver) wait(warnAfter time.Duration) { + done := make(chan struct{}) + defer close(done) + go func() { + select { + case <-done: + return + case <-time.After(warnAfter): + obs.debugWarnWaiting() + } + }() + + obs.cond.L.Lock() + for obs.unobservedIDs > 0 { + obs.cond.Wait() + } + obs.cond.L.Unlock() +} + +func (obs *spanObserver) debugWarnWaiting() { + obs.cond.L.Lock() + msg := startMsg(fmt.Sprintf("Waiting on %d unobserved spans:\n", obs.unobservedIDs)) + for id, via := range obs.referencedIDs { + if via.IsValid() { + fmt.Fprintf(msg, "%s via %s\n", id, via) + } + } + endMsg(msg) + obs.cond.L.Unlock() +} + +func (srv *ExporterServer) observeExport(ctx context.Context, req *coltracepb.ExportTraceServiceRequest) { + isLocal := len(metadata.ValueFromIncomingContext(ctx, localExporterMetadataKey)) != 0 + if isLocal { + return + } + for _, res := range req.ResourceSpans { + for _, scope := range res.ScopeSpans { + for _, span := range scope.Spans { + id, ok := ToSpanID(span.SpanId) + if !ok { + continue + } + srv.observer.Observe(id) + for _, attr := range span.Attributes { + if attr.Key != "pomerium.external-parent-span" { + continue + } + if bytes, err := hex.DecodeString(attr.Value.GetStringValue()); err == nil { + if id, ok := ToSpanID(bytes); ok { + srv.observer.Observe(id) + } + } + break + } + } + } + } +} diff --git a/internal/telemetry/trace/debug_test.go b/internal/telemetry/trace/debug_test.go new file mode 100644 index 000000000..95a1c0ff0 --- /dev/null +++ b/internal/telemetry/trace/debug_test.go @@ -0,0 +1,288 @@ +package trace_test + +import ( + "bytes" + "context" + "fmt" + "runtime" + "sync/atomic" + "testing" + "time" + + "github.com/pomerium/pomerium/internal/telemetry/trace" + . "github.com/pomerium/pomerium/internal/testutil/tracetest" //nolint:revive + "github.com/stretchr/testify/assert" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + oteltrace "go.opentelemetry.io/otel/trace" +) + +func TestSpanObserver(t *testing.T) { + t.Run("observe single reference", func(t *testing.T) { + obs := trace.NewSpanObserver() + assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) + + obs.ObserveReference(Span(1).ID(), Span(2).ID()) + assert.Equal(t, []oteltrace.SpanID{Span(1).ID()}, obs.XUnobservedIDs()) + obs.Observe(Span(1).ID()) + assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) + }) + t.Run("observe multiple references", func(t *testing.T) { + obs := trace.NewSpanObserver() + + obs.ObserveReference(Span(1).ID(), Span(2).ID()) + obs.ObserveReference(Span(1).ID(), Span(3).ID()) + obs.ObserveReference(Span(1).ID(), Span(4).ID()) + assert.Equal(t, []oteltrace.SpanID{Span(1).ID()}, obs.XUnobservedIDs()) + obs.Observe(Span(1).ID()) + assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) + }) + t.Run("observe before reference", func(t *testing.T) { + obs := trace.NewSpanObserver() + + obs.Observe(Span(1).ID()) + assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) + obs.ObserveReference(Span(1).ID(), Span(2).ID()) + assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) + }) + + t.Run("wait", func(t *testing.T) { + obs := trace.NewSpanObserver() + obs.ObserveReference(Span(1).ID(), Span(2).ID()) + obs.Observe(Span(2).ID()) + obs.ObserveReference(Span(3).ID(), Span(4).ID()) + obs.Observe(Span(4).ID()) + obs.ObserveReference(Span(5).ID(), Span(6).ID()) + obs.Observe(Span(6).ID()) + waitOkToExit := atomic.Bool{} + waitExited := atomic.Bool{} + go func() { + defer waitExited.Store(true) + obs.XWait() + assert.True(t, waitOkToExit.Load(), "wait exited early") + }() + + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + obs.Observe(Span(1).ID()) + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + obs.Observe(Span(3).ID()) + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + waitOkToExit.Store(true) + obs.Observe(Span(5).ID()) + assert.Eventually(t, waitExited.Load, 10*time.Millisecond, 1*time.Millisecond) + }) + + t.Run("new references observed during wait", func(t *testing.T) { + obs := trace.NewSpanObserver() + obs.ObserveReference(Span(1).ID(), Span(2).ID()) + obs.Observe(Span(2).ID()) + obs.ObserveReference(Span(3).ID(), Span(4).ID()) + obs.Observe(Span(4).ID()) + obs.ObserveReference(Span(5).ID(), Span(6).ID()) + obs.Observe(Span(6).ID()) + waitOkToExit := atomic.Bool{} + waitExited := atomic.Bool{} + go func() { + defer waitExited.Store(true) + obs.XWait() + assert.True(t, waitOkToExit.Load(), "wait exited early") + }() + + assert.Equal(t, []oteltrace.SpanID{Span(1).ID(), Span(3).ID(), Span(5).ID()}, obs.XUnobservedIDs()) + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + obs.Observe(Span(1).ID()) + assert.Equal(t, []oteltrace.SpanID{Span(3).ID(), Span(5).ID()}, obs.XUnobservedIDs()) + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + obs.Observe(Span(3).ID()) + assert.Equal(t, []oteltrace.SpanID{Span(5).ID()}, obs.XUnobservedIDs()) + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + // observe a new reference + obs.ObserveReference(Span(7).ID(), Span(8).ID()) + obs.Observe(Span(8).ID()) + assert.Equal(t, []oteltrace.SpanID{Span(5).ID(), Span(7).ID()}, obs.XUnobservedIDs()) + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + obs.Observe(Span(5).ID()) + assert.Equal(t, []oteltrace.SpanID{Span(7).ID()}, obs.XUnobservedIDs()) + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + waitOkToExit.Store(true) + obs.Observe(Span(7).ID()) + assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) + assert.Eventually(t, waitExited.Load, 10*time.Millisecond, 1*time.Millisecond) + }) + + t.Run("multiple waiters", func(t *testing.T) { + t.Parallel() + obs := trace.NewSpanObserver() + obs.ObserveReference(Span(1).ID(), Span(2).ID()) + obs.Observe(Span(2).ID()) + + waitersExited := atomic.Int32{} + for range 10 { + go func() { + defer waitersExited.Add(1) + obs.XWait() + }() + } + + assert.Equal(t, []oteltrace.SpanID{Span(1).ID()}, obs.XUnobservedIDs()) + time.Sleep(10 * time.Millisecond) + assert.Equal(t, int32(0), waitersExited.Load()) + + obs.Observe(Span(1).ID()) + + startTime := time.Now() + for waitersExited.Load() != 10 { + if time.Since(startTime) > 1*time.Millisecond { + t.Fatal("timed out") + } + runtime.Gosched() + } + }) +} + +func TestSpanTracker(t *testing.T) { + t.Run("no debug flags", func(t *testing.T) { + t.Parallel() + obs := trace.NewSpanObserver() + tracker := trace.NewSpanTracker(obs, 0) + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracker)) + tracer := tp.Tracer("test") + assert.Equal(t, []oteltrace.SpanID{}, tracker.XInflightSpans()) + _, span1 := tracer.Start(context.Background(), "span 1") + assert.Equal(t, []oteltrace.SpanID{span1.SpanContext().SpanID()}, tracker.XInflightSpans()) + assert.Equal(t, []oteltrace.SpanID{}, obs.XObservedIDs()) + span1.End() + assert.Equal(t, []oteltrace.SpanID{}, tracker.XInflightSpans()) + assert.Equal(t, []oteltrace.SpanID{}, obs.XObservedIDs()) + }) + t.Run("with TrackSpanReferences debug flag", func(t *testing.T) { + t.Parallel() + obs := trace.NewSpanObserver() + tracker := trace.NewSpanTracker(obs, trace.TrackSpanReferences) + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracker)) + tracer := tp.Tracer("test") + assert.Equal(t, []oteltrace.SpanID{}, tracker.XInflightSpans()) + _, span1 := tracer.Start(context.Background(), "span 1") + assert.Equal(t, []oteltrace.SpanID{span1.SpanContext().SpanID()}, tracker.XInflightSpans()) + assert.Equal(t, []oteltrace.SpanID{span1.SpanContext().SpanID()}, obs.XObservedIDs()) + span1.End() + assert.Equal(t, []oteltrace.SpanID{}, tracker.XInflightSpans()) + assert.Equal(t, []oteltrace.SpanID{span1.SpanContext().SpanID()}, obs.XObservedIDs()) + }) +} + +func TestSpanTrackerWarnings(t *testing.T) { + t.Run("WarnOnIncompleteSpans", func(t *testing.T) { + var buf bytes.Buffer + trace.SetDebugMessageWriterForTest(t, &buf) + + obs := trace.NewSpanObserver() + tracker := trace.NewSpanTracker(obs, trace.WarnOnIncompleteSpans) + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracker)) + tracer := tp.Tracer("test") + _, span1 := tracer.Start(context.Background(), "span 1") + + assert.ErrorIs(t, tp.Shutdown(context.Background()), trace.ErrIncompleteSpans) + + assert.Equal(t, fmt.Sprintf(` +================================================== +WARNING: spans not ended: +%s +Note: set TrackAllSpans flag for more info +================================================== +`, span1.SpanContext().SpanID()), buf.String()) + }) + + t.Run("WarnOnIncompleteSpans with TrackAllSpans", func(t *testing.T) { + var buf bytes.Buffer + trace.SetDebugMessageWriterForTest(t, &buf) + + obs := trace.NewSpanObserver() + tracker := trace.NewSpanTracker(obs, trace.WarnOnIncompleteSpans|trace.TrackAllSpans) + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracker)) + tracer := tp.Tracer("test") + _, span1 := tracer.Start(context.Background(), "span 1") + + assert.ErrorIs(t, tp.Shutdown(context.Background()), trace.ErrIncompleteSpans) + + assert.Equal(t, fmt.Sprintf(` +================================================== +WARNING: spans not ended: +'span 1' (trace: %s | span: %s | parent: 0000000000000000) +================================================== +`, span1.SpanContext().TraceID(), span1.SpanContext().SpanID()), buf.String()) + }) + + t.Run("WarnOnIncompleteSpans with TrackAllSpans and stackTraceProcessor", func(t *testing.T) { + var buf bytes.Buffer + trace.SetDebugMessageWriterForTest(t, &buf) + + obs := trace.NewSpanObserver() + tracker := trace.NewSpanTracker(obs, trace.WarnOnIncompleteSpans|trace.TrackAllSpans) + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(&trace.XStackTraceProcessor{}), sdktrace.WithSpanProcessor(tracker)) + tracer := tp.Tracer("test") + _, span1 := tracer.Start(context.Background(), "span 1") + _, file, line, _ := runtime.Caller(0) + line-- + + assert.ErrorIs(t, tp.Shutdown(context.Background()), trace.ErrIncompleteSpans) + + assert.Equal(t, fmt.Sprintf(` +================================================== +WARNING: spans not ended: +'span 1' (trace: %s | span: %s | parent: 0000000000000000 | started at: %s:%d) +================================================== +`, span1.SpanContext().TraceID(), span1.SpanContext().SpanID(), file, line), buf.String()) + }) + + t.Run("LogAllSpansOnWarn", func(t *testing.T) { + var buf bytes.Buffer + trace.SetDebugMessageWriterForTest(t, &buf) + + obs := trace.NewSpanObserver() + tracker := trace.NewSpanTracker(obs, trace.WarnOnIncompleteSpans|trace.TrackAllSpans|trace.LogAllSpansOnWarn) + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(&trace.XStackTraceProcessor{}), sdktrace.WithSpanProcessor(tracker)) + tracer := tp.Tracer("test") + _, span1 := tracer.Start(context.Background(), "span 1") + time.Sleep(10 * time.Millisecond) + span1.End() + time.Sleep(10 * time.Millisecond) + _, span2 := tracer.Start(context.Background(), "span 2") + _, file, line, _ := runtime.Caller(0) + line-- + + tp.Shutdown(context.Background()) + + assert.Equal(t, + fmt.Sprintf(` +================================================== +WARNING: spans not ended: +'span 2' (trace: %[1]s | span: %[2]s | parent: 0000000000000000 | started at: %[3]s:%[4]d) +================================================== + +================================================== +All observed spans: +'span 1' (trace: %[5]s | span: %[6]s | parent: 0000000000000000 | started at: %[3]s:%[7]d) +'span 2' (trace: %[1]s | span: %[2]s | parent: 0000000000000000 | started at: %[3]s:%[4]d) +================================================== +`, + span2.SpanContext().TraceID(), span2.SpanContext().SpanID(), file, line, + span1.SpanContext().TraceID(), span1.SpanContext().SpanID(), line-4, + ), buf.String()) + }) +} diff --git a/internal/telemetry/trace/doc.go b/internal/telemetry/trace/doc.go deleted file mode 100644 index c1da792c4..000000000 --- a/internal/telemetry/trace/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package trace contains support for OpenCensus distributed tracing. -package trace diff --git a/internal/telemetry/trace/global.go b/internal/telemetry/trace/global.go new file mode 100644 index 000000000..16b7a5a4f --- /dev/null +++ b/internal/telemetry/trace/global.go @@ -0,0 +1,46 @@ +package trace + +import ( + "context" + + "go.opentelemetry.io/contrib/propagators/autoprop" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/embedded" +) + +// PomeriumCoreTracer should be used for all tracers created in pomerium core. +const PomeriumCoreTracer = "pomerium.io/core" + +func init() { + otel.SetTextMapPropagator(autoprop.NewTextMapPropagator()) +} + +// UseGlobalPanicTracer sets the global tracer provider to one whose tracers +// panic when starting spans. This can be used to locate errant usages of the +// global tracer, and is enabled automatically in some tests. It is otherwise +// not used by default, since pomerium is used as a library in some places that +// might use the global tracer provider. +func UseGlobalPanicTracer() { + otel.SetTracerProvider(panicTracerProvider{}) +} + +type panicTracerProvider struct { + embedded.TracerProvider +} + +// Tracer implements trace.TracerProvider. +func (w panicTracerProvider) Tracer(string, ...trace.TracerOption) trace.Tracer { + return panicTracer{} +} + +type panicTracer struct { + embedded.Tracer +} + +var _ trace.Tracer = panicTracer{} + +// Start implements trace.Tracer. +func (p panicTracer) Start(context.Context, string, ...trace.SpanStartOption) (context.Context, trace.Span) { + panic("global tracer used") +} diff --git a/internal/telemetry/trace/global_test.go b/internal/telemetry/trace/global_test.go new file mode 100644 index 000000000..852d2c82a --- /dev/null +++ b/internal/telemetry/trace/global_test.go @@ -0,0 +1,22 @@ +package trace_test + +import ( + "context" + "testing" + + "github.com/pomerium/pomerium/internal/telemetry/trace" + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace/noop" +) + +func TestUseGlobalPanicTracer(t *testing.T) { + t.Cleanup(func() { + otel.SetTracerProvider(noop.NewTracerProvider()) + }) + trace.UseGlobalPanicTracer() + tracer := otel.GetTracerProvider().Tracer("test") + assert.Panics(t, func() { + tracer.Start(context.Background(), "span") + }) +} diff --git a/internal/telemetry/trace/jaeger.go b/internal/telemetry/trace/jaeger.go deleted file mode 100644 index 23502ef24..000000000 --- a/internal/telemetry/trace/jaeger.go +++ /dev/null @@ -1,37 +0,0 @@ -package trace - -import ( - "contrib.go.opencensus.io/exporter/jaeger" - octrace "go.opencensus.io/trace" -) - -type jaegerProvider struct { - exporter *jaeger.Exporter -} - -func (provider *jaegerProvider) Register(opts *TracingOptions) error { - jOpts := jaeger.Options{ - ServiceName: opts.Service, - AgentEndpoint: opts.JaegerAgentEndpoint, - } - if opts.JaegerCollectorEndpoint != nil { - jOpts.CollectorEndpoint = opts.JaegerCollectorEndpoint.String() - } - jex, err := jaeger.NewExporter(jOpts) - if err != nil { - return err - } - octrace.RegisterExporter(jex) - provider.exporter = jex - return nil -} - -func (provider *jaegerProvider) Unregister() error { - if provider.exporter == nil { - return nil - } - octrace.UnregisterExporter(provider.exporter) - provider.exporter.Flush() - provider.exporter = nil - return nil -} diff --git a/internal/telemetry/trace/main_test.go b/internal/telemetry/trace/main_test.go new file mode 100644 index 000000000..b546c8e45 --- /dev/null +++ b/internal/telemetry/trace/main_test.go @@ -0,0 +1,13 @@ +package trace_test + +import ( + "os" + "testing" + + "github.com/pomerium/pomerium/internal/telemetry/trace" +) + +func TestMain(m *testing.M) { + trace.UseGlobalPanicTracer() + os.Exit(m.Run()) +} diff --git a/internal/telemetry/trace/middleware.go b/internal/telemetry/trace/middleware.go new file mode 100644 index 000000000..771e44265 --- /dev/null +++ b/internal/telemetry/trace/middleware.go @@ -0,0 +1,96 @@ +package trace + +import ( + "context" + "fmt" + "net/http" + "reflect" + + "github.com/gorilla/mux" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "google.golang.org/grpc/stats" +) + +func NewHTTPMiddleware(opts ...otelhttp.Option) mux.MiddlewareFunc { + return otelhttp.NewMiddleware("Server: %s %s", append(opts, otelhttp.WithSpanNameFormatter(func(operation string, r *http.Request) string { + routeStr := "" + route := mux.CurrentRoute(r) + if route != nil { + var err error + routeStr, err = route.GetPathTemplate() + if err != nil { + routeStr, err = route.GetPathRegexp() + if err != nil { + routeStr = "" + } + } + } + return fmt.Sprintf(operation, r.Method, routeStr) + }))...) +} + +type clientStatsHandlerWrapper struct { + ClientStatsHandlerOptions + base stats.Handler +} + +type ClientStatsHandlerOptions struct { + statsInterceptor func(ctx context.Context, rs stats.RPCStats) stats.RPCStats +} + +type ClientStatsHandlerOption func(*ClientStatsHandlerOptions) + +func (o *ClientStatsHandlerOptions) apply(opts ...ClientStatsHandlerOption) { + for _, op := range opts { + op(o) + } +} + +// WithStatsInterceptor calls the given function to modify the rpc stats before +// passing it to the stats handler during HandleRPC events. +// +// The interceptor MUST NOT modify the RPCStats it is given. It should instead +// return a copy of the underlying object with the same type, with any +// modifications made to the copy. +func WithStatsInterceptor(statsInterceptor func(ctx context.Context, rs stats.RPCStats) stats.RPCStats) ClientStatsHandlerOption { + return func(o *ClientStatsHandlerOptions) { + o.statsInterceptor = statsInterceptor + } +} + +func NewClientStatsHandler(base stats.Handler, opts ...ClientStatsHandlerOption) stats.Handler { + options := ClientStatsHandlerOptions{} + options.apply(opts...) + return &clientStatsHandlerWrapper{ + ClientStatsHandlerOptions: options, + base: base, + } +} + +// HandleConn implements stats.Handler. +func (w *clientStatsHandlerWrapper) HandleConn(ctx context.Context, stats stats.ConnStats) { + w.base.HandleConn(ctx, stats) +} + +// HandleRPC implements stats.Handler. +func (w *clientStatsHandlerWrapper) HandleRPC(ctx context.Context, stats stats.RPCStats) { + if w.statsInterceptor != nil { + modified := w.statsInterceptor(ctx, stats) + if reflect.TypeOf(stats) != reflect.TypeOf(modified) { + panic("bug: stats interceptor returned a message of a different type") + } + w.base.HandleRPC(ctx, modified) + } else { + w.base.HandleRPC(ctx, stats) + } +} + +// TagConn implements stats.Handler. +func (w *clientStatsHandlerWrapper) TagConn(ctx context.Context, info *stats.ConnTagInfo) context.Context { + return w.base.TagConn(ctx, info) +} + +// TagRPC implements stats.Handler. +func (w *clientStatsHandlerWrapper) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context { + return w.base.TagRPC(ctx, info) +} diff --git a/internal/telemetry/trace/middleware_test.go b/internal/telemetry/trace/middleware_test.go new file mode 100644 index 000000000..ff01cc4d2 --- /dev/null +++ b/internal/telemetry/trace/middleware_test.go @@ -0,0 +1,210 @@ +package trace_test + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/gorilla/mux" + "github.com/pomerium/pomerium/internal/telemetry/trace" + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + oteltrace "go.opentelemetry.io/otel/trace" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/stats" +) + +func TestHTTPMiddleware(t *testing.T) { + router := mux.NewRouter() + tp := sdktrace.NewTracerProvider() + router.Use(trace.NewHTTPMiddleware( + otelhttp.WithTracerProvider(tp), + )) + router.Path("/foo").HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + span := oteltrace.SpanFromContext(r.Context()) + assert.Equal(t, "Server: GET /foo", span.(interface{ Name() string }).Name()) + }).Methods(http.MethodGet) + w := httptest.NewRecorder() + ctx, span := tp.Tracer("test").Start(context.Background(), "test") + router.ServeHTTP(w, httptest.NewRequestWithContext(ctx, http.MethodGet, "/foo", nil)) + span.End() +} + +type mockHandler struct { + handleConn func(ctx context.Context, stats stats.ConnStats) + handleRPC func(ctx context.Context, stats stats.RPCStats) + tagConn func(ctx context.Context, info *stats.ConnTagInfo) context.Context + tagRPC func(ctx context.Context, info *stats.RPCTagInfo) context.Context +} + +// HandleConn implements stats.Handler. +func (m *mockHandler) HandleConn(ctx context.Context, stats stats.ConnStats) { + m.handleConn(ctx, stats) +} + +// HandleRPC implements stats.Handler. +func (m *mockHandler) HandleRPC(ctx context.Context, stats stats.RPCStats) { + m.handleRPC(ctx, stats) +} + +// TagConn implements stats.Handler. +func (m *mockHandler) TagConn(ctx context.Context, info *stats.ConnTagInfo) context.Context { + return m.tagConn(ctx, info) +} + +// TagRPC implements stats.Handler. +func (m *mockHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context { + return m.tagRPC(ctx, info) +} + +var _ stats.Handler = (*mockHandler)(nil) + +func TestStatsInterceptor(t *testing.T) { + var outBegin *stats.Begin + var outEnd *stats.End + base := &mockHandler{ + handleRPC: func(_ context.Context, rs stats.RPCStats) { + switch rs := rs.(type) { + case *stats.Begin: + outBegin = rs + case *stats.End: + outEnd = rs + } + }, + } + interceptor := func(_ context.Context, rs stats.RPCStats) stats.RPCStats { + switch rs := rs.(type) { + case *stats.Begin: + return &stats.Begin{ + Client: rs.Client, + BeginTime: rs.BeginTime.Add(-1 * time.Minute), + FailFast: rs.FailFast, + IsClientStream: rs.IsClientStream, + IsServerStream: rs.IsServerStream, + IsTransparentRetryAttempt: rs.IsTransparentRetryAttempt, + } + case *stats.End: + return &stats.End{ + Client: rs.Client, + BeginTime: rs.BeginTime, + EndTime: rs.EndTime, + Trailer: rs.Trailer, + Error: errors.New("modified"), + } + } + return rs + } + handler := trace.NewClientStatsHandler( + base, + trace.WithStatsInterceptor(interceptor), + ) + inBegin := &stats.Begin{ + Client: true, + BeginTime: time.Now(), + FailFast: true, + IsClientStream: true, + IsServerStream: false, + IsTransparentRetryAttempt: false, + } + handler.HandleRPC(context.Background(), inBegin) + assert.NotNil(t, outBegin) + assert.NotSame(t, inBegin, outBegin) + assert.Equal(t, inBegin.BeginTime.Add(-1*time.Minute), outBegin.BeginTime) + assert.Equal(t, inBegin.Client, outBegin.Client) + assert.Equal(t, inBegin.FailFast, outBegin.FailFast) + assert.Equal(t, inBegin.IsClientStream, outBegin.IsClientStream) + assert.Equal(t, inBegin.IsServerStream, outBegin.IsServerStream) + assert.Equal(t, inBegin.IsTransparentRetryAttempt, outBegin.IsTransparentRetryAttempt) + + inEnd := &stats.End{ + Client: true, + BeginTime: time.Now(), + EndTime: time.Now().Add(1 * time.Minute), + Trailer: metadata.Pairs("a", "b", "c", "d"), + Error: errors.New("input"), + } + handler.HandleRPC(context.Background(), inEnd) + assert.NotNil(t, outEnd) + assert.NotSame(t, inEnd, outEnd) + assert.Equal(t, inEnd.Client, outEnd.Client) + assert.Equal(t, inEnd.BeginTime, outEnd.BeginTime) + assert.Equal(t, inEnd.EndTime, outEnd.EndTime) + assert.Equal(t, inEnd.Trailer, outEnd.Trailer) + assert.Equal(t, "input", inEnd.Error.Error()) + assert.Equal(t, "modified", outEnd.Error.Error()) +} + +func TestStatsInterceptor_Nil(t *testing.T) { + var outCtx context.Context + var outConnStats stats.ConnStats + var outRPCStats stats.RPCStats + var outConnTagInfo *stats.ConnTagInfo + var outRPCTagInfo *stats.RPCTagInfo + base := &mockHandler{ + handleConn: func(ctx context.Context, stats stats.ConnStats) { + outCtx = ctx + outConnStats = stats + }, + handleRPC: func(ctx context.Context, stats stats.RPCStats) { + outCtx = ctx + outRPCStats = stats + }, + tagConn: func(ctx context.Context, info *stats.ConnTagInfo) context.Context { + outCtx = ctx + outConnTagInfo = info + return ctx + }, + tagRPC: func(ctx context.Context, info *stats.RPCTagInfo) context.Context { + outCtx = ctx + outRPCTagInfo = info + return ctx + }, + } + handler := trace.NewClientStatsHandler( + base, + trace.WithStatsInterceptor(nil), + ) + + inCtx := context.Background() + inConnStats := &stats.ConnBegin{} + inRPCStats := &stats.Begin{} + inConnTagInfo := &stats.ConnTagInfo{} + inRPCTagInfo := &stats.RPCTagInfo{} + + handler.HandleConn(inCtx, inConnStats) + assert.Equal(t, inCtx, outCtx) + assert.Same(t, inConnStats, outConnStats) + + handler.HandleRPC(inCtx, inRPCStats) + assert.Equal(t, inCtx, outCtx) + assert.Same(t, inRPCStats, outRPCStats) + + handler.TagConn(inCtx, inConnTagInfo) + assert.Equal(t, inCtx, outCtx) + assert.Same(t, inConnTagInfo, outConnTagInfo) + + handler.TagRPC(inCtx, inRPCTagInfo) + assert.Equal(t, inCtx, outCtx) + assert.Same(t, inRPCTagInfo, outRPCTagInfo) +} + +func TestStatsInterceptor_Bug(t *testing.T) { + handler := trace.NewClientStatsHandler( + &mockHandler{ + handleRPC: func(_ context.Context, _ stats.RPCStats) { + t.Error("should not be reached") + }, + }, + trace.WithStatsInterceptor(func(_ context.Context, rs stats.RPCStats) stats.RPCStats { + _ = rs.(*stats.Begin) + return &stats.End{} + }), + ) + assert.PanicsWithValue(t, "bug: stats interceptor returned a message of a different type", func() { + handler.HandleRPC(context.Background(), &stats.Begin{}) + }) +} diff --git a/internal/telemetry/trace/server.go b/internal/telemetry/trace/server.go new file mode 100644 index 000000000..1bd14c7d0 --- /dev/null +++ b/internal/telemetry/trace/server.go @@ -0,0 +1,99 @@ +package trace + +import ( + "context" + "errors" + "net" + "time" + + "github.com/pomerium/pomerium/internal/log" + coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/test/bufconn" + + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" +) + +const localExporterMetadataKey = "x-local-exporter" + +// Export implements ptraceotlp.GRPCServer. +func (srv *ExporterServer) Export(ctx context.Context, req *coltracepb.ExportTraceServiceRequest) (*coltracepb.ExportTraceServiceResponse, error) { + if srv.observer != nil { + srv.observeExport(ctx, req) + } + if err := srv.remoteClient.UploadTraces(ctx, req.GetResourceSpans()); err != nil { + log.Ctx(ctx).Err(err).Msg("error uploading traces") + return nil, err + } + return &coltracepb.ExportTraceServiceResponse{}, nil +} + +type ExporterServer struct { + coltracepb.UnimplementedTraceServiceServer + server *grpc.Server + observer *spanObserver + remoteClient otlptrace.Client + cc *grpc.ClientConn +} + +func NewServer(ctx context.Context) *ExporterServer { + sys := systemContextFromContext(ctx) + ex := &ExporterServer{ + remoteClient: sys.options.RemoteClient, + observer: sys.observer, + server: grpc.NewServer(grpc.Creds(insecure.NewCredentials())), + } + coltracepb.RegisterTraceServiceServer(ex.server, ex) + return ex +} + +func (srv *ExporterServer) Start(ctx context.Context) { + lis := bufconn.Listen(4096) + go func() { + if err := srv.remoteClient.Start(ctx); err != nil { + panic(err) + } + _ = srv.server.Serve(lis) + }() + cc, err := grpc.NewClient("passthrough://ignore", + grpc.WithContextDialer(func(context.Context, string) (net.Conn, error) { + return lis.Dial() + }), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + panic(err) + } + srv.cc = cc +} + +func (srv *ExporterServer) NewClient() otlptrace.Client { + return otlptracegrpc.NewClient( + otlptracegrpc.WithGRPCConn(srv.cc), + otlptracegrpc.WithTimeout(1*time.Minute), + otlptracegrpc.WithHeaders(map[string]string{ + localExporterMetadataKey: "1", + }), + ) +} + +func (srv *ExporterServer) Shutdown(ctx context.Context) error { + stopped := make(chan struct{}) + go func() { + srv.server.GracefulStop() + close(stopped) + }() + select { + case <-stopped: + case <-ctx.Done(): + return context.Cause(ctx) + } + var errs []error + if err := WaitForSpans(ctx, 30*time.Second); err != nil { + errs = append(errs, err) + } + if err := srv.remoteClient.Stop(ctx); err != nil { + errs = append(errs, err) + } + return errors.Join(errs...) +} diff --git a/internal/telemetry/trace/trace.go b/internal/telemetry/trace/trace.go index 95099050f..0f03ec07e 100644 --- a/internal/telemetry/trace/trace.go +++ b/internal/telemetry/trace/trace.go @@ -2,87 +2,214 @@ package trace import ( "context" + "errors" "fmt" - "net/url" + "runtime" + "sync" + "sync/atomic" - octrace "go.opencensus.io/trace" - - "github.com/pomerium/pomerium/internal/log" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" + coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" ) -const ( - // DatadogTracingProviderName is the name of the tracing provider Datadog. - DatadogTracingProviderName = "datadog" - // JaegerTracingProviderName is the name of the tracing provider Jaeger. - JaegerTracingProviderName = "jaeger" - // ZipkinTracingProviderName is the name of the tracing provider Zipkin. - ZipkinTracingProviderName = "zipkin" -) - -// Provider is a trace provider. -type Provider interface { - Register(options *TracingOptions) error - Unregister() error +type Options struct { + DebugFlags DebugFlags + RemoteClient otlptrace.Client } -// TracingOptions contains the configurations settings for a http server. -type TracingOptions struct { - // Shared - Provider string - Service string - Debug bool - - // Datadog - DatadogAddress string - - // Jaeger - - // CollectorEndpoint is the full url to the Jaeger HTTP Thrift collector. - // For example, http://localhost:14268/api/traces - JaegerCollectorEndpoint *url.URL - // AgentEndpoint instructs exporter to send spans to jaeger-agent at this address. - // For example, localhost:6831. - JaegerAgentEndpoint string - - // Zipkin - - // ZipkinEndpoint configures the zipkin collector URI - // Example: http://zipkin:9411/api/v2/spans - ZipkinEndpoint *url.URL - - // SampleRate is percentage of requests which are sampled - SampleRate float64 -} - -// Enabled indicates whether tracing is enabled on a given TracingOptions -func (t *TracingOptions) Enabled() bool { - return t.Provider != "" -} - -// GetProvider creates a new trace provider from TracingOptions. -func GetProvider(opts *TracingOptions) (Provider, error) { - var provider Provider - switch opts.Provider { - case DatadogTracingProviderName: - provider = new(datadogProvider) - case JaegerTracingProviderName: - provider = new(jaegerProvider) - case ZipkinTracingProviderName: - provider = new(zipkinProvider) - default: - return nil, fmt.Errorf("telemetry/trace: provider %s unknown", opts.Provider) +func (op Options) NewContext(parent context.Context) context.Context { + if systemContextFromContext(parent) != nil { + panic("parent already contains trace system context") } - octrace.ApplyConfig(octrace.Config{DefaultSampler: octrace.ProbabilitySampler(opts.SampleRate)}) - - log.Debug().Interface("Opts", opts).Msg("telemetry/trace: provider created") - return provider, nil + if op.RemoteClient == nil { + op.RemoteClient = NewRemoteClientFromEnv() + } + sys := &systemContext{ + options: op, + tpm: &tracerProviderManager{}, + } + if op.DebugFlags.Check(TrackSpanReferences) { + sys.observer = newSpanObserver() + } + ctx := context.WithValue(parent, systemContextKey, sys) + sys.exporterServer = NewServer(ctx) + sys.exporterServer.Start(ctx) + return ctx } -// StartSpan starts a new child span of the current span in the context. If -// there is no span in the context, creates a new trace and span. +// NewContext creates a new top-level background context with tracing machinery +// and configuration that will be used when creating new tracer providers. // -// Returned context contains the newly created span. You can use it to -// propagate the returned span in process. -func StartSpan(ctx context.Context, name string, o ...octrace.StartOption) (context.Context, *octrace.Span) { - return octrace.StartSpan(ctx, name, o...) +// Any context created with NewContext should eventually be shut down by calling +// [ShutdownContext] to ensure all traces are exported. +// +// The parent context should be context.Background(), or a background context +// containing a logger. If any context in the parent's hierarchy was created +// by NewContext, this will panic. +func NewContext(parent context.Context) context.Context { + return Options{}.NewContext(parent) +} + +// NewTracerProvider creates a new [trace.TracerProvider] with the given service +// name and options. +// +// A context returned by [NewContext] must exist somewhere in the hierarchy of +// ctx, otherwise a no-op TracerProvider is returned. The configuration embedded +// within that context will be used to configure its resource attributes and +// exporter automatically. +func NewTracerProvider(ctx context.Context, serviceName string, opts ...sdktrace.TracerProviderOption) trace.TracerProvider { + sys := systemContextFromContext(ctx) + if sys == nil { + return noop.NewTracerProvider() + } + _, file, line, _ := runtime.Caller(1) + exp, err := otlptrace.New(ctx, sys.exporterServer.NewClient()) + if err != nil { + panic(err) + } + r, err := resource.Merge( + resource.Default(), + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceName(serviceName), + attribute.String("provider.created_at", fmt.Sprintf("%s:%d", file, line)), + ), + ) + if err != nil { + panic(err) + } + options := []sdktrace.TracerProviderOption{} + if sys.options.DebugFlags.Check(TrackSpanCallers) { + options = append(options, sdktrace.WithSpanProcessor(&stackTraceProcessor{})) + } + if sys.options.DebugFlags.Check(TrackSpanReferences) { + tracker := newSpanTracker(sys.observer, sys.options.DebugFlags) + options = append(options, sdktrace.WithSpanProcessor(tracker)) + } + options = append(append(options, + sdktrace.WithBatcher(exp), + sdktrace.WithResource(r), + ), opts...) + tp := sdktrace.NewTracerProvider(options...) + sys.tpm.Add(tp) + return tp +} + +// Continue starts a new span using the tracer provider of the span in the given +// context. +// +// In most cases, it is better to start spans directly from a specific tracer, +// obtained via dependency injection or some other mechanism. This function is +// useful in shared code where the tracer used to start the span is not +// necessarily the same every time, but can change based on the call site. +func Continue(ctx context.Context, name string, o ...trace.SpanStartOption) (context.Context, trace.Span) { + return trace.SpanFromContext(ctx). + TracerProvider(). + Tracer(PomeriumCoreTracer). + Start(ctx, name, o...) +} + +// ShutdownContext will gracefully shut down all tracing resources created with +// a context returned by [NewContext], including all tracer providers and the +// underlying exporter and remote client. +// +// This should only be called once before exiting, but subsequent calls are +// a no-op. +// +// The provided context does not necessarily need to be the exact context +// returned by [NewContext]; it can be anywhere in its context hierarchy and +// this function will have the same effect. +func ShutdownContext(ctx context.Context) error { + sys := systemContextFromContext(ctx) + if sys == nil { + panic("context was not created with trace.NewContext") + } + + if !sys.shutdown.CompareAndSwap(false, true) { + return nil + } + + var errs []error + if err := sys.tpm.ShutdownAll(context.Background()); err != nil { + errs = append(errs, fmt.Errorf("error shutting down tracer providers: %w", err)) + } + if err := sys.exporterServer.Shutdown(context.Background()); err != nil { + errs = append(errs, fmt.Errorf("error shutting down trace exporter: %w", err)) + } + return errors.Join(errs...) +} + +func ExporterServerFromContext(ctx context.Context) coltracepb.TraceServiceServer { + if sys := systemContextFromContext(ctx); sys != nil { + return sys.exporterServer + } + return nil +} + +func RemoteClientFromContext(ctx context.Context) otlptrace.Client { + if sys := systemContextFromContext(ctx); sys != nil { + return sys.options.RemoteClient + } + return nil +} + +// ForceFlush immediately exports all spans that have not yet been exported for +// all tracer providers created using the given context. +func ForceFlush(ctx context.Context) error { + if sys := systemContextFromContext(ctx); sys != nil { + var errs []error + for _, tp := range sys.tpm.tracerProviders { + errs = append(errs, tp.ForceFlush(ctx)) + } + return errors.Join(errs...) + } + return nil +} + +type systemContextKeyType struct{} + +var systemContextKey systemContextKeyType + +type systemContext struct { + options Options + tpm *tracerProviderManager + observer *spanObserver + exporterServer *ExporterServer + shutdown atomic.Bool +} + +func systemContextFromContext(ctx context.Context) *systemContext { + sys, _ := ctx.Value(systemContextKey).(*systemContext) + return sys +} + +type tracerProviderManager struct { + mu sync.Mutex + tracerProviders []*sdktrace.TracerProvider +} + +func (tpm *tracerProviderManager) ShutdownAll(ctx context.Context) error { + tpm.mu.Lock() + defer tpm.mu.Unlock() + var errs []error + for _, tp := range tpm.tracerProviders { + errs = append(errs, tp.ForceFlush(ctx)) + } + for _, tp := range tpm.tracerProviders { + errs = append(errs, tp.Shutdown(ctx)) + } + clear(tpm.tracerProviders) + return errors.Join(errs...) +} + +func (tpm *tracerProviderManager) Add(tp *sdktrace.TracerProvider) { + tpm.mu.Lock() + defer tpm.mu.Unlock() + tpm.tracerProviders = append(tpm.tracerProviders, tp) } diff --git a/internal/telemetry/trace/trace_export_test.go b/internal/telemetry/trace/trace_export_test.go new file mode 100644 index 000000000..62f6cdcf0 --- /dev/null +++ b/internal/telemetry/trace/trace_export_test.go @@ -0,0 +1,72 @@ +package trace + +import ( + "cmp" + "io" + "slices" + "testing" + "time" + + oteltrace "go.opentelemetry.io/otel/trace" +) + +var ( + NewSpanObserver = newSpanObserver + NewSpanTracker = newSpanTracker +) + +type XStackTraceProcessor = stackTraceProcessor + +func (obs *spanObserver) XWait() { + obs.wait(5 * time.Second) +} + +func (obs *spanObserver) XUnobservedIDs() []oteltrace.SpanID { + obs.cond.L.Lock() + defer obs.cond.L.Unlock() + ids := []oteltrace.SpanID{} + for k, v := range obs.referencedIDs { + if v.IsValid() { + ids = append(ids, k) + } + } + slices.SortFunc(ids, func(a, b oteltrace.SpanID) int { + return cmp.Compare(a.String(), b.String()) + }) + return ids +} + +func (obs *spanObserver) XObservedIDs() []oteltrace.SpanID { + obs.cond.L.Lock() + defer obs.cond.L.Unlock() + ids := []oteltrace.SpanID{} + for k, v := range obs.referencedIDs { + if !v.IsValid() { + ids = append(ids, k) + } + } + slices.SortFunc(ids, func(a, b oteltrace.SpanID) int { + return cmp.Compare(a.String(), b.String()) + }) + return ids +} + +func (t *spanTracker) XInflightSpans() []oteltrace.SpanID { + ids := []oteltrace.SpanID{} + t.inflightSpansMu.LockAll() + t.inflightSpans.Range(func(key oteltrace.SpanID) { + ids = append(ids, key) + }) + t.inflightSpansMu.UnlockAll() + slices.SortFunc(ids, func(a, b oteltrace.SpanID) int { + return cmp.Compare(a.String(), b.String()) + }) + return ids +} + +func SetDebugMessageWriterForTest(t testing.TB, w io.Writer) { + debugMessageWriter = w + t.Cleanup(func() { + debugMessageWriter = nil + }) +} diff --git a/internal/telemetry/trace/trace_test.go b/internal/telemetry/trace/trace_test.go deleted file mode 100644 index b22afb98e..000000000 --- a/internal/telemetry/trace/trace_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package trace - -import ( - "net/url" - "testing" -) - -func TestGetProvider(t *testing.T) { - tests := []struct { - name string - opts *TracingOptions - wantErr bool - }{ - {"jaeger", &TracingOptions{JaegerAgentEndpoint: "localhost:6831", Service: "all", Provider: "jaeger"}, false}, - {"jaeger with debug", &TracingOptions{JaegerAgentEndpoint: "localhost:6831", Service: "all", Provider: "jaeger", Debug: true}, false}, - {"jaeger no endpoint", &TracingOptions{JaegerAgentEndpoint: "", Service: "all", Provider: "jaeger"}, false}, - {"unknown provider", &TracingOptions{JaegerAgentEndpoint: "localhost:0", Service: "all", Provider: "Lucius Cornelius Sulla"}, true}, - {"zipkin with debug", &TracingOptions{ZipkinEndpoint: &url.URL{Host: "localhost"}, Service: "all", Provider: "zipkin", Debug: true}, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if _, err := GetProvider(tt.opts); (err != nil) != tt.wantErr { - t.Errorf("RegisterTracing() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/internal/telemetry/trace/util.go b/internal/telemetry/trace/util.go new file mode 100644 index 000000000..ec1b28437 --- /dev/null +++ b/internal/telemetry/trace/util.go @@ -0,0 +1,32 @@ +package trace + +import ( + "unique" + + oteltrace "go.opentelemetry.io/otel/trace" +) + +var ( + zeroSpanID oteltrace.SpanID + zeroTraceID = unique.Make(oteltrace.TraceID([16]byte{})) +) + +func ToSpanID(bytes []byte) (oteltrace.SpanID, bool) { + switch len(bytes) { + case 0: + return zeroSpanID, true + case 8: + return oteltrace.SpanID(bytes), true + } + return zeroSpanID, false +} + +func ToTraceID(bytes []byte) (unique.Handle[oteltrace.TraceID], bool) { + switch len(bytes) { + case 0: + return zeroTraceID, true + case 16: + return unique.Make(oteltrace.TraceID(bytes)), true + } + return zeroTraceID, false +} diff --git a/internal/telemetry/trace/zipkin.go b/internal/telemetry/trace/zipkin.go deleted file mode 100644 index de4188277..000000000 --- a/internal/telemetry/trace/zipkin.go +++ /dev/null @@ -1,49 +0,0 @@ -package trace - -import ( - "fmt" - stdlog "log" - - oczipkin "contrib.go.opencensus.io/exporter/zipkin" - "github.com/openzipkin/zipkin-go" - "github.com/openzipkin/zipkin-go/reporter" - zipkinHTTP "github.com/openzipkin/zipkin-go/reporter/http" - octrace "go.opencensus.io/trace" - - "github.com/pomerium/pomerium/internal/log" -) - -type zipkinProvider struct { - reporter reporter.Reporter - exporter *oczipkin.Exporter -} - -func (provider *zipkinProvider) Register(opts *TracingOptions) error { - localEndpoint, err := zipkin.NewEndpoint(opts.Service, "") - if err != nil { - return fmt.Errorf("telemetry/trace: could not create local endpoint: %w", err) - } - - logger := log.With().Str("service", "zipkin").Logger() - logWriter := &log.StdLogWrapper{Logger: &logger} - stdLogger := stdlog.New(logWriter, "", 0) - - provider.reporter = zipkinHTTP.NewReporter(opts.ZipkinEndpoint.String(), zipkinHTTP.Logger(stdLogger)) - provider.exporter = oczipkin.NewExporter(provider.reporter, localEndpoint) - octrace.RegisterExporter(provider.exporter) - return nil -} - -func (provider *zipkinProvider) Unregister() error { - if provider.exporter != nil { - octrace.UnregisterExporter(provider.exporter) - provider.exporter = nil - } - - var err error - if provider.reporter != nil { - err = provider.reporter.Close() - provider.reporter = nil - } - return err -} diff --git a/internal/testenv/environment.go b/internal/testenv/environment.go index beef11ec2..e4f4d9857 100644 --- a/internal/testenv/environment.go +++ b/internal/testenv/environment.go @@ -17,7 +17,6 @@ import ( "fmt" "io" "math/big" - "math/bits" "net" "net/url" "os" @@ -26,25 +25,34 @@ import ( "path/filepath" "runtime" "strconv" + "strings" "sync" + "sync/atomic" "syscall" "testing" "time" "github.com/pomerium/pomerium/config" "github.com/pomerium/pomerium/config/envoyconfig/filemgr" + databroker_service "github.com/pomerium/pomerium/databroker" "github.com/pomerium/pomerium/internal/log" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/testenv/envutil" "github.com/pomerium/pomerium/internal/testenv/values" "github.com/pomerium/pomerium/pkg/cmd/pomerium" "github.com/pomerium/pomerium/pkg/envoy" "github.com/pomerium/pomerium/pkg/grpc/databroker" "github.com/pomerium/pomerium/pkg/health" + "github.com/pomerium/pomerium/pkg/identity/legacymanager" + "github.com/pomerium/pomerium/pkg/identity/manager" "github.com/pomerium/pomerium/pkg/netutil" "github.com/pomerium/pomerium/pkg/slices" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + oteltrace "go.opentelemetry.io/otel/trace" "golang.org/x/sync/errgroup" "google.golang.org/grpc/grpclog" ) @@ -56,6 +64,7 @@ type Environment interface { // top-level logger scoped to this environment. It will be canceled when // Stop() is called, or during test cleanup. Context() context.Context + Tracer() oteltrace.Tracer Assert() *assert.Assertions Require() *require.Assertions @@ -133,10 +142,29 @@ type Environment interface { // the Pomerium server and Envoy. NewLogRecorder(opts ...LogRecorderOption) *LogRecorder + // GetState returns the current state of the test environment. + GetState() EnvironmentState + // OnStateChanged registers a callback to be invoked when the environment's - // state changes to the given state. The callback is invoked in a separate - // goroutine. - OnStateChanged(state EnvironmentState, callback func()) + // state changes to the given state. Each callback is invoked in a separate + // goroutine, but the test environment will wait for all callbacks to return + // before continuing, after triggering the state change. + // + // Calling the returned stop function will prevent the callback from being + // run. Returns true if it stopped the callback from being run, or false if + // it already ran or is currently running. + // + // If the environment is already in the given state, the callback will be run + // in a separate goroutine immediately and the returned stop function will + // have no effect. A callback run in this way will prevent the state from + // advancing until the callback returns. + // + // State changes are triggered in the following places: + // - NotRunning->Starting: in Start(), as the first operation + // - Starting->Running: in Start(), just before returning + // - Running->Stopping: in Stop(), just before the env context is canceled + // - Stopping->Stopped: in Stop(), after all tasks have completed + OnStateChanged(state EnvironmentState, callback func()) (stop func() bool) } type Certificate tls.Certificate @@ -153,10 +181,9 @@ func (c *Certificate) SPKIHash() string { type EnvironmentState uint32 -const NotRunning EnvironmentState = 0 - const ( - Starting EnvironmentState = 1 << iota + NotRunning EnvironmentState = iota + Starting Running Stopping Stopped @@ -192,10 +219,13 @@ type environment struct { workspaceFolder string silent bool - ctx context.Context - cancel context.CancelCauseFunc - cleanupOnce sync.Once - logWriter *log.MultiWriter + ctx context.Context + cancel context.CancelCauseFunc + cleanupOnce sync.Once + logWriter *log.MultiWriter + tracerProvider oteltrace.TracerProvider + tracer oteltrace.Tracer + rootSpan oteltrace.Span mods []WithCaller[Modifier] tasks []WithCaller[Task] @@ -204,14 +234,17 @@ type environment struct { stateMu sync.Mutex state EnvironmentState stateChangeListeners map[EnvironmentState][]func() + stateChangeBlockers sync.WaitGroup src *configSource } type EnvironmentOptions struct { - debug bool - pauseOnFailure bool - forceSilent bool + debug bool + pauseOnFailure bool + forceSilent bool + traceDebugFlags trace.DebugFlags + traceClient otlptrace.Client } type EnvironmentOption func(*EnvironmentOptions) @@ -249,28 +282,57 @@ func Silent(silent ...bool) EnvironmentOption { } } +const StandardTraceDebugFlags = trace.TrackSpanCallers | + trace.WarnOnIncompleteSpans | + trace.WarnOnIncompleteTraces | + trace.WarnOnUnresolvedReferences | + trace.LogTraceIDsOnWarn | + trace.LogAllSpansOnWarn + +func WithTraceDebugFlags(flags trace.DebugFlags) EnvironmentOption { + return func(o *EnvironmentOptions) { + o.traceDebugFlags = flags + } +} + +func WithTraceClient(traceClient otlptrace.Client) EnvironmentOption { + return func(o *EnvironmentOptions) { + o.traceClient = traceClient + } +} + var setGrpcLoggerOnce sync.Once +const defaultTraceDebugFlags = trace.TrackSpanCallers | trace.TrackSpanReferences + var ( - flagDebug = flag.Bool("env.debug", false, "enables test environment debug logging (equivalent to Debug() option)") - flagPauseOnFailure = flag.Bool("env.pause-on-failure", false, "enables pausing the test environment on failure (equivalent to PauseOnFailure() option)") - flagSilent = flag.Bool("env.silent", false, "suppresses all test environment output (equivalent to Silent() option)") + flagDebug = flag.Bool("env.debug", false, "enables test environment debug logging (equivalent to Debug() option)") + flagPauseOnFailure = flag.Bool("env.pause-on-failure", false, "enables pausing the test environment on failure (equivalent to PauseOnFailure() option)") + flagSilent = flag.Bool("env.silent", false, "suppresses all test environment output (equivalent to Silent() option)") + flagTraceDebugFlags = flag.String("env.trace-debug-flags", strconv.Itoa(defaultTraceDebugFlags), "trace debug flags (equivalent to TraceDebugFlags() option)") ) func New(t testing.TB, opts ...EnvironmentOption) Environment { - if runtime.GOOS != "linux" { - t.Skip("test environment only supported on linux") + addTraceDebugFlags := strings.HasPrefix(*flagTraceDebugFlags, "+") + defaultTraceDebugFlags, err := strconv.Atoi(strings.TrimPrefix(*flagTraceDebugFlags, "+")) + if err != nil { + panic("malformed value for --env.trace-debug-flags: " + err.Error()) } options := EnvironmentOptions{ - debug: *flagDebug, - pauseOnFailure: *flagPauseOnFailure, - forceSilent: *flagSilent, + debug: *flagDebug, + pauseOnFailure: *flagPauseOnFailure, + forceSilent: *flagSilent, + traceDebugFlags: trace.DebugFlags(defaultTraceDebugFlags), } options.apply(opts...) if testing.Short() { t.Helper() t.Skip("test environment disabled in short mode") } + if addTraceDebugFlags { + options.traceDebugFlags |= trace.DebugFlags(defaultTraceDebugFlags) + } + trace.UseGlobalPanicTracer() databroker.DebugUseFasterBackoff.Store(true) workspaceFolder, err := os.Getwd() require.NoError(t, err) @@ -305,7 +367,16 @@ func New(t testing.TB, opts ...EnvironmentOption) Environment { }) logger := zerolog.New(writer).With().Timestamp().Logger().Level(zerolog.DebugLevel) - ctx, cancel := context.WithCancelCause(logger.WithContext(context.Background())) + ctx := trace.Options{ + DebugFlags: options.traceDebugFlags, + RemoteClient: options.traceClient, + }.NewContext(logger.WithContext(context.Background())) + tracerProvider := trace.NewTracerProvider(ctx, "Test Environment") + tracer := tracerProvider.Tracer(trace.PomeriumCoreTracer) + ctx, span := tracer.Start(ctx, t.Name(), oteltrace.WithNewRoot()) + require.NoError(t, err) + + ctx, cancel := context.WithCancelCause(ctx) taskErrGroup, ctx := errgroup.WithContext(ctx) e := &environment{ @@ -313,7 +384,7 @@ func New(t testing.TB, opts ...EnvironmentOption) Environment { t: t, assert: assert.New(t), require: require.New(t), - tempDir: t.TempDir(), + tempDir: tempDir(t), ports: Ports{ ProxyHTTP: values.Deferred[int](), ProxyGRPC: values.Deferred[int](), @@ -325,13 +396,18 @@ func New(t testing.TB, opts ...EnvironmentOption) Environment { Debug: values.Deferred[int](), ALPN: values.Deferred[int](), }, - workspaceFolder: workspaceFolder, - silent: silent, - ctx: ctx, - cancel: cancel, - logWriter: writer, - taskErrGroup: taskErrGroup, + workspaceFolder: workspaceFolder, + silent: silent, + ctx: ctx, + cancel: cancel, + tracerProvider: tracerProvider, + tracer: tracer, + logWriter: writer, + taskErrGroup: taskErrGroup, + stateChangeListeners: make(map[EnvironmentState][]func()), + rootSpan: span, } + _, err = rand.Read(e.sharedSecret[:]) require.NoError(t, err) _, err = rand.Read(e.cookieSecret[:]) @@ -362,11 +438,13 @@ func New(t testing.TB, opts ...EnvironmentOption) Environment { func (e *environment) debugf(format string, args ...any) { e.t.Helper() + if e.rootSpan.IsRecording() { + e.rootSpan.AddEvent(fmt.Sprintf(format, args...)) + } if !e.debug { return } - - e.t.Logf("\x1b[34m[debug] "+format+"\x1b[0m", args...) + e.t.Logf("\x1b[34mDEBUG ["+e.t.Name()+"] "+format+"\x1b[0m", args...) } type WithCaller[T any] struct { @@ -394,6 +472,10 @@ func (e *environment) Context() context.Context { return ContextWithEnv(e.ctx, e) } +func (e *environment) Tracer() oteltrace.Tracer { + return e.tracer +} + func (e *environment) Assert() *assert.Assertions { return e.assert } @@ -455,9 +537,11 @@ var ErrCauseTestCleanup = errors.New("test cleanup") var ErrCauseManualStop = errors.New("Stop() called") func (e *environment) Start() { + _, span := e.tracer.Start(e.Context(), "Start") + defer span.End() e.debugf("Start()") e.advanceState(Starting) - e.t.Cleanup(e.cleanup) + e.t.Cleanup(e.onTestCleanup) e.t.Setenv("TMPDIR", e.TempDir()) e.debugf("temp dir: %s", e.TempDir()) @@ -524,8 +608,13 @@ func (e *environment) Start() { require.NoError(e.t, cfg.Options.Validate(), "invoking modifier resulted in an invalid configuration:\nadded by: "+mod.Caller) } - opts := []pomerium.RunOption{ + opts := []pomerium.Option{ pomerium.WithOverrideFileManager(fileMgr), + pomerium.WithEnvoyServerOptions(envoy.WithExitGracePeriod(30 * time.Second)), + pomerium.WithDataBrokerServerOptions( + databroker_service.WithManagerOptions(manager.WithLeaseTTL(1*time.Second)), + databroker_service.WithLegacyManagerOptions(legacymanager.WithLeaseTTL(1*time.Second)), + ), } envoyBinaryPath := filepath.Join(e.workspaceFolder, fmt.Sprintf("pkg/envoy/files/envoy-%s-%s", runtime.GOOS, runtime.GOARCH)) if envutil.EnvoyProfilerAvailable(envoyBinaryPath) { @@ -556,23 +645,29 @@ func (e *environment) Start() { } if len(envVars) > 0 { e.debugf("adding envoy env vars: %v\n", envVars) - opts = append(opts, pomerium.WithEnvoyServerOptions( - envoy.WithExtraEnvVars(envVars...), - envoy.WithExitGracePeriod(10*time.Second), // allow envoy time to flush pprof data to disk - )) + opts = append(opts, pomerium.WithEnvoyServerOptions(envoy.WithExtraEnvVars(envVars...))) } } else { e.debugf("envoy profiling not available") } - return pomerium.Run(ctx, e.src, opts...) + pom := pomerium.New(opts...) + e.OnStateChanged(Stopping, func() { + if err := pom.Shutdown(ctx); err != nil { + log.Ctx(ctx).Err(err).Msg("error shutting down pomerium server") + } else { + e.debugf("pomerium server shut down without error") + } + }) + require.NoError(e.t, pom.Start(ctx, e.tracerProvider, e.src)) + return pom.Wait() })) for i, task := range e.tasks { - log.Ctx(e.ctx).Debug().Str("caller", task.Caller).Msgf("starting task %d", i) + log.Ctx(e.Context()).Debug().Str("caller", task.Caller).Msgf("starting task %d", i) e.taskErrGroup.Go(func() error { - defer log.Ctx(e.ctx).Debug().Str("caller", task.Caller).Msgf("task %d exited", i) - return task.Value.Run(e.ctx) + defer log.Ctx(e.Context()).Debug().Str("caller", task.Caller).Msgf("task %d exited", i) + return task.Value.Run(e.Context()) }) } @@ -695,14 +790,9 @@ func (e *environment) Stop() { b.StopTimer() defer b.StartTimer() } + _, file, line, _ := runtime.Caller(1) e.cleanupOnce.Do(func() { - e.debugf("stop: Stop() called manually") - e.advanceState(Stopping) - e.cancel(ErrCauseManualStop) - err := e.taskErrGroup.Wait() - e.advanceState(Stopped) - e.debugf("stop: done waiting") - assert.ErrorIs(e.t, err, ErrCauseManualStop) + e.cleanup(fmt.Errorf("%w (caller: %s:%d)", ErrCauseManualStop, file, line)) }) } @@ -714,33 +804,51 @@ func (e *environment) Pause() { e.t.Log("\x1b[31mctrl+c received, continuing\x1b[0m") } -func (e *environment) cleanup() { +func (e *environment) onTestCleanup() { e.cleanupOnce.Do(func() { - e.debugf("stop: test cleanup") - if e.t.Failed() { - if e.pauseOnFailure { - e.t.Log("\x1b[31m*** pausing on test failure; continue with ctrl+c ***\x1b[0m") - c := make(chan os.Signal, 1) - signal.Notify(c, syscall.SIGINT) - <-c - e.t.Log("\x1b[31mctrl+c received, continuing\x1b[0m") - signal.Stop(c) - } - } - e.advanceState(Stopping) - e.cancel(ErrCauseTestCleanup) - err := e.taskErrGroup.Wait() - e.advanceState(Stopped) - e.debugf("stop: done waiting") - assert.ErrorIs(e.t, err, ErrCauseTestCleanup) + e.cleanup(ErrCauseTestCleanup) }) } +func (e *environment) cleanup(cancelCause error) { + e.debugf("stop: %s", cancelCause.Error()) + if e.t.Failed() { + if e.pauseOnFailure { + e.t.Log("\x1b[31m*** pausing on test failure; continue with ctrl+c ***\x1b[0m") + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGINT) + <-c + e.t.Log("\x1b[31mctrl+c received, continuing\x1b[0m") + signal.Stop(c) + } + } + e.advanceState(Stopping) + e.cancel(cancelCause) + errs := []error{} + if err := e.taskErrGroup.Wait(); err != nil { + errs = append(errs, fmt.Errorf("error waiting for tasks: %w", err)) + } + e.rootSpan.End() + if err := trace.ShutdownContext(e.Context()); err != nil { + errs = append(errs, fmt.Errorf("error shutting down trace context: %w", err)) + } + e.advanceState(Stopped) + // Wait for any additional callbacks created during stopped callbacks + // (for consistency, we consider the stopped state to "end" here) + e.stateChangeBlockers.Wait() + e.debugf("stop: done") + // err can be nil if e.g. the only task is the internal pomerium task, which + // returns a nil error if it exits cleanly + if err := errors.Join(errs...); err != nil { + assert.ErrorIs(e.t, err, cancelCause) + } +} + func (e *environment) Add(m Modifier) { e.t.Helper() caller := getCaller() e.debugf("Add: %T from %s", m, caller) - switch e.getState() { + switch e.GetState() { case NotRunning: for _, mod := range e.mods { if mod.Value == m { @@ -757,11 +865,11 @@ func (e *environment) Add(m Modifier) { panic("test bug: cannot call Add() before Start() has returned") case Running: e.debugf("Add: state=Running; calling ModifyConfig") - e.src.ModifyConfig(e.ctx, m) + e.src.ModifyConfig(e.Context(), m) case Stopped, Stopping: panic("test bug: cannot call Add() after Stop()") default: - panic(fmt.Sprintf("unexpected environment state: %s", e.getState())) + panic(fmt.Sprintf("unexpected environment state: %s", e.GetState())) } } @@ -803,34 +911,63 @@ func (e *environment) advanceState(newState EnvironmentState) { if newState <= e.state { panic(fmt.Sprintf("internal test environment bug: changed state to <= current: newState=%s, current=%s", newState, e.state)) } + e.stateChangeBlockers.Wait() e.debugf("state %s -> %s", e.state.String(), newState.String()) e.state = newState - e.debugf("notifying %d listeners of state change", len(e.stateChangeListeners[newState])) - for _, listener := range e.stateChangeListeners[newState] { - go listener() + if len(e.stateChangeListeners[newState]) > 0 { + e.debugf("notifying %d listeners of state change", len(e.stateChangeListeners[newState])) + var wg sync.WaitGroup + for _, listener := range e.stateChangeListeners[newState] { + wg.Add(1) + go func() { + _, span := e.tracer.Start(e.Context(), "State Change Callback") + span.SetAttributes(attribute.String("state", newState.String())) + defer span.End() + defer wg.Done() + listener() + }() + } + wg.Wait() + e.debugf("done notifying state change listeners") } } -func (e *environment) getState() EnvironmentState { +func (e *environment) GetState() EnvironmentState { e.stateMu.Lock() defer e.stateMu.Unlock() return e.state } -func (e *environment) OnStateChanged(state EnvironmentState, callback func()) { +func (e *environment) OnStateChanged(state EnvironmentState, callback func()) (cancel func() bool) { e.stateMu.Lock() defer e.stateMu.Unlock() - if e.state&state != 0 { - go callback() - return - } - - // add change listeners for all states, if there are multiple bits set - for state > 0 { - stateBit := EnvironmentState(bits.TrailingZeros32(uint32(state))) - state &= (state - 1) - e.stateChangeListeners[stateBit] = append(e.stateChangeListeners[stateBit], callback) + _, file, line, _ := runtime.Caller(1) + switch { + case state < e.state: + panic(fmt.Sprintf("test bug: OnStateChanged called with state %s which is < current state (%s)", state, e.sharedSecret)) + case state == e.state: + e.stateChangeBlockers.Add(1) + e.debugf("invoking callback for current state (state: %s, caller: %s:%d)", state.String(), file, line) + go func() { + defer func() { + e.stateChangeBlockers.Done() + }() + callback() + }() + return func() bool { return false } + default: + canceled := &atomic.Bool{} + e.stateChangeListeners[state] = append(e.stateChangeListeners[state], func() { + if canceled.CompareAndSwap(false, true) { + e.debugf("invoking state change callback (caller: %s:%d)", file, line) + callback() + } + }) + return func() bool { + e.debugf("stopped state change callback (state: %s, caller: %s:%d)", state.String(), file, line) + return canceled.CompareAndSwap(false, true) + } } } diff --git a/internal/testenv/logs.go b/internal/testenv/logs.go index 73789f10d..e7346902b 100644 --- a/internal/testenv/logs.go +++ b/internal/testenv/logs.go @@ -12,6 +12,7 @@ import ( "io" "os" "reflect" + "regexp" "sync" "testing" "time" @@ -128,7 +129,7 @@ func (e *environment) NewLogRecorder(opts ...LogRecorderOption) *LogRecorder { lr := &LogRecorder{ LogRecorderOptions: options, t: e.t, - canceled: e.ctx.Done(), + canceled: e.Context().Done(), buf: newBuffer(), } e.logWriter.Add(lr.buf) @@ -139,7 +140,7 @@ func (e *environment) NewLogRecorder(opts ...LogRecorderOption) *LogRecorder { } e.logWriter.Remove(lr.buf) }) - context.AfterFunc(e.ctx, lr.removeGlobalWriterOnce) + context.AfterFunc(e.Context(), lr.removeGlobalWriterOnce) return lr } @@ -356,6 +357,11 @@ func match(expected, actual map[string]any, open bool) (matched bool, score int) return false, score } score++ + case *regexp.Regexp: + if !value.MatchString(actualValue) { + return false, score + } + score++ default: return false, score } diff --git a/internal/testenv/scenarios/mock_idp.go b/internal/testenv/scenarios/mock_idp.go index e6d6e539e..85babd674 100644 --- a/internal/testenv/scenarios/mock_idp.go +++ b/internal/testenv/scenarios/mock_idp.go @@ -45,7 +45,7 @@ type IDP struct { func (idp *IDP) Attach(ctx context.Context) { env := testenv.EnvFromContext(ctx) - router := upstreams.HTTP(nil) + router := upstreams.HTTP(nil, upstreams.WithDisplayName("IDP")) idp.url = values.Bind2(env.SubdomainURL("mock-idp"), router.Port(), func(urlStr string, port int) string { u, _ := url.Parse(urlStr) diff --git a/internal/testenv/scenarios/trace_receiver.go b/internal/testenv/scenarios/trace_receiver.go new file mode 100644 index 000000000..cebdf8962 --- /dev/null +++ b/internal/testenv/scenarios/trace_receiver.go @@ -0,0 +1,207 @@ +package scenarios + +import ( + "compress/gzip" + "context" + "fmt" + "io" + "net/http" + "sync" + "time" + + "github.com/pomerium/pomerium/config" + "github.com/pomerium/pomerium/internal/testenv" + "github.com/pomerium/pomerium/internal/testenv/upstreams" + "github.com/pomerium/pomerium/internal/testenv/values" + "github.com/pomerium/pomerium/internal/testutil/tracetest" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" + "google.golang.org/protobuf/proto" +) + +type OTLPTraceReceiver struct { + coltracepb.UnimplementedTraceServiceServer + + mu sync.Mutex + receivedRequests []*coltracepb.ExportTraceServiceRequest + grpcUpstream values.MutableValue[upstreams.GRPCUpstream] + httpUpstream values.MutableValue[upstreams.HTTPUpstream] +} + +func NewOTLPTraceReceiver() *OTLPTraceReceiver { + return &OTLPTraceReceiver{ + grpcUpstream: values.Deferred[upstreams.GRPCUpstream](), + httpUpstream: values.Deferred[upstreams.HTTPUpstream](), + } +} + +// Export implements v1.TraceServiceServer. +func (rec *OTLPTraceReceiver) Export(_ context.Context, req *coltracepb.ExportTraceServiceRequest) (*coltracepb.ExportTraceServiceResponse, error) { + rec.mu.Lock() + defer rec.mu.Unlock() + rec.receivedRequests = append(rec.receivedRequests, req) + return &coltracepb.ExportTraceServiceResponse{}, nil +} + +// Attach implements testenv.Modifier. +func (rec *OTLPTraceReceiver) Attach(ctx context.Context) { + env := testenv.EnvFromContext(ctx) + + // NB: we cannot install tracing middleware into the receiver server, since + // it will cause a feedback loop of spans created when exporting other spans + + grpcUpstream := upstreams.GRPC(nil, + upstreams.WithDisplayName("OTLP GRPC Receiver"), + upstreams.WithDelayedShutdown(), + upstreams.WithNoClientTracing(), + upstreams.WithNoServerTracing(), + ) + httpUpstream := upstreams.HTTP(nil, + upstreams.WithDisplayName("OTLP HTTP Receiver"), + upstreams.WithDelayedShutdown(), + upstreams.WithNoClientTracing(), + upstreams.WithNoServerTracing(), + ) + + coltracepb.RegisterTraceServiceServer(grpcUpstream, rec) + httpUpstream.Handle("/v1/traces", rec.handleV1Traces) + env.AddUpstream(grpcUpstream) + env.AddUpstream(httpUpstream) + rec.grpcUpstream.Resolve(grpcUpstream) + rec.httpUpstream.Resolve(httpUpstream) +} + +// Modify implements testenv.Modifier. +func (rec *OTLPTraceReceiver) Modify(*config.Config) {} + +func (rec *OTLPTraceReceiver) handleV1Traces(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Content-Type") != "application/x-protobuf" { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte("invalid content type")) + return + } + reader := r.Body + if r.Header.Get("Content-Encoding") == "gzip" { + var err error + reader, err = gzip.NewReader(r.Body) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(err.Error())) + return + } + } + data, err := io.ReadAll(reader) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(err.Error())) + return + } + var req coltracepb.ExportTraceServiceRequest + if err := proto.Unmarshal(data, &req); err != nil { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(err.Error())) + return + } + resp, err := rec.Export(context.TODO(), &req) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + respData, err := proto.Marshal(resp) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + w.Header().Set("Content-Type", "application/x-protobuf") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(respData) +} + +func (rec *OTLPTraceReceiver) ReceivedRequests() []*coltracepb.ExportTraceServiceRequest { + rec.mu.Lock() + defer rec.mu.Unlock() + return rec.receivedRequests +} + +func (rec *OTLPTraceReceiver) PeekResourceSpans() []*tracev1.ResourceSpans { + rec.mu.Lock() + defer rec.mu.Unlock() + + return rec.peekResourceSpansLocked() +} + +func (rec *OTLPTraceReceiver) peekResourceSpansLocked() []*tracev1.ResourceSpans { + return tracetest.FlattenExportRequests(rec.receivedRequests) +} + +func (rec *OTLPTraceReceiver) FlushResourceSpans() []*tracev1.ResourceSpans { + rec.mu.Lock() + defer rec.mu.Unlock() + spans := rec.peekResourceSpansLocked() + rec.receivedRequests = nil + return spans +} + +// GRPCEndpointURL returns a url suitable for use with the environment variable +// $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT or with [otlptracegrpc.WithEndpointURL]. +func (rec *OTLPTraceReceiver) GRPCEndpointURL() values.Value[string] { + return values.Chain(rec.grpcUpstream, upstreams.GRPCUpstream.Port, func(port int) string { + return fmt.Sprintf("http://127.0.0.1:%d", port) + }) +} + +// GRPCEndpointURL returns a url suitable for use with the environment variable +// $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT or with [otlptracehttp.WithEndpointURL]. +func (rec *OTLPTraceReceiver) HTTPEndpointURL() values.Value[string] { + return values.Chain(rec.httpUpstream, upstreams.HTTPUpstream.Port, func(port int) string { + return fmt.Sprintf("http://127.0.0.1:%d/v1/traces", port) + }) +} + +func (rec *OTLPTraceReceiver) NewGRPCClient(opts ...otlptracegrpc.Option) otlptrace.Client { + return &deferredClient{ + client: values.Bind(rec.grpcUpstream, func(up upstreams.GRPCUpstream) otlptrace.Client { + return otlptracegrpc.NewClient(append(opts, + otlptracegrpc.WithGRPCConn(up.DirectConnect()), + otlptracegrpc.WithTimeout(1*time.Minute), + )...) + }), + } +} + +func (rec *OTLPTraceReceiver) NewHTTPClient(opts ...otlptracehttp.Option) otlptrace.Client { + return &deferredClient{ + client: values.Chain(rec.httpUpstream, upstreams.HTTPUpstream.Port, func(port int) otlptrace.Client { + return otlptracehttp.NewClient(append(opts, + otlptracehttp.WithEndpointURL(fmt.Sprintf("http://127.0.0.1:%d/v1/traces", port)), + otlptracehttp.WithTimeout(1*time.Minute), + )...) + }), + } +} + +type deferredClient struct { + client values.Value[otlptrace.Client] +} + +// Start implements otlptrace.Client. +func (o *deferredClient) Start(ctx context.Context) error { + return o.client.Value().Start(ctx) +} + +// Stop implements otlptrace.Client. +func (o *deferredClient) Stop(ctx context.Context) error { + return o.client.Value().Stop(ctx) +} + +// UploadTraces implements otlptrace.Client. +func (o *deferredClient) UploadTraces(ctx context.Context, protoSpans []*tracev1.ResourceSpans) error { + return o.client.Value().UploadTraces(ctx, protoSpans) +} + +var _ otlptrace.Client = (*deferredClient)(nil) diff --git a/internal/testenv/selftests/tracing_test.go b/internal/testenv/selftests/tracing_test.go new file mode 100644 index 000000000..0a799d872 --- /dev/null +++ b/internal/testenv/selftests/tracing_test.go @@ -0,0 +1,402 @@ +package selftests_test + +import ( + "context" + "fmt" + "io" + "maps" + "net/http" + "os" + "slices" + "sync/atomic" + "testing" + "time" + + "github.com/pomerium/pomerium/config" + "github.com/pomerium/pomerium/internal/telemetry/trace" + "github.com/pomerium/pomerium/internal/testenv" + "github.com/pomerium/pomerium/internal/testenv/scenarios" + "github.com/pomerium/pomerium/internal/testenv/snippets" + "github.com/pomerium/pomerium/internal/testenv/upstreams" + . "github.com/pomerium/pomerium/internal/testutil/tracetest" //nolint:revive + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.17.0" + oteltrace "go.opentelemetry.io/otel/trace" +) + +func otlpTraceReceiverOrFromEnv(t *testing.T) (modifier testenv.Modifier, newRemoteClient func() otlptrace.Client, getResults func() *TraceResults) { + t.Setenv("OTEL_TRACES_EXPORTER", "otlp") + tracesEndpoint := os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") + if tracesEndpoint == "" { + tracesEndpoint = os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") + if tracesEndpoint == "" { + srv := scenarios.NewOTLPTraceReceiver() + return srv, + func() otlptrace.Client { + return srv.NewGRPCClient() + }, + func() *TraceResults { + return NewTraceResults(srv.FlushResourceSpans()) + } + } + } + return testenv.NoopModifier(), trace.NewRemoteClientFromEnv, nil +} + +var allServices = []string{ + "Test Environment", + "Authorize", + "Authenticate", + "Control Plane", + "Data Broker", + "Proxy", + "Upstream", + "IDP", + "HTTP Client", + "Envoy", +} + +func TestOTLPTracing(t *testing.T) { + modifier, newRemoteClient, getResults := otlpTraceReceiverOrFromEnv(t) + env := testenv.New(t, testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags), testenv.WithTraceClient(newRemoteClient())) + env.Add(modifier) + + up := upstreams.HTTP(nil, upstreams.WithDisplayName("Upstream")) + up.Handle("/foo", func(w http.ResponseWriter, _ *http.Request) { + w.Write([]byte("OK")) + }) + env.Add(scenarios.NewIDP([]*scenarios.User{ + { + Email: "foo@example.com", + FirstName: "Firstname", + LastName: "Lastname", + }, + })) + + route := up.Route(). + From(env.SubdomainURL("foo")). + PPL(`{"allow":{"and":["email":{"is":"foo@example.com"}]}}`) + + env.AddUpstream(up) + env.Start() + snippets.WaitStartupComplete(env) + + ctx, span := env.Tracer().Start(env.Context(), "Authenticate", oteltrace.WithNewRoot()) + resp, err := up.Get(route, upstreams.AuthenticateAs("foo@example.com"), upstreams.Path("/foo"), upstreams.Context(ctx)) + span.End() + require.NoError(t, err) + body, err := io.ReadAll(resp.Body) + assert.NoError(t, err) + assert.NoError(t, resp.Body.Close()) + assert.Equal(t, resp.StatusCode, 200) + assert.Equal(t, "OK", string(body)) + + env.Stop() + + if getResults != nil { + results := getResults() + var ( + testEnvironmentLocalTest = fmt.Sprintf("Test Environment: %s", t.Name()) + testEnvironmentAuthenticate = "Test Environment: Authenticate" + authenticateOAuth2Client = "Authenticate: OAuth2 Client: GET /.well-known/jwks.json" + idpServerGetUserinfo = "IDP: Server: GET /oidc/userinfo" + idpServerPostToken = "IDP: Server: POST /oidc/token" + controlPlaneEnvoyAccessLogs = "Control Plane: envoy.service.accesslog.v3.AccessLogService/StreamAccessLogs" + controlPlaneEnvoyDiscovery = "Control Plane: envoy.service.discovery.v3.AggregatedDiscoveryService/DeltaAggregatedResources" + controlPlaneExport = "Control Plane: opentelemetry.proto.collector.trace.v1.TraceService/Export" + ) + + results.MatchTraces(t, + MatchOptions{ + Exact: true, + CheckDetachedSpans: true, + }, + Match{Name: testEnvironmentLocalTest, TraceCount: 1, Services: []string{"Test Environment", "Control Plane", "Data Broker"}}, + Match{Name: testEnvironmentAuthenticate, TraceCount: 1, Services: allServices}, + Match{Name: authenticateOAuth2Client, TraceCount: Greater(0)}, + Match{Name: idpServerGetUserinfo, TraceCount: EqualToMatch(authenticateOAuth2Client)}, + Match{Name: idpServerPostToken, TraceCount: EqualToMatch(authenticateOAuth2Client)}, + Match{Name: controlPlaneEnvoyDiscovery, TraceCount: 1}, + Match{Name: controlPlaneExport, TraceCount: Greater(0)}, + Match{Name: controlPlaneEnvoyAccessLogs, TraceCount: Any{}}, + ) + } +} + +func TestOTLPTracing_TraceCorrelation(t *testing.T) { + modifier, newRemoteClient, getResults := otlpTraceReceiverOrFromEnv(t) + env := testenv.New(t, testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags), testenv.WithTraceClient(newRemoteClient())) + env.Add(modifier) + + up := upstreams.HTTP(nil, upstreams.WithDisplayName("Upstream"), upstreams.WithNoClientTracing()) + up.Handle("/foo", func(w http.ResponseWriter, _ *http.Request) { + w.Write([]byte("OK")) + }) + env.Add(scenarios.NewIDP([]*scenarios.User{ + { + Email: "foo@example.com", + FirstName: "Firstname", + LastName: "Lastname", + }, + })) + + route := up.Route(). + From(env.SubdomainURL("foo")). + PPL(`{"allow":{"and":["email":{"is":"foo@example.com"}]}}`) + + env.AddUpstream(up) + env.Start() + snippets.WaitStartupComplete(env) + + resp, err := up.Get(route, upstreams.AuthenticateAs("foo@example.com"), upstreams.Path("/foo"), upstreams.Context(context.Background())) + require.NoError(t, err) + body, err := io.ReadAll(resp.Body) + assert.NoError(t, err) + assert.NoError(t, resp.Body.Close()) + assert.Equal(t, resp.StatusCode, 200) + assert.Equal(t, "OK", string(body)) + + env.Stop() + if getResults != nil { + results := getResults() + traces := results.GetTraces() + // one unauthenticated (ends in /.pomerium/callback redirect), one authenticated + assert.Len(t, traces.ByName[fmt.Sprintf("Envoy: ingress: GET foo.localhost.pomerium.io:%d/foo", env.Ports().ProxyHTTP.Value())].WithoutErrors(), 2) + } +} + +type SamplingTestSuite struct { + suite.Suite + env testenv.Environment + getResults func() *TraceResults + route testenv.Route + upstream upstreams.HTTPUpstream + + sampled atomic.Int32 + notSampled atomic.Int32 +} + +func (s *SamplingTestSuite) SetupTest() { + modifier, newRemoteClient, getResults := otlpTraceReceiverOrFromEnv(s.T()) + s.getResults = getResults + s.env = testenv.New(s.T(), + testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags|trace.EnvoyFlushEverySpan), + testenv.WithTraceClient(newRemoteClient()), + ) + s.env.Add(modifier) + + s.sampled.Store(0) + s.notSampled.Store(0) + + s.env.Add(testenv.ModifierFunc(func(_ context.Context, cfg *config.Config) { + half := 0.5 + cfg.Options.TracingSampleRate = &half + })) + s.env.Add(scenarios.NewIDP([]*scenarios.User{ + { + Email: "foo@example.com", + FirstName: "Firstname", + LastName: "Lastname", + }, + })) + + s.upstream = upstreams.HTTP(nil, upstreams.WithNoClientTracing(), upstreams.WithDisplayName("Upstream")) + s.upstream.Handle("/", s.handleRequest) + + s.route = s.upstream.Route(). + From(s.env.SubdomainURL("sampling-50pct")). + PPL(`{"allow":{"and":["email":{"is":"foo@example.com"}]}}`) + + s.env.AddUpstream(s.upstream) + s.env.Start() + snippets.WaitStartupComplete(s.env) +} + +func (s *SamplingTestSuite) TearDownTest() { + s.env.Stop() +} + +func (s *SamplingTestSuite) handleRequest(w http.ResponseWriter, req *http.Request) { + span := oteltrace.SpanFromContext(req.Context()) + flags := span.SpanContext().TraceFlags() + if flags.IsSampled() { + s.sampled.Add(1) + } else { + s.notSampled.Add(1) + } + w.Write([]byte("OK")) +} + +func (s *SamplingTestSuite) doRequest(ctx context.Context) { + resp, err := s.upstream.Get(s.route, upstreams.AuthenticateAs("foo@example.com"), upstreams.Path("/"), upstreams.Context(ctx)) + s.Require().NoError(err) + body, err := io.ReadAll(resp.Body) + s.Assert().NoError(err) + s.Assert().NoError(resp.Body.Close()) + s.Assert().Equal(resp.StatusCode, 200) + s.Assert().Equal("OK", string(body)) +} + +func (s *SamplingTestSuite) TestNoExternalTraceparent() { + for { + s.doRequest(context.Background()) + if s.sampled.Load() == 20 { + break + } + } + + s.Assert().NoError(trace.ForceFlush(s.env.Context())) + trace.WaitForSpans(s.env.Context(), 10*time.Second) + + s.Assert().Equal(int32(20), s.sampled.Load()) // 10 sampled + // Ideally we get ~50% sample rate, but CI will always be unlucky. + s.Assert().Greater(s.notSampled.Load(), int32(0)) + + if s.getResults != nil { + results := s.getResults() + traces := results.GetTraces() + s.Assert().Len(traces.ByParticipant["Upstream"], 20) + } +} + +func (s *SamplingTestSuite) TestExternalTraceparentAlwaysSample() { + tracer := trace.NewTracerProvider(s.env.Context(), "Always Sample", + sdktrace.WithSampler(sdktrace.AlwaysSample())).Tracer(trace.PomeriumCoreTracer) + for range 100 { + ctx, span := tracer.Start(context.Background(), "should sample") + s.doRequest(ctx) + span.End() + } + + s.Assert().NoError(trace.ForceFlush(s.env.Context())) + trace.WaitForSpans(s.env.Context(), 10*time.Second) + + // if the request already has a traceparent header, they will always be sampled + // regardless of the random sample rate we configured + s.Assert().Equal(int32(100), s.sampled.Load()) + s.Assert().Equal(int32(0), s.notSampled.Load()) + + if s.getResults != nil { + results := s.getResults() + traces := results.GetTraces() + s.Assert().Len(traces.ByParticipant["Envoy"], 100) + } +} + +func (s *SamplingTestSuite) TestExternalTraceparentNeverSample() { + tracer := trace.NewTracerProvider(s.env.Context(), "Never Sample", sdktrace.WithSampler(sdktrace.NeverSample())).Tracer(trace.PomeriumCoreTracer) + for range 100 { + ctx, span := tracer.Start(context.Background(), "should not sample") + s.doRequest(ctx) + span.End() + } + + s.Assert().NoError(trace.ForceFlush(s.env.Context())) + trace.WaitForSpans(s.env.Context(), 10*time.Second) + + s.Assert().Equal(int32(0), s.sampled.Load()) + s.Assert().Equal(int32(100), s.notSampled.Load()) + + if s.getResults != nil { + results := s.getResults() + traces := results.GetTraces() + if (len(traces.ByParticipant)) != 0 { + // whether or not these show up is timing dependent, but not important + possibleTraces := map[string]struct{}{ + "Test Environment: Start": {}, + "IDP: Server: POST /oidc/token": {}, + "IDP: Server: GET /oidc/userinfo": {}, + "Authenticate: OAuth2 Client: GET /.well-known/jwks.json": {}, + } + actual := slices.Collect(maps.Keys(traces.ByName)) + for _, name := range actual { + if _, ok := possibleTraces[name]; !ok { + s.Fail("unexpected trace: " + name) + } + } + } + } +} + +func TestSampling(t *testing.T) { + suite.Run(t, &SamplingTestSuite{}) +} + +func TestExternalSpans(t *testing.T) { + modifier, newRemoteClient, getResults := otlpTraceReceiverOrFromEnv(t) + + // set up external tracer + external := otlptrace.NewUnstarted(newRemoteClient()) + r, err := resource.Merge( + resource.Empty(), + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceName("External"), + ), + ) + require.NoError(t, err) + + externalTracerProvider := sdktrace.NewTracerProvider(sdktrace.WithBatcher(external), sdktrace.WithResource(r)) + + env := testenv.New(t, testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags|trace.EnvoyFlushEverySpan), testenv.WithTraceClient(newRemoteClient())) + env.Add(modifier) + + up := upstreams.HTTP(nil, upstreams.WithNoClientTracing()) + up.Handle("/foo", func(w http.ResponseWriter, _ *http.Request) { + w.Write([]byte("OK")) + }) + env.Add(scenarios.NewIDP([]*scenarios.User{ + { + Email: "foo@example.com", + FirstName: "Firstname", + LastName: "Lastname", + }, + })) + + route := up.Route(). + From(env.SubdomainURL("foo")). + PPL(`{"allow":{"and":["email":{"is":"foo@example.com"}]}}`) + + env.AddUpstream(up) + env.Start() + require.NoError(t, external.Start(env.Context())) + snippets.WaitStartupComplete(env) + + ctx, span := externalTracerProvider.Tracer("external").Start(context.Background(), "External Root", oteltrace.WithNewRoot()) + t.Logf("external span id: %s", span.SpanContext().SpanID().String()) + resp, err := up.Get(route, upstreams.AuthenticateAs("foo@example.com"), upstreams.Path("/foo"), upstreams.Context(ctx)) + span.End() + require.NoError(t, err) + body, err := io.ReadAll(resp.Body) + assert.NoError(t, err) + assert.NoError(t, resp.Body.Close()) + assert.Equal(t, resp.StatusCode, 200) + assert.Equal(t, "OK", string(body)) + + assert.NoError(t, externalTracerProvider.ForceFlush(context.Background())) + assert.NoError(t, externalTracerProvider.Shutdown(context.Background())) + assert.NoError(t, external.Shutdown(ctx)) + env.Stop() + + if getResults != nil { + results := getResults() + results.MatchTraces(t, MatchOptions{CheckDetachedSpans: true}, + Match{Name: "External: External Root", TraceCount: 1, Services: []string{ + "Authorize", + "Authenticate", + "Control Plane", + "Data Broker", + "Proxy", + "IDP", + "Envoy", + "External", + "HTTP Upstream", + }}, + ) + } +} diff --git a/internal/testenv/snippets/shutdown.go b/internal/testenv/snippets/shutdown.go new file mode 100644 index 000000000..879c37725 --- /dev/null +++ b/internal/testenv/snippets/shutdown.go @@ -0,0 +1,40 @@ +package snippets + +import ( + "context" + "errors" + "net/http" + + "github.com/pomerium/pomerium/internal/log" + "github.com/pomerium/pomerium/internal/testenv" +) + +func RunWithDelayedShutdown(ctx context.Context, serve func() error, stop func()) func() error { + env := testenv.EnvFromContext(ctx) + + stopping := make(chan struct{}) + serveExited := make(chan error, 1) + env.OnStateChanged(testenv.Stopping, func() { + close(stopping) + }) + cancel := env.OnStateChanged(testenv.Stopped, func() { + stop() + if err := <-serveExited; err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Ctx(ctx).Err(err).Msg("error stopping server") + } + }) + go func() { + serveExited <- serve() + close(serveExited) + }() + + return func() error { + select { + case <-stopping: + return nil + case err := <-serveExited: + cancel() + return err + } + } +} diff --git a/internal/testenv/snippets/wait.go b/internal/testenv/snippets/wait.go index 4342f9c42..eb44ea36c 100644 --- a/internal/testenv/snippets/wait.go +++ b/internal/testenv/snippets/wait.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/testenv" "github.com/pomerium/pomerium/pkg/grpcutil" "google.golang.org/grpc" @@ -12,6 +13,11 @@ import ( ) func WaitStartupComplete(env testenv.Environment, timeout ...time.Duration) time.Duration { + if env.GetState() == testenv.NotRunning { + panic("test bug: WaitStartupComplete called before starting the test environment") + } + _, span := trace.Continue(env.Context(), "snippets.WaitStartupComplete") + defer span.End() start := time.Now() recorder := env.NewLogRecorder() if len(timeout) == 0 { @@ -31,5 +37,6 @@ func WaitStartupComplete(env testenv.Environment, timeout ...time.Duration) time ) env.Require().NoError(err) env.Require().True(cc.WaitForStateChange(ctx, connectivity.Ready)) + cc.Close() return time.Since(start) } diff --git a/internal/testenv/temp.go b/internal/testenv/temp.go new file mode 100644 index 000000000..b00c38bea --- /dev/null +++ b/internal/testenv/temp.go @@ -0,0 +1,9 @@ +//go:build !darwin + +package testenv + +import "testing" + +func tempDir(t testing.TB) string { + return t.TempDir() +} diff --git a/internal/testenv/temp_darwin.go b/internal/testenv/temp_darwin.go new file mode 100644 index 000000000..ddee7c434 --- /dev/null +++ b/internal/testenv/temp_darwin.go @@ -0,0 +1,21 @@ +//go:build darwin + +package testenv + +import ( + "os" + "testing" +) + +// macos temp directory names are too long +// https://github.com/golang/go/issues/62614 +func tempDir(t testing.TB) string { + dir, err := os.MkdirTemp("", "test") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + os.RemoveAll(dir) + }) + return dir +} diff --git a/internal/testenv/types.go b/internal/testenv/types.go index 014114f22..8e863d3eb 100644 --- a/internal/testenv/types.go +++ b/internal/testenv/types.go @@ -84,7 +84,7 @@ type Aggregate struct { func (d *Aggregate) Add(mod Modifier) { if d.env != nil { - if d.env.(*environment).getState() == NotRunning { + if d.env.(*environment).GetState() == NotRunning { // If the test environment is running, adding to an aggregate is a no-op. // If the test environment has not been started yet, the aggregate is // being used like in the following example, which is incorrect: @@ -161,6 +161,20 @@ func ModifierFunc(fn func(ctx context.Context, cfg *config.Config)) Modifier { return &modifierFunc{fn: fn} } +func NoopModifier() Modifier { + return noopModifier{} +} + +type noopModifier struct{} + +// Attach implements Modifier. +func (n noopModifier) Attach(context.Context) {} + +// Modify implements Modifier. +func (n noopModifier) Modify(*config.Config) {} + +var _ Modifier = (noopModifier{}) + // Task represents a background task that can be added to an [Environment] to // have it run automatically on startup. // @@ -169,10 +183,17 @@ type Task interface { Run(ctx context.Context) error } -type TaskFunc func(ctx context.Context) error +func TaskFunc(fn func(ctx context.Context) error) Task { + return &taskFunc{fn: fn} +} -func (f TaskFunc) Run(ctx context.Context) error { - return f(ctx) +// this is a struct wrapper type instead of a typed func so it can be compared +type taskFunc struct { + fn func(ctx context.Context) error +} + +func (f *taskFunc) Run(ctx context.Context) error { + return f.fn(ctx) } // Upstream represents an upstream server. It is both a [Task] and a [Modifier] diff --git a/internal/testenv/upstreams/grpc.go b/internal/testenv/upstreams/grpc.go index fb22e08a7..90773d8c1 100644 --- a/internal/testenv/upstreams/grpc.go +++ b/internal/testenv/upstreams/grpc.go @@ -6,28 +6,36 @@ import ( "net" "strings" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/testenv" + "github.com/pomerium/pomerium/internal/testenv/snippets" "github.com/pomerium/pomerium/internal/testenv/values" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + oteltrace "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" ) -type Options struct { +type GRPCUpstreamOptions struct { + CommonUpstreamOptions serverOpts []grpc.ServerOption } -type Option func(*Options) - -func (o *Options) apply(opts ...Option) { - for _, op := range opts { - op(o) - } +type GRPCUpstreamOption interface { + applyGRPC(*GRPCUpstreamOptions) } -func ServerOpts(opt ...grpc.ServerOption) Option { - return func(o *Options) { +type GRPCUpstreamOptionFunc func(*GRPCUpstreamOptions) + +func (f GRPCUpstreamOptionFunc) applyGRPC(o *GRPCUpstreamOptions) { + f(o) +} + +func ServerOpts(opt ...grpc.ServerOption) GRPCUpstreamOption { + return GRPCUpstreamOptionFunc(func(o *GRPCUpstreamOptions) { o.serverOpts = append(o.serverOpts, opt...) - } + }) } // GRPCUpstream represents a GRPC server which can be used as the target for @@ -42,13 +50,18 @@ type GRPCUpstream interface { testenv.Upstream grpc.ServiceRegistrar Dial(r testenv.Route, dialOpts ...grpc.DialOption) *grpc.ClientConn + + // Dials the server directly instead of going through a Pomerium route. + DirectConnect(dialOpts ...grpc.DialOption) *grpc.ClientConn } type grpcUpstream struct { - Options + GRPCUpstreamOptions testenv.Aggregate - serverPort values.MutableValue[int] - creds credentials.TransportCredentials + serverPort values.MutableValue[int] + creds credentials.TransportCredentials + serverTracerProvider values.MutableValue[oteltrace.TracerProvider] + clientTracerProvider values.MutableValue[oteltrace.TracerProvider] services []service } @@ -59,13 +72,21 @@ var ( ) // GRPC creates a new GRPC upstream server. -func GRPC(creds credentials.TransportCredentials, opts ...Option) GRPCUpstream { - options := Options{} - options.apply(opts...) +func GRPC(creds credentials.TransportCredentials, opts ...GRPCUpstreamOption) GRPCUpstream { + options := GRPCUpstreamOptions{ + CommonUpstreamOptions: CommonUpstreamOptions{ + displayName: "GRPC Upstream", + }, + } + for _, op := range opts { + op.applyGRPC(&options) + } up := &grpcUpstream{ - Options: options, - creds: creds, - serverPort: values.Deferred[int](), + GRPCUpstreamOptions: options, + creds: creds, + serverPort: values.Deferred[int](), + serverTracerProvider: values.Deferred[oteltrace.TracerProvider](), + clientTracerProvider: values.Deferred[oteltrace.TracerProvider](), } up.RecordCaller() return up @@ -109,30 +130,65 @@ func (g *grpcUpstream) Run(ctx context.Context) error { return err } g.serverPort.Resolve(listener.Addr().(*net.TCPAddr).Port) - server := grpc.NewServer(append(g.serverOpts, grpc.Creds(g.creds))...) + if g.serverTracerProviderOverride != nil { + g.serverTracerProvider.Resolve(g.serverTracerProviderOverride) + } else { + g.serverTracerProvider.Resolve(trace.NewTracerProvider(ctx, g.displayName)) + } + if g.clientTracerProviderOverride != nil { + g.clientTracerProvider.Resolve(g.clientTracerProviderOverride) + } else { + g.clientTracerProvider.Resolve(trace.NewTracerProvider(ctx, "GRPC Client")) + } + server := grpc.NewServer(append(g.serverOpts, + grpc.Creds(g.creds), + grpc.StatsHandler(otelgrpc.NewServerHandler( + otelgrpc.WithTracerProvider(g.serverTracerProvider.Value()), + )), + )...) for _, s := range g.services { server.RegisterService(s.desc, s.impl) } + if g.delayShutdown { + return snippets.RunWithDelayedShutdown(ctx, + func() error { + return server.Serve(listener) + }, + server.GracefulStop, + )() + } errC := make(chan error, 1) go func() { errC <- server.Serve(listener) }() select { case <-ctx.Done(): - server.Stop() + server.GracefulStop() return context.Cause(ctx) case err := <-errC: return err } } -func (g *grpcUpstream) Dial(r testenv.Route, dialOpts ...grpc.DialOption) *grpc.ClientConn { - dialOpts = append(dialOpts, - grpc.WithContextDialer(testenv.GRPCContextDialer), +func (g *grpcUpstream) withDefaultDialOpts(extraDialOpts []grpc.DialOption) []grpc.DialOption { + return append(extraDialOpts, grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(g.Env().ServerCAs(), "")), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), + grpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(g.clientTracerProvider.Value()))), ) - cc, err := grpc.NewClient(strings.TrimPrefix(r.URL().Value(), "https://"), dialOpts...) +} + +func (g *grpcUpstream) Dial(r testenv.Route, dialOpts ...grpc.DialOption) *grpc.ClientConn { + cc, err := grpc.NewClient(strings.TrimPrefix(r.URL().Value(), "https://"), g.withDefaultDialOpts(dialOpts)...) + if err != nil { + panic(err) + } + return cc +} + +func (g *grpcUpstream) DirectConnect(dialOpts ...grpc.DialOption) *grpc.ClientConn { + cc, err := grpc.NewClient(fmt.Sprintf("127.0.0.1:%d", g.Port().Value()), + append(g.withDefaultDialOpts(dialOpts), grpc.WithTransportCredentials(insecure.NewCredentials()))...) if err != nil { panic(err) } diff --git a/internal/testenv/upstreams/http.go b/internal/testenv/upstreams/http.go index 2ec020108..fd8d53ffd 100644 --- a/internal/testenv/upstreams/http.go +++ b/internal/testenv/upstreams/http.go @@ -21,13 +21,21 @@ import ( "github.com/gorilla/mux" "github.com/pomerium/pomerium/integration/forms" "github.com/pomerium/pomerium/internal/retry" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/testenv" + "github.com/pomerium/pomerium/internal/testenv/snippets" "github.com/pomerium/pomerium/internal/testenv/values" - "github.com/pomerium/pomerium/pkg/telemetry/requestid" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" + oteltrace "go.opentelemetry.io/otel/trace" "google.golang.org/protobuf/proto" ) type RequestOptions struct { + requestCtx context.Context path string query url.Values headers map[string]string @@ -80,6 +88,12 @@ func Client(c *http.Client) RequestOption { } } +func Context(ctx context.Context) RequestOption { + return func(o *RequestOptions) { + o.requestCtx = ctx + } +} + func WithClientTrace(ct *httptrace.ClientTrace) RequestOption { return func(o *RequestOptions) { o.trace = ct @@ -111,6 +125,14 @@ func ClientCert[T interface { } } +type HTTPUpstreamOptions struct { + CommonUpstreamOptions +} + +type HTTPUpstreamOption interface { + applyHTTP(*HTTPUpstreamOptions) +} + // HTTPUpstream represents a HTTP server which can be used as the target for // one or more Pomerium routes in a test environment. // @@ -128,13 +150,17 @@ type HTTPUpstream interface { } type httpUpstream struct { + HTTPUpstreamOptions testenv.Aggregate serverPort values.MutableValue[int] tlsConfig values.Value[*tls.Config] clientCache sync.Map // map[testenv.Route]*http.Client - router *mux.Router + router *mux.Router + serverTracerProvider values.MutableValue[oteltrace.TracerProvider] + clientTracerProvider values.MutableValue[oteltrace.TracerProvider] + clientTracer values.Value[oteltrace.Tracer] } var ( @@ -143,12 +169,26 @@ var ( ) // HTTP creates a new HTTP upstream server. -func HTTP(tlsConfig values.Value[*tls.Config]) HTTPUpstream { - up := &httpUpstream{ - serverPort: values.Deferred[int](), - router: mux.NewRouter(), - tlsConfig: tlsConfig, +func HTTP(tlsConfig values.Value[*tls.Config], opts ...HTTPUpstreamOption) HTTPUpstream { + options := HTTPUpstreamOptions{ + CommonUpstreamOptions: CommonUpstreamOptions{ + displayName: "HTTP Upstream", + }, } + for _, op := range opts { + op.applyHTTP(&options) + } + up := &httpUpstream{ + HTTPUpstreamOptions: options, + serverPort: values.Deferred[int](), + router: mux.NewRouter(), + tlsConfig: tlsConfig, + serverTracerProvider: values.Deferred[oteltrace.TracerProvider](), + clientTracerProvider: values.Deferred[oteltrace.TracerProvider](), + } + up.clientTracer = values.Bind(up.clientTracerProvider, func(tp oteltrace.TracerProvider) oteltrace.Tracer { + return tp.Tracer(trace.PomeriumCoreTracer) + }) up.RecordCaller() return up } @@ -185,12 +225,34 @@ func (h *httpUpstream) Run(ctx context.Context) error { if h.tlsConfig != nil { tlsConfig = h.tlsConfig.Value() } + if h.serverTracerProviderOverride != nil { + h.serverTracerProvider.Resolve(h.serverTracerProviderOverride) + } else { + h.serverTracerProvider.Resolve(trace.NewTracerProvider(ctx, h.displayName)) + } + if h.clientTracerProviderOverride != nil { + h.clientTracerProvider.Resolve(h.clientTracerProviderOverride) + } else { + h.clientTracerProvider.Resolve(trace.NewTracerProvider(ctx, "HTTP Client")) + } + h.router.Use(trace.NewHTTPMiddleware(otelhttp.WithTracerProvider(h.serverTracerProvider.Value()))) + server := &http.Server{ Handler: h.router, TLSConfig: tlsConfig, - BaseContext: func(net.Listener) context.Context { - return ctx - }, + // BaseContext: func(net.Listener) context.Context { + // return ctx + // }, + } + if h.delayShutdown { + return snippets.RunWithDelayedShutdown(ctx, + func() error { + return server.Serve(listener) + }, + func() { + _ = server.Shutdown(context.Background()) + }, + )() } errC := make(chan error, 1) go func() { @@ -198,7 +260,7 @@ func (h *httpUpstream) Run(ctx context.Context) error { }() select { case <-ctx.Done(): - server.Close() + _ = server.Shutdown(context.Background()) return context.Cause(ctx) case err := <-errC: return err @@ -217,7 +279,9 @@ func (h *httpUpstream) Post(r testenv.Route, opts ...RequestOption) (*http.Respo // Do implements HTTPUpstream. func (h *httpUpstream) Do(method string, r testenv.Route, opts ...RequestOption) (*http.Response, error) { - options := RequestOptions{} + options := RequestOptions{ + requestCtx: h.Env().Context(), + } options.apply(opts...) u, err := url.Parse(r.URL().Value()) if err != nil { @@ -229,37 +293,15 @@ func (h *httpUpstream) Do(method string, r testenv.Route, opts ...RequestOption) RawQuery: options.query.Encode(), }) } - ctx := h.Env().Context() + ctx, span := h.clientTracer.Value().Start(options.requestCtx, "httpUpstream.Do", oteltrace.WithAttributes( + attribute.String("method", method), + attribute.String("url", u.String()), + )) if options.trace != nil { ctx = httptrace.WithClientTrace(ctx, options.trace) } - req, err := http.NewRequestWithContext(ctx, method, u.String(), nil) - if err != nil { - return nil, err - } - switch body := options.body.(type) { - case string: - req.Body = io.NopCloser(strings.NewReader(body)) - case []byte: - req.Body = io.NopCloser(bytes.NewReader(body)) - case io.Reader: - req.Body = io.NopCloser(body) - case proto.Message: - buf, err := proto.Marshal(body) - if err != nil { - return nil, err - } - req.Body = io.NopCloser(bytes.NewReader(buf)) - req.Header.Set("Content-Type", "application/octet-stream") - default: - buf, err := json.Marshal(body) - if err != nil { - panic(fmt.Sprintf("unsupported body type: %T", body)) - } - req.Body = io.NopCloser(bytes.NewReader(buf)) - req.Header.Set("Content-Type", "application/json") - case nil: - } + options.requestCtx = ctx + defer span.End() newClient := func() *http.Client { transport := http.DefaultTransport.(*http.Transport).Clone() @@ -269,7 +311,12 @@ func (h *httpUpstream) Do(method string, r testenv.Route, opts ...RequestOption) } transport.DialTLSContext = nil c := http.Client{ - Transport: requestid.NewRoundTripper(transport), + Transport: otelhttp.NewTransport(transport, + otelhttp.WithTracerProvider(h.clientTracerProvider.Value()), + otelhttp.WithSpanNameFormatter(func(_ string, r *http.Request) string { + return fmt.Sprintf("Client: %s %s", r.Method, r.URL.Path) + }), + ), } c.Jar, _ = cookiejar.New(&cookiejar.Options{}) return &c @@ -281,14 +328,45 @@ func (h *httpUpstream) Do(method string, r testenv.Route, opts ...RequestOption) var cachedClient any var ok bool if cachedClient, ok = h.clientCache.Load(r); !ok { + span.AddEvent("creating new http client") cachedClient, _ = h.clientCache.LoadOrStore(r, newClient()) + } else { + span.AddEvent("using cached http client") } client = cachedClient.(*http.Client) } var resp *http.Response + resendCount := 0 if err := retry.Retry(ctx, "http", func(ctx context.Context) error { - var err error + req, err := http.NewRequestWithContext(ctx, method, u.String(), nil) + if err != nil { + return retry.NewTerminalError(err) + } + switch body := options.body.(type) { + case string: + req.Body = io.NopCloser(strings.NewReader(body)) + case []byte: + req.Body = io.NopCloser(bytes.NewReader(body)) + case io.Reader: + req.Body = io.NopCloser(body) + case proto.Message: + buf, err := proto.Marshal(body) + if err != nil { + return retry.NewTerminalError(err) + } + req.Body = io.NopCloser(bytes.NewReader(buf)) + req.Header.Set("Content-Type", "application/octet-stream") + default: + buf, err := json.Marshal(body) + if err != nil { + panic(fmt.Sprintf("unsupported body type: %T", body)) + } + req.Body = io.NopCloser(bytes.NewReader(buf)) + req.Header.Set("Content-Type", "application/json") + case nil: + } + if options.authenticateAs != "" { resp, err = authenticateFlow(ctx, client, req, options.authenticateAs) //nolint:bodyclose } else { @@ -296,46 +374,70 @@ func (h *httpUpstream) Do(method string, r testenv.Route, opts ...RequestOption) } // retry on connection refused if err != nil { + span.RecordError(err) var opErr *net.OpError if errors.As(err, &opErr) && opErr.Op == "dial" && opErr.Err.Error() == "connect: connection refused" { + span.AddEvent("Retrying on dial error") return err } return retry.NewTerminalError(err) } - if resp.StatusCode == http.StatusInternalServerError { + if resp.StatusCode/100 == 5 { + resendCount++ + _, _ = io.ReadAll(resp.Body) + _ = resp.Body.Close() + span.SetAttributes(semconv.HTTPRequestResendCount(resendCount)) + span.AddEvent("Retrying on 5xx error", oteltrace.WithAttributes( + attribute.String("status", resp.Status), + )) return errors.New(http.StatusText(resp.StatusCode)) } + span.SetStatus(codes.Ok, "request completed successfully") return nil - }, retry.WithMaxInterval(100*time.Millisecond)); err != nil { + }, + retry.WithInitialInterval(1*time.Millisecond), + retry.WithMaxInterval(100*time.Millisecond), + ); err != nil { return nil, err } return resp, nil } func authenticateFlow(ctx context.Context, client *http.Client, req *http.Request, email string) (*http.Response, error) { + span := oteltrace.SpanFromContext(ctx) var res *http.Response originalHostname := req.URL.Hostname() res, err := client.Do(req) if err != nil { + span.RecordError(err) return nil, err } - location := res.Request.URL if location.Hostname() == originalHostname { // already authenticated - return res, err + span.SetStatus(codes.Ok, "already authenticated") + return res, nil } - defer res.Body.Close() fs := forms.Parse(res.Body) + _, _ = io.ReadAll(res.Body) + _ = res.Body.Close() if len(fs) > 0 { f := fs[0] f.Inputs["email"] = email f.Inputs["token_expiration"] = strconv.Itoa(int((time.Hour * 24).Seconds())) + span.AddEvent("submitting form", oteltrace.WithAttributes(attribute.String("location", location.String()))) formReq, err := f.NewRequestWithContext(ctx, location) if err != nil { + span.RecordError(err) return nil, err } - return client.Do(formReq) + resp, err := client.Do(formReq) + if err != nil { + span.RecordError(err) + return nil, err + } + span.SetStatus(codes.Ok, "form submitted successfully") + return resp, nil } return nil, fmt.Errorf("test bug: expected IDP login form") } diff --git a/internal/testenv/upstreams/options.go b/internal/testenv/upstreams/options.go new file mode 100644 index 000000000..c0bad32b5 --- /dev/null +++ b/internal/testenv/upstreams/options.go @@ -0,0 +1,64 @@ +package upstreams + +import ( + oteltrace "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" +) + +type CommonUpstreamOptions struct { + displayName string + clientTracerProviderOverride oteltrace.TracerProvider + serverTracerProviderOverride oteltrace.TracerProvider + delayShutdown bool +} + +type CommonUpstreamOption interface { + GRPCUpstreamOption + HTTPUpstreamOption +} + +type commonUpstreamOption func(o *CommonUpstreamOptions) + +// applyGRPC implements CommonUpstreamOption. +func (c commonUpstreamOption) applyGRPC(o *GRPCUpstreamOptions) { c(&o.CommonUpstreamOptions) } + +// applyHTTP implements CommonUpstreamOption. +func (c commonUpstreamOption) applyHTTP(o *HTTPUpstreamOptions) { c(&o.CommonUpstreamOptions) } + +func WithDisplayName(displayName string) CommonUpstreamOption { + return commonUpstreamOption(func(o *CommonUpstreamOptions) { + o.displayName = displayName + }) +} + +func WithNoClientTracing() CommonUpstreamOption { + return commonUpstreamOption(func(o *CommonUpstreamOptions) { + o.clientTracerProviderOverride = noop.NewTracerProvider() + }) +} + +func WithNoServerTracing() CommonUpstreamOption { + return commonUpstreamOption(func(o *CommonUpstreamOptions) { + o.serverTracerProviderOverride = noop.NewTracerProvider() + }) +} + +func WithClientTracerProvider(tp oteltrace.TracerProvider) CommonUpstreamOption { + return commonUpstreamOption(func(o *CommonUpstreamOptions) { + o.clientTracerProviderOverride = tp + }) +} + +func WithServerTracerProvider(tp oteltrace.TracerProvider) CommonUpstreamOption { + return commonUpstreamOption(func(o *CommonUpstreamOptions) { + o.serverTracerProviderOverride = tp + }) +} + +// WithDelayedShutdown keeps the server alive until the test environment has +// fully shut down, instead of stopping it during the shutdown sequence. +func WithDelayedShutdown() CommonUpstreamOption { + return commonUpstreamOption(func(o *CommonUpstreamOptions) { + o.delayShutdown = true + }) +} diff --git a/internal/testenv/values/value.go b/internal/testenv/values/value.go index 03d9ac8f2..39c25e782 100644 --- a/internal/testenv/values/value.go +++ b/internal/testenv/values/value.go @@ -69,7 +69,7 @@ func (p *value[T]) Resolve(value T) { p.ResolveFunc(func() T { return value }) } -// Bind creates a new [MutableValue] whose ultimate value depends on the result +// Bind creates a new [Value] whose ultimate value depends on the result // of another [Value] that may not yet be available. When Value() is called on // the result, it will cascade and trigger the full chain of initialization // functions necessary to produce the final value. @@ -118,3 +118,49 @@ func (s List[T]) Value() []T { } return values } + +// Chain is like [Bind], returning a Value[V] whose ultimate value depends on +// the result of a Value[U] which is obtained from the concrete T given in dt. +// It is intended to be used when T has a method returning a Value[U], and +// you want to access U, but don't need T. +// +// Example usage: +// +// type Foo interface { Number() Value[int] } +// var foo Foo +// values.Chain(foo, Foo.Number, func(number int) string { ... }) +func Chain[T any, U any, V any]( + dt Value[T], + callback1 func(value T) Value[U], + callback2 func(value U) V, +) Value[V] { + dv := Deferred[V]() + dv.ResolveFunc(func() V { + du := callback1(dt.Value()) + return callback2(du.Value()) + }) + return dv +} + +// Chain2 is like [Chain], but with two levels of indirection. +// +// Example usage: +// +// type Foo interface { Bar() Value[Bar] } +// type Bar interface { Number() Value[int] } +// var foo Foo +// values.Chain(foo, Foo.Bar, bar.Number, func(number int) string { ... }) +func Chain2[T any, U any, V any, W any]( + dt Value[T], + callback1 func(value T) Value[U], + callback2 func(value U) Value[V], + callback3 func(value V) W, +) Value[W] { + dw := Deferred[W]() + dw.ResolveFunc(func() W { + du := callback1(dt.Value()) + dv := callback2(du.Value()) + return callback3(dv.Value()) + }) + return dw +} diff --git a/internal/testutil/tracetest/buffer.go b/internal/testutil/tracetest/buffer.go new file mode 100644 index 000000000..a39cdd9d8 --- /dev/null +++ b/internal/testutil/tracetest/buffer.go @@ -0,0 +1,193 @@ +package tracetest + +import ( + "cmp" + "encoding/base64" + "maps" + "slices" + "sync" + + "github.com/pomerium/pomerium/internal/hashutil" + commonv1 "go.opentelemetry.io/proto/otlp/common/v1" + resourcev1 "go.opentelemetry.io/proto/otlp/resource/v1" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" + "google.golang.org/protobuf/proto" +) + +type ScopeBuffer struct { + scope *ScopeInfo + spans []*tracev1.Span +} + +func (sb *ScopeBuffer) Insert(spans ...*tracev1.Span) { + sb.spans = append(sb.spans, spans...) +} + +func NewScopeBuffer(scope *ScopeInfo) *ScopeBuffer { + return &ScopeBuffer{ + scope: scope, + } +} + +type ResourceBuffer struct { + resource *ResourceInfo + spansByScope map[string]*ScopeBuffer +} + +func NewResourceBuffer(resource *ResourceInfo) *ResourceBuffer { + return &ResourceBuffer{ + resource: resource, + spansByScope: make(map[string]*ScopeBuffer), + } +} + +func (rb *ResourceBuffer) Insert(scope *ScopeInfo, span *tracev1.Span) { + var spans *ScopeBuffer + if sp, ok := rb.spansByScope[scope.ID()]; ok { + spans = sp + } else { + spans = NewScopeBuffer(scope) + rb.spansByScope[scope.ID()] = spans + } + spans.Insert(span) +} + +func (rb *ResourceBuffer) Flush() []*tracev1.ScopeSpans { + out := make([]*tracev1.ScopeSpans, 0, len(rb.spansByScope)) + for _, key := range slices.Sorted(maps.Keys(rb.spansByScope)) { + spans := rb.spansByScope[key] + slices.SortStableFunc(spans.spans, func(a, b *tracev1.Span) int { + return cmp.Compare(a.StartTimeUnixNano, b.StartTimeUnixNano) + }) + scopeSpans := &tracev1.ScopeSpans{ + Scope: spans.scope.Scope, + SchemaUrl: spans.scope.Schema, + Spans: spans.spans, + } + out = append(out, scopeSpans) + } + clear(rb.spansByScope) + return out +} + +func (rb *ResourceBuffer) Merge(other *ResourceBuffer) { + for scope, otherSpans := range other.spansByScope { + if ourSpans, ok := rb.spansByScope[scope]; !ok { + rb.spansByScope[scope] = otherSpans + } else { + ourSpans.Insert(otherSpans.spans...) + } + } + clear(other.spansByScope) +} + +type Buffer struct { + scopesByResourceID map[string]*ResourceBuffer +} + +func NewBuffer() *Buffer { + return &Buffer{ + scopesByResourceID: make(map[string]*ResourceBuffer), + } +} + +func (b *Buffer) Insert(resource *ResourceInfo, scope *ScopeInfo, span *tracev1.Span) { + resourceEq := resource.ID() + var scopes *ResourceBuffer + if sc, ok := b.scopesByResourceID[resourceEq]; ok { + scopes = sc + } else { + scopes = NewResourceBuffer(resource) + b.scopesByResourceID[resourceEq] = scopes + } + scopes.Insert(scope, span) +} + +func (b *Buffer) Flush() []*tracev1.ResourceSpans { + out := make([]*tracev1.ResourceSpans, 0, len(b.scopesByResourceID)) + for _, key := range slices.Sorted(maps.Keys(b.scopesByResourceID)) { + scopes := b.scopesByResourceID[key] + resourceSpans := &tracev1.ResourceSpans{ + Resource: scopes.resource.Resource, + ScopeSpans: scopes.Flush(), + SchemaUrl: scopes.resource.Schema, + } + out = append(out, resourceSpans) + } + clear(b.scopesByResourceID) + return out +} + +func (b *Buffer) Merge(other *Buffer) { + if b != nil { + for k, otherV := range other.scopesByResourceID { + if v, ok := b.scopesByResourceID[k]; !ok { + b.scopesByResourceID[k] = otherV + } else { + v.Merge(otherV) + } + } + } + clear(other.scopesByResourceID) +} + +func (b *Buffer) IsEmpty() bool { + return len(b.scopesByResourceID) == 0 +} + +type ResourceInfo struct { + Resource *resourcev1.Resource + Schema string + ID func() string +} + +func NewResourceInfo(resource *resourcev1.Resource, resourceSchema string) *ResourceInfo { + ri := &ResourceInfo{ + Resource: resource, + Schema: resourceSchema, + } + ri.ID = sync.OnceValue(ri.computeID) + return ri +} + +func (ri *ResourceInfo) computeID() string { + hash := hashutil.NewDigest() + tmp := resourcev1.Resource{ + Attributes: ri.Resource.Attributes, + } + bytes, _ := proto.Marshal(&tmp) + hash.WriteStringWithLen(ri.Schema) + hash.WriteWithLen(bytes) + return base64.StdEncoding.EncodeToString(hash.Sum(nil)) +} + +type ScopeInfo struct { + Scope *commonv1.InstrumentationScope + Schema string + ID func() string +} + +func NewScopeInfo(scope *commonv1.InstrumentationScope, scopeSchema string) *ScopeInfo { + si := &ScopeInfo{ + Scope: scope, + Schema: scopeSchema, + } + si.ID = sync.OnceValue(si.computeID) + return si +} + +func (si *ScopeInfo) computeID() string { + if si.Scope == nil { + return "(unknown)" + } + hash := hashutil.NewDigest() + tmp := commonv1.InstrumentationScope{ + Name: si.Scope.Name, + Version: si.Scope.Version, + Attributes: si.Scope.Attributes, + } + bytes, _ := proto.Marshal(&tmp) + hash.WriteStringWithLen(si.Schema) + hash.WriteWithLen(bytes) + return base64.StdEncoding.EncodeToString(hash.Sum(nil)) +} diff --git a/internal/testutil/tracetest/buffer_test.go b/internal/testutil/tracetest/buffer_test.go new file mode 100644 index 000000000..5bb9146e9 --- /dev/null +++ b/internal/testutil/tracetest/buffer_test.go @@ -0,0 +1,118 @@ +package tracetest + +import ( + "testing" + + "github.com/pomerium/pomerium/internal/testutil" + "github.com/stretchr/testify/assert" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" +) + +func TestBuffer(t *testing.T) { + t.Parallel() + + // start time determines sort order of spans within a resource+scope group + s := []*tracev1.Span{ + {TraceId: Trace(1).B(), SpanId: Span(1).B(), StartTimeUnixNano: 1}, + {TraceId: Trace(1).B(), SpanId: Span(2).B(), StartTimeUnixNano: 2}, + {TraceId: Trace(2).B(), SpanId: Span(3).B(), StartTimeUnixNano: 3}, + {TraceId: Trace(2).B(), SpanId: Span(4).B(), StartTimeUnixNano: 4}, + {TraceId: Trace(1).B(), SpanId: Span(5).B(), StartTimeUnixNano: 5}, + {TraceId: Trace(1).B(), SpanId: Span(6).B(), StartTimeUnixNano: 6}, + {TraceId: Trace(2).B(), SpanId: Span(7).B(), StartTimeUnixNano: 7}, + {TraceId: Trace(2).B(), SpanId: Span(8).B(), StartTimeUnixNano: 8}, + {TraceId: Trace(1).B(), SpanId: Span(9).B(), StartTimeUnixNano: 9}, + {TraceId: Trace(1).B(), SpanId: Span(10).B(), StartTimeUnixNano: 10}, + {TraceId: Trace(2).B(), SpanId: Span(11).B(), StartTimeUnixNano: 11}, + {TraceId: Trace(2).B(), SpanId: Span(12).B(), StartTimeUnixNano: 12}, + {TraceId: Trace(1).B(), SpanId: Span(13).B(), StartTimeUnixNano: 13}, + {TraceId: Trace(1).B(), SpanId: Span(14).B(), StartTimeUnixNano: 14}, + {TraceId: Trace(2).B(), SpanId: Span(15).B(), StartTimeUnixNano: 15}, + {TraceId: Trace(2).B(), SpanId: Span(16).B(), StartTimeUnixNano: 16}, + } + + newTestBuffer := func() *Buffer { + b := NewBuffer() + b.Insert(Resource(1).Make(), Scope(1).Make(), s[0]) + b.Insert(Resource(1).Make(), Scope(1).Make(), s[1]) + b.Insert(Resource(1).Make(), Scope(1).Make(), s[2]) + b.Insert(Resource(1).Make(), Scope(1).Make(), s[3]) + b.Insert(Resource(1).Make(), Scope(2).Make(), s[4]) + b.Insert(Resource(1).Make(), Scope(2).Make(), s[5]) + b.Insert(Resource(1).Make(), Scope(2).Make(), s[6]) + b.Insert(Resource(1).Make(), Scope(2).Make(), s[7]) + b.Insert(Resource(2).Make(), Scope(1).Make(), s[8]) + b.Insert(Resource(2).Make(), Scope(1).Make(), s[9]) + b.Insert(Resource(2).Make(), Scope(1).Make(), s[10]) + b.Insert(Resource(2).Make(), Scope(1).Make(), s[11]) + b.Insert(Resource(2).Make(), Scope(2).Make(), s[12]) + b.Insert(Resource(2).Make(), Scope(2).Make(), s[13]) + b.Insert(Resource(2).Make(), Scope(2).Make(), s[14]) + b.Insert(Resource(2).Make(), Scope(2).Make(), s[15]) + return b + } + + newExpectedSpans := func() []*tracev1.ResourceSpans { + return []*tracev1.ResourceSpans{ + { + Resource: Resource(1).Make().Resource, + ScopeSpans: []*tracev1.ScopeSpans{ + { + Scope: Scope(1).Make().Scope, + Spans: []*tracev1.Span{s[0], s[1], s[2], s[3]}, + SchemaUrl: Schema(0).String(), + }, + { + Scope: Scope(2).Make().Scope, + Spans: []*tracev1.Span{s[4], s[5], s[6], s[7]}, + SchemaUrl: Schema(0).String(), + }, + }, + SchemaUrl: Schema(0).String(), + }, + { + Resource: Resource(2).Make().Resource, + ScopeSpans: []*tracev1.ScopeSpans{ + { + Scope: Scope(1).Make().Scope, + Spans: []*tracev1.Span{s[8], s[9], s[10], s[11]}, + SchemaUrl: Schema(0).String(), + }, + { + Scope: Scope(2).Make().Scope, + Spans: []*tracev1.Span{s[12], s[13], s[14], s[15]}, + SchemaUrl: Schema(0).String(), + }, + }, + SchemaUrl: Schema(0).String(), + }, + } + } + t.Run("Flush", func(t *testing.T) { + b := newTestBuffer() + actual := b.Flush() + assert.True(t, b.IsEmpty()) + testutil.AssertProtoEqual(t, newExpectedSpans(), actual) + }) + + t.Run("Default scope", func(t *testing.T) { + b := NewBuffer() + b.Insert(Resource(1).Make(Schema(2)), NewScopeInfo(nil, ""), s[0]) + b.Insert(Resource(1).Make(Schema(2)), NewScopeInfo(nil, ""), s[1]) + b.Insert(Resource(1).Make(Schema(2)), NewScopeInfo(nil, ""), s[2]) + actual := b.Flush() + testutil.AssertProtoEqual(t, []*tracev1.ResourceSpans{ + { + Resource: Resource(1).Make(Schema(2)).Resource, + ScopeSpans: []*tracev1.ScopeSpans{ + { + Scope: nil, + Spans: []*tracev1.Span{s[0], s[1], s[2]}, + SchemaUrl: "", + }, + }, + SchemaUrl: Schema(2).String(), + }, + }, actual) + }) +} diff --git a/internal/testutil/tracetest/mock_otlptrace/mock_client.go b/internal/testutil/tracetest/mock_otlptrace/mock_client.go new file mode 100644 index 000000000..d0f4c37be --- /dev/null +++ b/internal/testutil/tracetest/mock_otlptrace/mock_client.go @@ -0,0 +1,84 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: go.opentelemetry.io/otel/exporters/otlp/otlptrace (interfaces: Client) +// +// Generated by this command: +// +// mockgen go.opentelemetry.io/otel/exporters/otlp/otlptrace Client +// + +// Package mock_otlptrace is a generated GoMock package. +package mock_otlptrace + +import ( + context "context" + reflect "reflect" + + v1 "go.opentelemetry.io/proto/otlp/trace/v1" + gomock "go.uber.org/mock/gomock" +) + +// MockClient is a mock of Client interface. +type MockClient struct { + ctrl *gomock.Controller + recorder *MockClientMockRecorder + isgomock struct{} +} + +// MockClientMockRecorder is the mock recorder for MockClient. +type MockClientMockRecorder struct { + mock *MockClient +} + +// NewMockClient creates a new mock instance. +func NewMockClient(ctrl *gomock.Controller) *MockClient { + mock := &MockClient{ctrl: ctrl} + mock.recorder = &MockClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockClient) EXPECT() *MockClientMockRecorder { + return m.recorder +} + +// Start mocks base method. +func (m *MockClient) Start(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Start", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Start indicates an expected call of Start. +func (mr *MockClientMockRecorder) Start(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockClient)(nil).Start), ctx) +} + +// Stop mocks base method. +func (m *MockClient) Stop(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Stop", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Stop indicates an expected call of Stop. +func (mr *MockClientMockRecorder) Stop(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*MockClient)(nil).Stop), ctx) +} + +// UploadTraces mocks base method. +func (m *MockClient) UploadTraces(ctx context.Context, protoSpans []*v1.ResourceSpans) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UploadTraces", ctx, protoSpans) + ret0, _ := ret[0].(error) + return ret0 +} + +// UploadTraces indicates an expected call of UploadTraces. +func (mr *MockClientMockRecorder) UploadTraces(ctx, protoSpans any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UploadTraces", reflect.TypeOf((*MockClient)(nil).UploadTraces), ctx, protoSpans) +} diff --git a/internal/testutil/tracetest/tracing.go b/internal/testutil/tracetest/tracing.go new file mode 100644 index 000000000..06db43f62 --- /dev/null +++ b/internal/testutil/tracetest/tracing.go @@ -0,0 +1,629 @@ +package tracetest + +import ( + "cmp" + "context" + "encoding/binary" + "encoding/json" + "fmt" + "maps" + "runtime" + "slices" + "strings" + "sync" + "testing" + "time" + "unique" + + gocmp "github.com/google/go-cmp/cmp" + "github.com/pomerium/pomerium/internal/telemetry/trace" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + oteltrace "go.opentelemetry.io/otel/trace" + coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" + commonv1 "go.opentelemetry.io/proto/otlp/common/v1" + resourcev1 "go.opentelemetry.io/proto/otlp/resource/v1" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/testing/protocmp" +) + +type ( + Trace uint32 + Span uint32 + Scope uint32 + Schema uint32 + Resource uint32 +) + +func (n Trace) String() string { return fmt.Sprintf("Trace %d", n) } +func (n Span) String() string { return fmt.Sprintf("Span %d", n) } +func (n Scope) String() string { return fmt.Sprintf("Scope %d", n) } +func (n Schema) String() string { return fmt.Sprintf("Schema %d", n) } +func (n Resource) String() string { return fmt.Sprintf("Resource %d", n) } + +func (n Trace) ID() unique.Handle[oteltrace.TraceID] { + id, _ := trace.ToTraceID(n.B()) + return id +} + +func (n Trace) B() []byte { + var id oteltrace.TraceID + binary.BigEndian.PutUint32(id[12:], uint32(n)) + return id[:] +} + +func (n Span) ID() oteltrace.SpanID { + id, _ := trace.ToSpanID(n.B()) + return id +} + +func (n Span) B() []byte { + var id oteltrace.SpanID + binary.BigEndian.PutUint32(id[4:], uint32(n)) + return id[:] +} + +func (n Scope) Make(s ...Schema) *ScopeInfo { + if len(s) == 0 { + s = append(s, Schema(0)) + } + return NewScopeInfo(&commonv1.InstrumentationScope{ + Name: n.String(), + Version: "v1", + Attributes: []*commonv1.KeyValue{ + { + Key: "id", + Value: &commonv1.AnyValue{ + Value: &commonv1.AnyValue_IntValue{ + IntValue: int64(n), + }, + }, + }, + }, + }, s[0].String()) +} + +func (n Resource) Make(s ...Schema) *ResourceInfo { + if len(s) == 0 { + s = append(s, Schema(0)) + } + return NewResourceInfo(&resourcev1.Resource{ + Attributes: []*commonv1.KeyValue{ + { + Key: "name", + Value: &commonv1.AnyValue{ + Value: &commonv1.AnyValue_StringValue{ + StringValue: n.String(), + }, + }, + }, + { + Key: "id", + Value: &commonv1.AnyValue{ + Value: &commonv1.AnyValue_IntValue{ + IntValue: int64(n), + }, + }, + }, + }, + }, s[0].String()) +} + +func Traceparent(trace Trace, span Span, sampled bool) string { + sampledStr := "00" + if sampled { + sampledStr = "01" + } + return fmt.Sprintf("00-%s-%s-%s", trace.ID().Value(), span.ID(), sampledStr) +} + +type TraceResults struct { + resourceSpans []*tracev1.ResourceSpans + + GetResources func() []*resourcev1.Resource + GetTraces func() *Traces +} + +type Traces struct { + ByID map[unique.Handle[oteltrace.TraceID]]*TraceDetails + ByName map[string]TraceDetailsList + ByParticipant map[string]TraceDetailsList +} + +func (t *Traces) WithoutErrors() *Traces { + byID := make(map[unique.Handle[oteltrace.TraceID]]*TraceDetails, len(t.ByID)) + for k, v := range t.ByID { + if len(v.Errors) > 0 { + continue + } + byID[k] = v + } + byName := make(map[string]TraceDetailsList) + for k, v := range t.ByName { + filtered := v.WithoutErrors() + if len(filtered) == 0 { + continue + } + byName[k] = filtered + } + byParticipant := make(map[string]TraceDetailsList) + for k, v := range t.ByParticipant { + filtered := v.WithoutErrors() + if len(filtered) == 0 { + continue + } + byParticipant[k] = filtered + } + return &Traces{ + ByID: byID, + ByName: byName, + ByParticipant: byParticipant, + } +} + +type TraceDetails struct { + ID unique.Handle[oteltrace.TraceID] + Name string + Spans []*SpanDetails + Services []string + StartTime time.Time + EndTime time.Time + Duration time.Duration + Errors []int // indexes into Spans +} + +func (td *TraceDetails) Equal(other *TraceDetails) (bool, string) { + diffSpans := func(a, b []*SpanDetails) (bool, string) { + for i := range len(a) { + diff := gocmp.Diff(a[i], b[i], protocmp.Transform()) + if diff != "" { + return false, diff + } + } + return true, "" + } + if td.ID != other.ID { + return false, fmt.Sprintf("traces are trivially not equal: ID %s (actual) != %s (expected)", td.ID.Value(), other.ID.Value()) + } + if len(td.Spans) != len(other.Spans) { + return false, fmt.Sprintf("traces are trivially not equal: len(spans) %d (actual) != %d (expected)", len(td.Spans), len(other.Spans)) + } + if !td.StartTime.Equal(other.StartTime) { + return false, fmt.Sprintf("traces are trivially not equal: start time %s (actual) != %s (expected)", td.StartTime, other.StartTime) + } + if !td.EndTime.Equal(other.EndTime) { + return false, fmt.Sprintf("traces are trivially not equal: end time %s (actual) != %s (expected)", td.EndTime, other.EndTime) + } + return diffSpans(td.Spans, other.Spans) +} + +type TraceDetailsList []*TraceDetails + +func (list TraceDetailsList) WithoutExportRPCs() TraceDetailsList { + out := make(TraceDetailsList, 0, len(list)) + for _, td := range list { + if strings.Contains(td.Name, "opentelemetry.proto.collector.trace.v1.TraceService/Export") { + continue + } + out = append(out, td) + } + return out +} + +func (list TraceDetailsList) WithoutErrors() TraceDetailsList { + out := make(TraceDetailsList, 0, len(list)) + for _, td := range list { + if len(td.Errors) > 0 { + continue + } + out = append(out, td) + } + return out +} + +func (td *TraceDetails) SpanTree() *SpanTree { + nodesByID := map[oteltrace.SpanID]*SpanTreeNode{} + nodesByID[oteltrace.SpanID([8]byte{})] = &SpanTreeNode{} // root node + for _, span := range td.Spans { + spanID, _ := trace.ToSpanID(span.Raw.SpanId) + nodesByID[spanID] = &SpanTreeNode{ + Span: span, + } + } + detachedNodesByID := map[oteltrace.SpanID]*SpanTreeNode{} + for _, span := range td.Spans { + spanID, _ := trace.ToSpanID(span.Raw.SpanId) + parentSpanID, _ := trace.ToSpanID(span.Raw.ParentSpanId) + if _, ok := nodesByID[parentSpanID]; !ok { + detachedNodesByID[parentSpanID] = &SpanTreeNode{} + nodesByID[parentSpanID] = detachedNodesByID[parentSpanID] + } + nodesByID[spanID].Parent = nodesByID[parentSpanID] + nodesByID[parentSpanID].Children = append(nodesByID[parentSpanID].Children, nodesByID[spanID]) + } + for _, node := range nodesByID { + slices.SortFunc(node.Children, func(a, b *SpanTreeNode) int { + return cmp.Compare(a.Span.Raw.StartTimeUnixNano, b.Span.Raw.StartTimeUnixNano) + }) + } + return &SpanTree{ + Root: nodesByID[oteltrace.SpanID([8]byte{})], + DetachedParents: detachedNodesByID, + } +} + +type SpanDetails struct { + Raw *tracev1.Span + Resource *resourcev1.Resource + Scope *commonv1.InstrumentationScope + StartTime time.Time + EndTime time.Time + Duration time.Duration + Service string +} + +func NewTraceResults(resourceSpans []*tracev1.ResourceSpans) *TraceResults { + tr := &TraceResults{ + resourceSpans: resourceSpans, + } + tr.GetResources = sync.OnceValue(tr.computeResources) + tr.GetTraces = sync.OnceValue(tr.computeTraces) + return tr +} + +func (tr *TraceResults) computeResources() []*resourcev1.Resource { + resources := []*resourcev1.Resource{} + for _, res := range tr.resourceSpans { + resources = append(resources, res.Resource) + } + return resources +} + +func (tr *TraceResults) computeTraces() *Traces { + tracesByID := map[unique.Handle[oteltrace.TraceID]]*TraceDetails{} + for _, resSpan := range tr.resourceSpans { + resource := resSpan.Resource + for _, scopeSpans := range resSpan.ScopeSpans { + scope := scopeSpans.Scope + for _, span := range scopeSpans.Spans { + traceID, _ := trace.ToTraceID(span.TraceId) + var details *TraceDetails + if d, ok := tracesByID[traceID]; ok { + details = d + } else { + details = &TraceDetails{ + ID: traceID, + } + tracesByID[traceID] = details + } + svc := "" + for _, attr := range resource.Attributes { + if attr.Key == "service.name" { + svc = attr.Value.GetStringValue() + break + } + } + details.Spans = append(details.Spans, &SpanDetails{ + Raw: span, + Resource: resource, + Scope: scope, + StartTime: time.Unix(0, int64(span.StartTimeUnixNano)), + EndTime: time.Unix(0, int64(span.EndTimeUnixNano)), + Duration: time.Duration(span.EndTimeUnixNano - span.StartTimeUnixNano), + Service: svc, + }) + if span.Status != nil { + if span.Status.Code == tracev1.Status_STATUS_CODE_ERROR { + details.Errors = append(details.Errors, len(details.Spans)-1) + } + } + } + } + } + + tracesByName := map[string]TraceDetailsList{} + tracesByParticipant := map[string]TraceDetailsList{} + // sort spans by start time and compute durations + for _, td := range tracesByID { + slices.SortFunc(td.Spans, func(a, b *SpanDetails) int { + return cmp.Compare(a.Raw.StartTimeUnixNano, b.Raw.StartTimeUnixNano) + }) + startTime := td.Spans[0].Raw.StartTimeUnixNano + endTime := td.Spans[0].Raw.EndTimeUnixNano + serviceNames := map[string]struct{}{} + for _, span := range td.Spans { + startTime = min(startTime, span.Raw.StartTimeUnixNano) + endTime = max(endTime, span.Raw.EndTimeUnixNano) + if span.Service != "" { + serviceNames[span.Service] = struct{}{} + } + } + td.StartTime = time.Unix(0, int64(startTime)) + td.EndTime = time.Unix(0, int64(endTime)) + td.Duration = td.EndTime.Sub(td.StartTime) + td.Services = slices.Sorted(maps.Keys(serviceNames)) + td.Name = fmt.Sprintf("%s: %s", td.Spans[0].Service, td.Spans[0].Raw.Name) + tracesByName[td.Name] = append(tracesByName[td.Name], td) + for svc := range serviceNames { + tracesByParticipant[svc] = append(tracesByParticipant[svc], td) + } + } + + return &Traces{ + ByID: tracesByID, + ByName: tracesByName, + ByParticipant: tracesByParticipant, + } +} + +type SpanTree struct { + Root *SpanTreeNode + DetachedParents map[oteltrace.SpanID]*SpanTreeNode +} + +type SpanTreeNode struct { + Span *SpanDetails + Parent *SpanTreeNode + Children []*SpanTreeNode +} + +type Match struct { + Name string + TraceCount any + Services []string +} + +type ( + GreaterOrEqual int + Greater int + + // Any makes no assertions on the trace count. If the trace is not found, it + // doesn't count against the Exact match option. + Any struct{} + + // EqualToMatch asserts that the value is the same as the value of another + // match (by name) + EqualToMatch string + // GreaterThanMatch asserts that the value is greater than the value of + // another match (by name) + GreaterThanMatch string +) + +type MatchOptions struct { + // If true, asserts that there is exactly one [Match] entry per result + Exact bool + // If true, asserts that no traces contain detached spans + CheckDetachedSpans bool +} + +func (tr *TraceResults) MatchTraces(t testing.TB, opts MatchOptions, matches ...Match) { + t.Helper() + traces := tr.GetTraces() + matchArgsByName := map[string]Match{} + for i, m := range matches { + if m.Name != "" { + require.NotContains(t, matchArgsByName, m.Name, "duplicate name") + matchArgsByName[m.Name] = m + if traceDetails, ok := traces.ByName[m.Name]; ok { + switch tc := m.TraceCount.(type) { + case GreaterOrEqual: + assert.GreaterOrEqualf(t, len(traceDetails), int(tc), + "[match %d]: expected %q to have >=%d traces, but found %d", + i+1, m.Name, int(tc), len(traceDetails)) + case Greater: + assert.Greaterf(t, len(traceDetails), int(tc), + "[match %d]: expected %q to have >%d traces, but found %d", + i+1, m.Name, int(tc), len(traceDetails)) + case GreaterThanMatch: + assert.Greaterf(t, len(traceDetails), len(traces.ByName[string(tc)]), + "[match %d]: expected %q to have >%d traces (value of %s), but found %d", + i+1, m.Name, len(traces.ByName[string(tc)]), string(tc), len(traceDetails)) + case EqualToMatch: + assert.Equalf(t, len(traceDetails), len(traces.ByName[string(tc)]), + "[match %d]: expected %q to have %d traces (value of %s), but found %d", + i+1, m.Name, len(traces.ByName[string(tc)]), string(tc), len(traceDetails)) + case Any: + case int: + s := "s" + if tc == 1 { + s = "" + } + assert.Lenf(t, traceDetails, tc, + "[match %d]: expected %q to have %d trace%s, but found %d", + i+1, m.Name, tc, s, len(traceDetails)) + } + + if m.Services != nil { + for _, trace := range traceDetails { + assert.ElementsMatch(t, m.Services, trace.Services) + } + } + } else if _, ok := m.TraceCount.(Any); !ok { + t.Errorf("no traces with name %q found", m.Name) + } + } + } + if opts.CheckDetachedSpans { + for _, trace := range traces.ByID { + tree := trace.SpanTree() + if !assert.Empty(t, tree.DetachedParents) { + for spanID, node := range tree.DetachedParents { + t.Log("------------------------------------") + t.Logf("span id: %s", spanID) + if len(node.Children) != 0 { + t.Log("children:") + } + for _, c := range node.Children { + t.Log(protojson.Format(c.Span.Raw)) + } + t.Log("------------------------------------") + } + } + } + } + if opts.Exact { + expected := slices.Sorted(maps.Keys(matchArgsByName)) + actual := slices.Sorted(maps.Keys(traces.ByName)) + for name, match := range matchArgsByName { + if _, ok := traces.ByName[name]; !ok { + if _, ok := match.TraceCount.(Any); ok { + expected = slices.DeleteFunc(expected, func(s string) bool { return s == name }) + } + } + } + assert.Equal(t, expected, actual) + } +} + +func (tr *TraceResults) AssertEqual(t testing.TB, expectedResults *TraceResults, msgFmtAndArgs ...any) { + t.Helper() + actualTraces := tr.GetTraces() + expectedTraces := expectedResults.GetTraces() + for traceID, expected := range expectedTraces.ByID { + if actual, ok := actualTraces.ByID[traceID]; !ok { + if len(msgFmtAndArgs) > 0 { + t.Errorf("expected trace id %s not found (%s)", traceID.Value().String(), + fmt.Sprintf(msgFmtAndArgs[0].(string), msgFmtAndArgs[1:]...)) + } else { + t.Errorf("expected trace id %s not found", traceID.Value().String()) + } + } else { + if equal, diff := actual.Equal(expected); !equal { + if len(msgFmtAndArgs) > 0 { + t.Errorf("trace %s is not equal (%s):\n%s", traceID.Value().String(), + fmt.Sprintf(msgFmtAndArgs[0].(string), msgFmtAndArgs[1:]...), diff) + } else { + t.Errorf("trace %s is not equal:\n%s", traceID.Value().String(), diff) + } + } + } + } + for traceID := range actualTraces.ByID { + if _, ok := expectedTraces.ByID[traceID]; !ok { + if len(msgFmtAndArgs) > 0 { + t.Errorf("unexpected trace id %s found (%s)", traceID.Value().String(), + fmt.Sprintf(msgFmtAndArgs[0].(string), msgFmtAndArgs[1:]...)) + } else { + t.Errorf("unexpected trace id %s found", traceID.Value().String()) + } + } + } +} + +func FlattenResourceSpans(lists [][]*tracev1.ResourceSpans) []*tracev1.ResourceSpans { + res := NewBuffer() + for _, list := range lists { + for _, resource := range list { + resInfo := NewResourceInfo(resource.Resource, resource.SchemaUrl) + for _, scope := range resource.ScopeSpans { + scopeInfo := NewScopeInfo(scope.Scope, scope.SchemaUrl) + for _, span := range scope.Spans { + res.Insert(resInfo, scopeInfo, span) + } + } + } + } + return res.Flush() +} + +func FlattenExportRequests(reqs []*coltracepb.ExportTraceServiceRequest) []*tracev1.ResourceSpans { + lists := make([][]*tracev1.ResourceSpans, len(reqs)) + for i, req := range reqs { + lists[i] = req.ResourceSpans + } + return FlattenResourceSpans(lists) +} + +type EventRecording struct { + events []trace.DebugEvent + normalizedTo time.Time +} + +func LoadEventRecording(raw []byte) (*EventRecording, error) { + events := []trace.DebugEvent{} + if err := json.Unmarshal(raw, &events); err != nil { + return nil, err + } + for i := 1; i < len(events); i++ { + if events[i].Timestamp.Before(events[i-1].Timestamp) { + return nil, fmt.Errorf("invalid timestamps: event %d occurred before event %d", i, i-1) + } + } + return &EventRecording{ + events: events, + }, nil +} + +func (er *EventRecording) Normalize(startTime time.Time) { + if len(er.events) == 0 { + return + } + er.normalizedTo = startTime + offset := startTime.Sub(er.events[0].Timestamp) + for i, ev := range er.events { + er.events[i].Timestamp = ev.Timestamp.Add(offset) + for _, resSpan := range ev.Request.ResourceSpans { + for _, scopeSpans := range resSpan.ScopeSpans { + for _, span := range scopeSpans.Spans { + span.StartTimeUnixNano += uint64(offset) + span.EndTimeUnixNano += uint64(offset) + for _, event := range span.Events { + event.TimeUnixNano += uint64(offset) + } + } + } + } + } +} + +func (er *EventRecording) NormalizedTo() time.Time { + return er.normalizedTo +} + +type EventCallbackFunc = func(ctx context.Context, req *coltracepb.ExportTraceServiceRequest) (*coltracepb.ExportTraceServiceResponse, error) + +func (er *EventRecording) Events() []trace.DebugEvent { + return er.events +} + +func (er *EventRecording) Clone() *EventRecording { + clonedEvents := make([]trace.DebugEvent, 0, len(er.events)) + for _, ev := range er.events { + clonedEvents = append(clonedEvents, trace.DebugEvent{ + Timestamp: ev.Timestamp, + Request: proto.Clone(ev.Request).(*coltracepb.ExportTraceServiceRequest), + }) + } + c := &EventRecording{ + events: clonedEvents, + normalizedTo: er.normalizedTo, + } + return c +} + +func (er *EventRecording) Replay(callback EventCallbackFunc) error { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + durations := make([]time.Duration, 0, len(er.events)-1) + for i := 1; i < len(er.events); i++ { + durations = append(durations, er.events[i].Timestamp.Sub(er.events[i-1].Timestamp)) + } + + var wg sync.WaitGroup + wg.Add(len(er.events)) + er.Normalize(time.Now()) + for i, ev := range er.events { + go func() { + callback(context.Background(), ev.Request) + wg.Done() + }() + if i < len(er.events)-1 { + time.Sleep(durations[i]) + } + } + wg.Wait() + return nil +} diff --git a/internal/urlutil/proxy.go b/internal/urlutil/proxy.go index 87ef13ebf..7386b8e5d 100644 --- a/internal/urlutil/proxy.go +++ b/internal/urlutil/proxy.go @@ -47,6 +47,14 @@ func GetCallbackURLForRedirectURI(r *http.Request, encodedSessionJWT, rawRedirec if r.FormValue(QueryIsProgrammatic) == "true" { callbackParams.Set(QueryIsProgrammatic, "true") } + // propagate trace context + if tracecontext := r.FormValue(QueryTraceparent); tracecontext != "" { + callbackParams.Set(QueryTraceparent, tracecontext) + } + if tracestate := r.FormValue(QueryTracestate); tracestate != "" { + callbackParams.Set(QueryTracestate, tracestate) + } + // add our encoded and encrypted route-session JWT to a query param callbackParams.Set(QuerySessionEncrypted, encodedSessionJWT) callbackParams.Set(QueryRedirectURI, redirectURI.String()) diff --git a/internal/urlutil/query_params.go b/internal/urlutil/query_params.go index ac5a127bc..3ddee3cc6 100644 --- a/internal/urlutil/query_params.go +++ b/internal/urlutil/query_params.go @@ -20,6 +20,8 @@ const ( QuerySessionState = "pomerium_session_state" QueryVersion = "pomerium_version" QueryRequestUUID = "pomerium_request_uuid" + QueryTraceparent = "pomerium_traceparent" + QueryTracestate = "pomerium_tracestate" ) // URL signature based query params used for verifying the authenticity of a URL. diff --git a/internal/zero/controller/usagereporter/usagereporter_test.go b/internal/zero/controller/usagereporter/usagereporter_test.go index 633227abd..ce25671b2 100644 --- a/internal/zero/controller/usagereporter/usagereporter_test.go +++ b/internal/zero/controller/usagereporter/usagereporter_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "go.opentelemetry.io/otel/trace" "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/protobuf/types/known/timestamppb" @@ -37,7 +38,7 @@ func TestUsageReporter(t *testing.T) { t.Cleanup(cancel) cc := testutil.NewGRPCServer(t, func(srv *grpc.Server) { - databrokerpb.RegisterDataBrokerServiceServer(srv, databroker.New(ctx)) + databrokerpb.RegisterDataBrokerServiceServer(srv, databroker.New(ctx, trace.NewNoopTracerProvider())) }) t.Cleanup(func() { cc.Close() }) diff --git a/pkg/cmd/pomerium/pomerium.go b/pkg/cmd/pomerium/pomerium.go index 1014d247e..00f92767c 100644 --- a/pkg/cmd/pomerium/pomerium.go +++ b/pkg/cmd/pomerium/pomerium.go @@ -3,11 +3,9 @@ package pomerium import ( "context" + "errors" "fmt" "net/http" - "os" - "os/signal" - "syscall" envoy_service_auth_v3 "github.com/envoyproxy/go-control-plane/envoy/service/auth/v3" "go.uber.org/automaxprocs/maxprocs" @@ -24,50 +22,80 @@ import ( "github.com/pomerium/pomerium/internal/events" "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/registry" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/version" derivecert_config "github.com/pomerium/pomerium/pkg/derivecert/config" "github.com/pomerium/pomerium/pkg/envoy" "github.com/pomerium/pomerium/pkg/envoy/files" "github.com/pomerium/pomerium/proxy" + oteltrace "go.opentelemetry.io/otel/trace" ) -type RunOptions struct { +type Options struct { fileMgr *filemgr.Manager envoyServerOptions []envoy.ServerOption databrokerServerOptions []databroker_service.Option } -type RunOption func(*RunOptions) +type Option func(*Options) -func (o *RunOptions) apply(opts ...RunOption) { +func (o *Options) apply(opts ...Option) { for _, op := range opts { op(o) } } -func WithOverrideFileManager(fileMgr *filemgr.Manager) RunOption { - return func(o *RunOptions) { +func WithOverrideFileManager(fileMgr *filemgr.Manager) Option { + return func(o *Options) { o.fileMgr = fileMgr } } -func WithEnvoyServerOptions(opts ...envoy.ServerOption) RunOption { - return func(o *RunOptions) { +func WithEnvoyServerOptions(opts ...envoy.ServerOption) Option { + return func(o *Options) { o.envoyServerOptions = append(o.envoyServerOptions, opts...) } } -func WithDataBrokerServerOptions(opts ...databroker_service.Option) RunOption { - return func(o *RunOptions) { +func WithDataBrokerServerOptions(opts ...databroker_service.Option) Option { + return func(o *Options) { o.databrokerServerOptions = append(o.databrokerServerOptions, opts...) } } // Run runs the main pomerium application. -func Run(ctx context.Context, src config.Source, opts ...RunOption) error { - options := RunOptions{} +func Run(ctx context.Context, src config.Source, opts ...Option) error { + p := New(opts...) + tracerProvider := trace.NewTracerProvider(ctx, "Pomerium") + + if err := p.Start(ctx, tracerProvider, src); err != nil { + return err + } + return p.Wait() +} + +var ErrShutdown = errors.New("Shutdown() called") + +type Pomerium struct { + Options + errGroup *errgroup.Group + + cancel context.CancelCauseFunc + envoyServer *envoy.Server +} + +func New(opts ...Option) *Pomerium { + options := Options{} options.apply(opts...) + return &Pomerium{ + Options: options, + } +} + +func (p *Pomerium) Start(ctx context.Context, tracerProvider oteltrace.TracerProvider, src config.Source) error { + updateTraceClient(ctx, src.GetConfig()) + ctx, p.cancel = context.WithCancelCause(ctx) _, _ = maxprocs.Set(maxprocs.Logger(func(s string, i ...any) { log.Ctx(ctx).Debug().Msgf(s, i...) })) evt := log.Ctx(ctx).Info(). @@ -82,9 +110,8 @@ func Run(ctx context.Context, src config.Source, opts ...RunOption) error { if err != nil { return err } - src = databroker.NewConfigSource(ctx, src, databroker.EnableConfigValidation(true)) - logMgr := config.NewLogManager(ctx, src) - defer logMgr.Close() + src = databroker.NewConfigSource(ctx, tracerProvider, src, databroker.EnableConfigValidation(true)) + _ = config.NewLogManager(ctx, src) // trigger changes when underlying files are changed src = config.NewFileWatcherSource(ctx, src) @@ -98,18 +125,16 @@ func Run(ctx context.Context, src config.Source, opts ...RunOption) error { http.DefaultTransport = config.NewHTTPTransport(src) metricsMgr := config.NewMetricsManager(ctx, src) - defer metricsMgr.Close() - traceMgr := config.NewTraceManager(ctx, src) - defer traceMgr.Close() eventsMgr := events.New() - fileMgr := options.fileMgr + fileMgr := p.fileMgr if fileMgr == nil { fileMgr = filemgr.NewManager() } cfg := src.GetConfig() + src.OnConfigChange(ctx, updateTraceClient) // setup the control plane controlPlane, err := controlplane.NewServer(ctx, cfg, metricsMgr, eventsMgr, fileMgr) @@ -137,11 +162,13 @@ func Run(ctx context.Context, src config.Source, opts ...RunOption) error { Msg("server started") // create envoy server - envoyServer, err := envoy.NewServer(ctx, src, controlPlane.Builder, options.envoyServerOptions...) + p.envoyServer, err = envoy.NewServer(ctx, src, controlPlane.Builder, p.envoyServerOptions...) if err != nil { return fmt.Errorf("error creating envoy server: %w", err) } - defer envoyServer.Close() + context.AfterFunc(ctx, func() { + p.envoyServer.Close() + }) // add services if err := setupAuthenticate(ctx, src, controlPlane); err != nil { @@ -156,50 +183,52 @@ func Run(ctx context.Context, src config.Source, opts ...RunOption) error { } var dataBrokerServer *databroker_service.DataBroker if config.IsDataBroker(src.GetConfig().Options.Services) { - dataBrokerServer, err = setupDataBroker(ctx, src, controlPlane, eventsMgr, options.databrokerServerOptions...) + dataBrokerServer, err = setupDataBroker(ctx, src, controlPlane, eventsMgr, p.databrokerServerOptions...) if err != nil { return fmt.Errorf("setting up databroker: %w", err) } } - if err = setupRegistryReporter(ctx, src); err != nil { + if err = setupRegistryReporter(ctx, tracerProvider, src); err != nil { return fmt.Errorf("setting up registry reporter: %w", err) } if err := setupProxy(ctx, src, controlPlane); err != nil { return err } - ctx, cancel := context.WithCancel(ctx) - go func(ctx context.Context) { - ch := make(chan os.Signal, 2) - defer signal.Stop(ch) - - signal.Notify(ch, os.Interrupt) - signal.Notify(ch, syscall.SIGTERM) - - select { - case <-ch: - case <-ctx.Done(): - } - cancel() - }(ctx) - // run everything - eg, ctx := errgroup.WithContext(ctx) + p.errGroup, ctx = errgroup.WithContext(ctx) if authorizeServer != nil { - eg.Go(func() error { + p.errGroup.Go(func() error { return authorizeServer.Run(ctx) }) } - eg.Go(func() error { + p.errGroup.Go(func() error { return controlPlane.Run(ctx) }) if dataBrokerServer != nil { - eg.Go(func() error { + p.errGroup.Go(func() error { return dataBrokerServer.Run(ctx) }) } - return eg.Wait() + return nil +} + +func (p *Pomerium) Shutdown(ctx context.Context) error { + _ = trace.WaitForSpans(ctx, p.envoyServer.ExitGracePeriod()) + var errs []error + errs = append(errs, p.envoyServer.Close()) // this only errors if signaling envoy fails + p.cancel(ErrShutdown) + errs = append(errs, p.Wait()) + return errors.Join(errs...) +} + +func (p *Pomerium) Wait() error { + err := p.errGroup.Wait() + if errors.Is(err, ErrShutdown) { + return nil + } + return err } func setupAuthenticate(ctx context.Context, src config.Source, controlPlane *controlplane.Server) error { @@ -253,8 +282,8 @@ func setupDataBroker(ctx context.Context, return svc, nil } -func setupRegistryReporter(ctx context.Context, src config.Source) error { - reporter := registry.NewReporter() +func setupRegistryReporter(ctx context.Context, tracerProvider oteltrace.TracerProvider, src config.Source) error { + reporter := registry.NewReporter(tracerProvider) src.OnConfigChange(ctx, reporter.OnConfigChange) reporter.OnConfigChange(ctx, src.GetConfig()) return nil @@ -280,3 +309,31 @@ func setupProxy(ctx context.Context, src config.Source, controlPlane *controlpla return nil } + +func updateTraceClient(ctx context.Context, cfg *config.Config) { + sc, ok := trace.RemoteClientFromContext(ctx).(trace.SyncClient) + if !ok { + return + } + newClient, err := config.NewTraceClientFromOptions(cfg.Options) + if errors.Is(err, config.ErrNoTracingConfig) { + newClient = trace.NewRemoteClientFromEnv() + err = nil + } + if err != nil { + log.Ctx(ctx).Warn().Err(err).Msg("error configuring trace client") + } else { + go func() { + if err := sc.Update(ctx, newClient); err != nil { + log.Ctx(ctx). + Warn(). + Err(err). + Msg("error updating trace client") + } + log.Ctx(ctx). + Info(). + Str("provider", cfg.Options.TracingProvider). + Msg("trace client updated") + }() + } +} diff --git a/pkg/envoy/envoy.go b/pkg/envoy/envoy.go index 78c80a308..8224f3647 100644 --- a/pkg/envoy/envoy.go +++ b/pkg/envoy/envoy.go @@ -16,6 +16,7 @@ import ( "strconv" "strings" "sync" + stdatomic "sync/atomic" "syscall" "time" @@ -42,6 +43,7 @@ type Server struct { wd string cmd *exec.Cmd cmdExited chan struct{} + closing stdatomic.Bool builder *envoyconfig.Builder resourceMonitor ResourceMonitor @@ -59,6 +61,10 @@ type ServerOptions struct { exitGracePeriod time.Duration } +func (o *ServerOptions) ExitGracePeriod() time.Duration { + return o.exitGracePeriod +} + type ServerOption func(*ServerOptions) func (o *ServerOptions) apply(opts ...ServerOption) { @@ -122,8 +128,13 @@ func NewServer(ctx context.Context, src config.Source, builder *envoyconfig.Buil return srv, nil } -// Close kills any underlying envoy process. +// Close attempts to gracefully shut down a running envoy server. If envoy +// does not exit within the defined grace period, it will be killed. Server +// cannot be used again after Close is called. func (srv *Server) Close() error { + if !srv.closing.CompareAndSwap(false, true) { + return nil + } srv.monitorProcessCancel() srv.mu.Lock() @@ -156,6 +167,10 @@ func (srv *Server) Close() error { } func (srv *Server) onConfigChange(ctx context.Context, cfg *config.Config) { + if srv.closing.Load() { + // do not attempt to update the configuration after Close is called + return + } srv.update(ctx, cfg) } @@ -233,10 +248,10 @@ func (srv *Server) run(ctx context.Context, cfg *config.Config) error { go func() { pid := cmd.Process.Pid err := srv.monitorProcess(monitorProcessCtx, int32(pid)) - if err != nil && ctx.Err() == nil { - // If the envoy subprocess exits and ctx is not done, issue a fatal error. - // If ctx is done, the server is already exiting, and envoy is expected - // to be stopped along with it. + if err != nil && ctx.Err() == nil && !srv.closing.Load() { + // If the envoy subprocess exits and ctx is not done (or waiting for envoy + // to gracefully stop), issue a fatal error. If ctx is done, the server is + // already exiting, and envoy is expected to be stopped along with it. log.Ctx(ctx). Fatal(). Int("pid", pid). diff --git a/pkg/envoy/get-envoy/main.go b/pkg/envoy/get-envoy/main.go index 32c453533..f378a6c86 100644 --- a/pkg/envoy/get-envoy/main.go +++ b/pkg/envoy/get-envoy/main.go @@ -15,14 +15,14 @@ import ( ) var ( - envoyVersion = "1.31.0" + envoyVersion = "1.32.3" targets = []string{ "darwin-amd64", "darwin-arm64", "linux-amd64", "linux-arm64", } - baseURL = "https://github.com/pomerium/envoy-binaries/releases/download/v" + envoyVersion + baseURL = "https://github.com/pomerium/envoy-custom/releases/download/v" + envoyVersion ) func main() { diff --git a/pkg/grpc/client.go b/pkg/grpc/client.go index 0d145769f..4a965b603 100644 --- a/pkg/grpc/client.go +++ b/pkg/grpc/client.go @@ -10,7 +10,6 @@ import ( "google.golang.org/grpc/credentials/insecure" "github.com/pomerium/pomerium/internal/log" - "github.com/pomerium/pomerium/internal/telemetry" "github.com/pomerium/pomerium/pkg/grpcutil" "github.com/pomerium/pomerium/pkg/telemetry/requestid" ) @@ -32,11 +31,8 @@ type Options struct { // NewGRPCClientConn returns a new gRPC pomerium service client connection. func NewGRPCClientConn(ctx context.Context, opts *Options, other ...grpc.DialOption) (*grpc.ClientConn, error) { - clientStatsHandler := telemetry.NewGRPCClientStatsHandler(opts.ServiceName) - unaryClientInterceptors := []grpc.UnaryClientInterceptor{ requestid.UnaryClientInterceptor(), - clientStatsHandler.UnaryInterceptor, } streamClientInterceptors := []grpc.StreamClientInterceptor{ requestid.StreamClientInterceptor(), @@ -49,7 +45,6 @@ func NewGRPCClientConn(ctx context.Context, opts *Options, other ...grpc.DialOpt dialOptions := []grpc.DialOption{ grpc.WithChainUnaryInterceptor(unaryClientInterceptors...), grpc.WithChainStreamInterceptor(streamClientInterceptors...), - grpc.WithStatsHandler(clientStatsHandler.Handler), grpc.WithDisableServiceConfig(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), @@ -75,13 +70,13 @@ type OutboundOptions struct { } // newOutboundGRPCClientConn gets a new outbound gRPC client. -func newOutboundGRPCClientConn(ctx context.Context, opts *OutboundOptions) (*grpc.ClientConn, error) { +func newOutboundGRPCClientConn(ctx context.Context, opts *OutboundOptions, other ...grpc.DialOption) (*grpc.ClientConn, error) { return NewGRPCClientConn(ctx, &Options{ Address: net.JoinHostPort("127.0.0.1", opts.OutboundPort), InstallationID: opts.InstallationID, ServiceName: opts.ServiceName, SignedJWTKey: opts.SignedJWTKey, - }) + }, other...) } // CachedOutboundGRPClientConn keeps a cached outbound gRPC client connection open based on options. @@ -92,7 +87,7 @@ type CachedOutboundGRPClientConn struct { } // Get gets the cached outbound gRPC client, or creates a new one if the options have changed. -func (cache *CachedOutboundGRPClientConn) Get(ctx context.Context, opts *OutboundOptions) (*grpc.ClientConn, error) { +func (cache *CachedOutboundGRPClientConn) Get(ctx context.Context, opts *OutboundOptions, other ...grpc.DialOption) (*grpc.ClientConn, error) { cache.mu.Lock() defer cache.mu.Unlock() @@ -106,7 +101,7 @@ func (cache *CachedOutboundGRPClientConn) Get(ctx context.Context, opts *Outboun } var err error - cache.current, err = newOutboundGRPCClientConn(ctx, opts) + cache.current, err = newOutboundGRPCClientConn(ctx, opts, other...) if err != nil { return nil, err } diff --git a/pkg/grpc/config/config.pb.go b/pkg/grpc/config/config.pb.go index 8c2edcd20..433bbded3 100644 --- a/pkg/grpc/config/config.pb.go +++ b/pkg/grpc/config/config.pb.go @@ -1229,24 +1229,20 @@ type Settings struct { SigningKey *string `protobuf:"bytes,36,opt,name=signing_key,json=signingKey,proto3,oneof" json:"signing_key,omitempty"` SetResponseHeaders map[string]string `protobuf:"bytes,69,rep,name=set_response_headers,json=setResponseHeaders,proto3" json:"set_response_headers,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // repeated string jwt_claims_headers = 37; - JwtClaimsHeaders map[string]string `protobuf:"bytes,63,rep,name=jwt_claims_headers,json=jwtClaimsHeaders,proto3" json:"jwt_claims_headers,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - JwtGroupsFilter []string `protobuf:"bytes,119,rep,name=jwt_groups_filter,json=jwtGroupsFilter,proto3" json:"jwt_groups_filter,omitempty"` - DefaultUpstreamTimeout *durationpb.Duration `protobuf:"bytes,39,opt,name=default_upstream_timeout,json=defaultUpstreamTimeout,proto3,oneof" json:"default_upstream_timeout,omitempty"` - MetricsAddress *string `protobuf:"bytes,40,opt,name=metrics_address,json=metricsAddress,proto3,oneof" json:"metrics_address,omitempty"` - MetricsBasicAuth *string `protobuf:"bytes,64,opt,name=metrics_basic_auth,json=metricsBasicAuth,proto3,oneof" json:"metrics_basic_auth,omitempty"` - MetricsCertificate *Settings_Certificate `protobuf:"bytes,65,opt,name=metrics_certificate,json=metricsCertificate,proto3,oneof" json:"metrics_certificate,omitempty"` - MetricsClientCa *string `protobuf:"bytes,66,opt,name=metrics_client_ca,json=metricsClientCa,proto3,oneof" json:"metrics_client_ca,omitempty"` - TracingProvider *string `protobuf:"bytes,41,opt,name=tracing_provider,json=tracingProvider,proto3,oneof" json:"tracing_provider,omitempty"` - TracingSampleRate *float64 `protobuf:"fixed64,42,opt,name=tracing_sample_rate,json=tracingSampleRate,proto3,oneof" json:"tracing_sample_rate,omitempty"` - TracingOtlpEndpoint *string `protobuf:"bytes,120,opt,name=tracing_otlp_endpoint,json=tracingOtlpEndpoint,proto3,oneof" json:"tracing_otlp_endpoint,omitempty"` - TracingOtlpProtocol *string `protobuf:"bytes,121,opt,name=tracing_otlp_protocol,json=tracingOtlpProtocol,proto3,oneof" json:"tracing_otlp_protocol,omitempty"` - TracingDatadogAddress *string `protobuf:"bytes,98,opt,name=tracing_datadog_address,json=tracingDatadogAddress,proto3,oneof" json:"tracing_datadog_address,omitempty"` - TracingJaegerCollectorEndpoint *string `protobuf:"bytes,43,opt,name=tracing_jaeger_collector_endpoint,json=tracingJaegerCollectorEndpoint,proto3,oneof" json:"tracing_jaeger_collector_endpoint,omitempty"` - TracingJaegerAgentEndpoint *string `protobuf:"bytes,44,opt,name=tracing_jaeger_agent_endpoint,json=tracingJaegerAgentEndpoint,proto3,oneof" json:"tracing_jaeger_agent_endpoint,omitempty"` - TracingZipkinEndpoint *string `protobuf:"bytes,45,opt,name=tracing_zipkin_endpoint,json=tracingZipkinEndpoint,proto3,oneof" json:"tracing_zipkin_endpoint,omitempty"` - GrpcAddress *string `protobuf:"bytes,46,opt,name=grpc_address,json=grpcAddress,proto3,oneof" json:"grpc_address,omitempty"` - GrpcInsecure *bool `protobuf:"varint,47,opt,name=grpc_insecure,json=grpcInsecure,proto3,oneof" json:"grpc_insecure,omitempty"` - GrpcClientTimeout *durationpb.Duration `protobuf:"bytes,99,opt,name=grpc_client_timeout,json=grpcClientTimeout,proto3,oneof" json:"grpc_client_timeout,omitempty"` + JwtClaimsHeaders map[string]string `protobuf:"bytes,63,rep,name=jwt_claims_headers,json=jwtClaimsHeaders,proto3" json:"jwt_claims_headers,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + JwtGroupsFilter []string `protobuf:"bytes,119,rep,name=jwt_groups_filter,json=jwtGroupsFilter,proto3" json:"jwt_groups_filter,omitempty"` + DefaultUpstreamTimeout *durationpb.Duration `protobuf:"bytes,39,opt,name=default_upstream_timeout,json=defaultUpstreamTimeout,proto3,oneof" json:"default_upstream_timeout,omitempty"` + MetricsAddress *string `protobuf:"bytes,40,opt,name=metrics_address,json=metricsAddress,proto3,oneof" json:"metrics_address,omitempty"` + MetricsBasicAuth *string `protobuf:"bytes,64,opt,name=metrics_basic_auth,json=metricsBasicAuth,proto3,oneof" json:"metrics_basic_auth,omitempty"` + MetricsCertificate *Settings_Certificate `protobuf:"bytes,65,opt,name=metrics_certificate,json=metricsCertificate,proto3,oneof" json:"metrics_certificate,omitempty"` + MetricsClientCa *string `protobuf:"bytes,66,opt,name=metrics_client_ca,json=metricsClientCa,proto3,oneof" json:"metrics_client_ca,omitempty"` + TracingProvider *string `protobuf:"bytes,41,opt,name=tracing_provider,json=tracingProvider,proto3,oneof" json:"tracing_provider,omitempty"` + TracingSampleRate *float64 `protobuf:"fixed64,42,opt,name=tracing_sample_rate,json=tracingSampleRate,proto3,oneof" json:"tracing_sample_rate,omitempty"` + TracingOtlpEndpoint *string `protobuf:"bytes,120,opt,name=tracing_otlp_endpoint,json=tracingOtlpEndpoint,proto3,oneof" json:"tracing_otlp_endpoint,omitempty"` + TracingOtlpProtocol *string `protobuf:"bytes,121,opt,name=tracing_otlp_protocol,json=tracingOtlpProtocol,proto3,oneof" json:"tracing_otlp_protocol,omitempty"` + GrpcAddress *string `protobuf:"bytes,46,opt,name=grpc_address,json=grpcAddress,proto3,oneof" json:"grpc_address,omitempty"` + GrpcInsecure *bool `protobuf:"varint,47,opt,name=grpc_insecure,json=grpcInsecure,proto3,oneof" json:"grpc_insecure,omitempty"` + GrpcClientTimeout *durationpb.Duration `protobuf:"bytes,99,opt,name=grpc_client_timeout,json=grpcClientTimeout,proto3,oneof" json:"grpc_client_timeout,omitempty"` // optional string forward_auth_url = 50; DatabrokerServiceUrls []string `protobuf:"bytes,52,rep,name=databroker_service_urls,json=databrokerServiceUrls,proto3" json:"databroker_service_urls,omitempty"` DatabrokerInternalServiceUrl *string `protobuf:"bytes,84,opt,name=databroker_internal_service_url,json=databrokerInternalServiceUrl,proto3,oneof" json:"databroker_internal_service_url,omitempty"` @@ -1662,34 +1658,6 @@ func (x *Settings) GetTracingOtlpProtocol() string { return "" } -func (x *Settings) GetTracingDatadogAddress() string { - if x != nil && x.TracingDatadogAddress != nil { - return *x.TracingDatadogAddress - } - return "" -} - -func (x *Settings) GetTracingJaegerCollectorEndpoint() string { - if x != nil && x.TracingJaegerCollectorEndpoint != nil { - return *x.TracingJaegerCollectorEndpoint - } - return "" -} - -func (x *Settings) GetTracingJaegerAgentEndpoint() string { - if x != nil && x.TracingJaegerAgentEndpoint != nil { - return *x.TracingJaegerAgentEndpoint - } - return "" -} - -func (x *Settings) GetTracingZipkinEndpoint() string { - if x != nil && x.TracingZipkinEndpoint != nil { - return *x.TracingZipkinEndpoint - } - return "" -} - func (x *Settings) GetGrpcAddress() string { if x != nil && x.GrpcAddress != nil { return *x.GrpcAddress @@ -2519,7 +2487,7 @@ var file_config_proto_rawDesc = []byte{ 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x70, 0x70, 0x6c, 0x22, - 0xc3, 0x3b, 0x0a, 0x08, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x2c, 0x0a, 0x0f, + 0xc9, 0x38, 0x0a, 0x08, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x2c, 0x0a, 0x0f, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x47, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x88, 0x01, 0x01, 0x12, 0x20, 0x0a, 0x09, 0x6c, 0x6f, @@ -2693,356 +2661,332 @@ var file_config_proto_rawDesc = []byte{ 0x37, 0x0a, 0x15, 0x74, 0x72, 0x61, 0x63, 0x69, 0x6e, 0x67, 0x5f, 0x6f, 0x74, 0x6c, 0x70, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x79, 0x20, 0x01, 0x28, 0x09, 0x48, 0x29, 0x52, 0x13, 0x74, 0x72, 0x61, 0x63, 0x69, 0x6e, 0x67, 0x4f, 0x74, 0x6c, 0x70, 0x50, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x88, 0x01, 0x01, 0x12, 0x3b, 0x0a, 0x17, 0x74, 0x72, 0x61, 0x63, - 0x69, 0x6e, 0x67, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x5f, 0x61, 0x64, 0x64, 0x72, - 0x65, 0x73, 0x73, 0x18, 0x62, 0x20, 0x01, 0x28, 0x09, 0x48, 0x2a, 0x52, 0x15, 0x74, 0x72, 0x61, - 0x63, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x41, 0x64, 0x64, 0x72, 0x65, - 0x73, 0x73, 0x88, 0x01, 0x01, 0x12, 0x4e, 0x0a, 0x21, 0x74, 0x72, 0x61, 0x63, 0x69, 0x6e, 0x67, - 0x5f, 0x6a, 0x61, 0x65, 0x67, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x2b, 0x20, 0x01, 0x28, 0x09, - 0x48, 0x2b, 0x52, 0x1e, 0x74, 0x72, 0x61, 0x63, 0x69, 0x6e, 0x67, 0x4a, 0x61, 0x65, 0x67, 0x65, - 0x72, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, - 0x6e, 0x74, 0x88, 0x01, 0x01, 0x12, 0x46, 0x0a, 0x1d, 0x74, 0x72, 0x61, 0x63, 0x69, 0x6e, 0x67, - 0x5f, 0x6a, 0x61, 0x65, 0x67, 0x65, 0x72, 0x5f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x65, 0x6e, - 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x2c, 0x20, 0x01, 0x28, 0x09, 0x48, 0x2c, 0x52, 0x1a, - 0x74, 0x72, 0x61, 0x63, 0x69, 0x6e, 0x67, 0x4a, 0x61, 0x65, 0x67, 0x65, 0x72, 0x41, 0x67, 0x65, - 0x6e, 0x74, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x88, 0x01, 0x01, 0x12, 0x3b, 0x0a, - 0x17, 0x74, 0x72, 0x61, 0x63, 0x69, 0x6e, 0x67, 0x5f, 0x7a, 0x69, 0x70, 0x6b, 0x69, 0x6e, 0x5f, - 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x2d, 0x20, 0x01, 0x28, 0x09, 0x48, 0x2d, - 0x52, 0x15, 0x74, 0x72, 0x61, 0x63, 0x69, 0x6e, 0x67, 0x5a, 0x69, 0x70, 0x6b, 0x69, 0x6e, 0x45, - 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x88, 0x01, 0x01, 0x12, 0x26, 0x0a, 0x0c, 0x67, 0x72, - 0x70, 0x63, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x2e, 0x20, 0x01, 0x28, 0x09, - 0x48, 0x2e, 0x52, 0x0b, 0x67, 0x72, 0x70, 0x63, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x88, - 0x01, 0x01, 0x12, 0x28, 0x0a, 0x0d, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x69, 0x6e, 0x73, 0x65, 0x63, - 0x75, 0x72, 0x65, 0x18, 0x2f, 0x20, 0x01, 0x28, 0x08, 0x48, 0x2f, 0x52, 0x0c, 0x67, 0x72, 0x70, - 0x63, 0x49, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x65, 0x88, 0x01, 0x01, 0x12, 0x4e, 0x0a, 0x13, - 0x67, 0x72, 0x70, 0x63, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, - 0x6f, 0x75, 0x74, 0x18, 0x63, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x30, 0x52, 0x11, 0x67, 0x72, 0x70, 0x63, 0x43, 0x6c, 0x69, 0x65, - 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x88, 0x01, 0x01, 0x12, 0x36, 0x0a, 0x17, - 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x73, 0x18, 0x34, 0x20, 0x03, 0x28, 0x09, 0x52, 0x15, 0x64, - 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x55, 0x72, 0x6c, 0x73, 0x12, 0x4a, 0x0a, 0x1f, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, - 0x65, 0x72, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x54, 0x20, 0x01, 0x28, 0x09, 0x48, 0x31, 0x52, - 0x1c, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x49, 0x6e, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x55, 0x72, 0x6c, 0x88, 0x01, 0x01, - 0x12, 0x3b, 0x0a, 0x17, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x5f, 0x73, - 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x65, 0x20, 0x01, 0x28, - 0x09, 0x48, 0x32, 0x52, 0x15, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x53, - 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x88, 0x01, 0x01, 0x12, 0x54, 0x0a, - 0x24, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x6f, 0x72, - 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, - 0x74, 0x72, 0x69, 0x6e, 0x67, 0x18, 0x66, 0x20, 0x01, 0x28, 0x09, 0x48, 0x33, 0x52, 0x21, 0x64, - 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, - 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, - 0x88, 0x01, 0x01, 0x12, 0x55, 0x0a, 0x0f, 0x64, 0x6f, 0x77, 0x6e, 0x73, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x5f, 0x6d, 0x74, 0x6c, 0x73, 0x18, 0x74, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x70, - 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x44, - 0x6f, 0x77, 0x6e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x74, 0x6c, 0x73, 0x53, 0x65, 0x74, - 0x74, 0x69, 0x6e, 0x67, 0x73, 0x48, 0x34, 0x52, 0x0e, 0x64, 0x6f, 0x77, 0x6e, 0x73, 0x74, 0x72, - 0x65, 0x61, 0x6d, 0x4d, 0x74, 0x6c, 0x73, 0x88, 0x01, 0x01, 0x12, 0x76, 0x0a, 0x36, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x5f, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x6c, 0x65, 0x73, 0x73, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x63, 0x63, - 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x37, 0x20, 0x01, 0x28, 0x09, 0x48, 0x35, 0x52, 0x31, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x6c, - 0x65, 0x73, 0x73, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x88, - 0x01, 0x01, 0x12, 0x31, 0x0a, 0x12, 0x75, 0x73, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x5f, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x6b, 0x20, 0x01, 0x28, 0x08, 0x48, 0x36, - 0x52, 0x10, 0x75, 0x73, 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, - 0x6f, 0x6c, 0x88, 0x01, 0x01, 0x12, 0x1f, 0x0a, 0x08, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, - 0x74, 0x18, 0x38, 0x20, 0x01, 0x28, 0x08, 0x48, 0x37, 0x52, 0x08, 0x61, 0x75, 0x74, 0x6f, 0x63, - 0x65, 0x72, 0x74, 0x88, 0x01, 0x01, 0x12, 0x24, 0x0a, 0x0b, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, - 0x72, 0x74, 0x5f, 0x63, 0x61, 0x18, 0x4c, 0x20, 0x01, 0x28, 0x09, 0x48, 0x38, 0x52, 0x0a, 0x61, - 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x43, 0x61, 0x88, 0x01, 0x01, 0x12, 0x2a, 0x0a, 0x0e, - 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x4d, - 0x20, 0x01, 0x28, 0x09, 0x48, 0x39, 0x52, 0x0d, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, - 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x88, 0x01, 0x01, 0x12, 0x35, 0x0a, 0x14, 0x61, 0x75, 0x74, 0x6f, - 0x63, 0x65, 0x72, 0x74, 0x5f, 0x75, 0x73, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x67, 0x69, 0x6e, 0x67, - 0x18, 0x39, 0x20, 0x01, 0x28, 0x08, 0x48, 0x3a, 0x52, 0x12, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, - 0x72, 0x74, 0x55, 0x73, 0x65, 0x53, 0x74, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x88, 0x01, 0x01, 0x12, - 0x32, 0x0a, 0x13, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x65, 0x61, 0x62, 0x5f, - 0x6b, 0x65, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x48, 0x3b, 0x52, 0x10, - 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x45, 0x61, 0x62, 0x4b, 0x65, 0x79, 0x49, 0x64, - 0x88, 0x01, 0x01, 0x12, 0x34, 0x0a, 0x14, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, - 0x65, 0x61, 0x62, 0x5f, 0x6d, 0x61, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x4f, 0x20, 0x01, 0x28, - 0x09, 0x48, 0x3c, 0x52, 0x11, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x45, 0x61, 0x62, - 0x4d, 0x61, 0x63, 0x4b, 0x65, 0x79, 0x88, 0x01, 0x01, 0x12, 0x35, 0x0a, 0x14, 0x61, 0x75, 0x74, - 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x6d, 0x75, 0x73, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x70, 0x6c, - 0x65, 0x18, 0x3a, 0x20, 0x01, 0x28, 0x08, 0x48, 0x3d, 0x52, 0x12, 0x61, 0x75, 0x74, 0x6f, 0x63, - 0x65, 0x72, 0x74, 0x4d, 0x75, 0x73, 0x74, 0x53, 0x74, 0x61, 0x70, 0x6c, 0x65, 0x88, 0x01, 0x01, - 0x12, 0x26, 0x0a, 0x0c, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x64, 0x69, 0x72, - 0x18, 0x3b, 0x20, 0x01, 0x28, 0x09, 0x48, 0x3e, 0x52, 0x0b, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, - 0x72, 0x74, 0x44, 0x69, 0x72, 0x88, 0x01, 0x01, 0x12, 0x33, 0x0a, 0x13, 0x61, 0x75, 0x74, 0x6f, - 0x63, 0x65, 0x72, 0x74, 0x5f, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x63, 0x61, 0x18, - 0x50, 0x20, 0x01, 0x28, 0x09, 0x48, 0x3f, 0x52, 0x11, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, - 0x74, 0x54, 0x72, 0x75, 0x73, 0x74, 0x65, 0x64, 0x43, 0x61, 0x88, 0x01, 0x01, 0x12, 0x2b, 0x0a, - 0x0f, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x78, 0x66, 0x66, 0x5f, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, - 0x18, 0x3d, 0x20, 0x01, 0x28, 0x08, 0x48, 0x40, 0x52, 0x0d, 0x73, 0x6b, 0x69, 0x70, 0x58, 0x66, - 0x66, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x88, 0x01, 0x01, 0x12, 0x34, 0x0a, 0x14, 0x78, 0x66, - 0x66, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x68, 0x6f, - 0x70, 0x73, 0x18, 0x46, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x41, 0x52, 0x11, 0x78, 0x66, 0x66, 0x4e, - 0x75, 0x6d, 0x54, 0x72, 0x75, 0x73, 0x74, 0x65, 0x64, 0x48, 0x6f, 0x70, 0x73, 0x88, 0x01, 0x01, - 0x12, 0x41, 0x0a, 0x1b, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x5f, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x5f, - 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6c, 0x6f, 0x67, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, - 0x6c, 0x20, 0x01, 0x28, 0x09, 0x48, 0x42, 0x52, 0x17, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x64, - 0x6d, 0x69, 0x6e, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x6f, 0x67, 0x50, 0x61, 0x74, 0x68, - 0x88, 0x01, 0x01, 0x12, 0x3c, 0x0a, 0x18, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x5f, 0x61, 0x64, 0x6d, - 0x69, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, - 0x6d, 0x20, 0x01, 0x28, 0x09, 0x48, 0x43, 0x52, 0x15, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x64, - 0x6d, 0x69, 0x6e, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x50, 0x61, 0x74, 0x68, 0x88, 0x01, - 0x01, 0x12, 0x33, 0x0a, 0x13, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x5f, 0x61, 0x64, 0x6d, 0x69, 0x6e, - 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x6e, 0x20, 0x01, 0x28, 0x09, 0x48, 0x44, - 0x52, 0x11, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x41, 0x64, 0x64, 0x72, - 0x65, 0x73, 0x73, 0x88, 0x01, 0x01, 0x12, 0x4b, 0x0a, 0x20, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x5f, - 0x62, 0x69, 0x6e, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x6f, 0x20, 0x01, 0x28, 0x09, - 0x48, 0x45, 0x52, 0x1c, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x42, 0x69, 0x6e, 0x64, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, - 0x88, 0x01, 0x01, 0x12, 0x40, 0x0a, 0x1a, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x5f, 0x62, 0x69, 0x6e, + 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x88, 0x01, 0x01, 0x12, 0x26, 0x0a, 0x0c, 0x67, 0x72, 0x70, 0x63, + 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x2e, 0x20, 0x01, 0x28, 0x09, 0x48, 0x2a, + 0x52, 0x0b, 0x67, 0x72, 0x70, 0x63, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x88, 0x01, 0x01, + 0x12, 0x28, 0x0a, 0x0d, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x69, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x72, + 0x65, 0x18, 0x2f, 0x20, 0x01, 0x28, 0x08, 0x48, 0x2b, 0x52, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x49, + 0x6e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x65, 0x88, 0x01, 0x01, 0x12, 0x4e, 0x0a, 0x13, 0x67, 0x72, + 0x70, 0x63, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, + 0x74, 0x18, 0x63, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x48, 0x2c, 0x52, 0x11, 0x67, 0x72, 0x70, 0x63, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x88, 0x01, 0x01, 0x12, 0x36, 0x0a, 0x17, 0x64, 0x61, + 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x5f, 0x75, 0x72, 0x6c, 0x73, 0x18, 0x34, 0x20, 0x03, 0x28, 0x09, 0x52, 0x15, 0x64, 0x61, 0x74, + 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x55, 0x72, + 0x6c, 0x73, 0x12, 0x4a, 0x0a, 0x1f, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, + 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x54, 0x20, 0x01, 0x28, 0x09, 0x48, 0x2d, 0x52, 0x1c, 0x64, + 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x55, 0x72, 0x6c, 0x88, 0x01, 0x01, 0x12, 0x3b, + 0x0a, 0x17, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x6f, + 0x72, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x65, 0x20, 0x01, 0x28, 0x09, 0x48, + 0x2e, 0x52, 0x15, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x6f, + 0x72, 0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x88, 0x01, 0x01, 0x12, 0x54, 0x0a, 0x24, 0x64, + 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, + 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x72, + 0x69, 0x6e, 0x67, 0x18, 0x66, 0x20, 0x01, 0x28, 0x09, 0x48, 0x2f, 0x52, 0x21, 0x64, 0x61, 0x74, + 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x43, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x88, 0x01, + 0x01, 0x12, 0x55, 0x0a, 0x0f, 0x64, 0x6f, 0x77, 0x6e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, + 0x6d, 0x74, 0x6c, 0x73, 0x18, 0x74, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x70, 0x6f, 0x6d, + 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x44, 0x6f, 0x77, + 0x6e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x74, 0x6c, 0x73, 0x53, 0x65, 0x74, 0x74, 0x69, + 0x6e, 0x67, 0x73, 0x48, 0x30, 0x52, 0x0e, 0x64, 0x6f, 0x77, 0x6e, 0x73, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x4d, 0x74, 0x6c, 0x73, 0x88, 0x01, 0x01, 0x12, 0x76, 0x0a, 0x36, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x5f, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x6c, + 0x65, 0x73, 0x73, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x6f, 0x75, + 0x6e, 0x74, 0x18, 0x37, 0x20, 0x01, 0x28, 0x09, 0x48, 0x31, 0x52, 0x31, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x6c, 0x65, 0x73, + 0x73, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x88, 0x01, 0x01, + 0x12, 0x31, 0x0a, 0x12, 0x75, 0x73, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x5f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x6b, 0x20, 0x01, 0x28, 0x08, 0x48, 0x32, 0x52, 0x10, + 0x75, 0x73, 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, + 0x88, 0x01, 0x01, 0x12, 0x1f, 0x0a, 0x08, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x18, + 0x38, 0x20, 0x01, 0x28, 0x08, 0x48, 0x33, 0x52, 0x08, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, + 0x74, 0x88, 0x01, 0x01, 0x12, 0x24, 0x0a, 0x0b, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, + 0x5f, 0x63, 0x61, 0x18, 0x4c, 0x20, 0x01, 0x28, 0x09, 0x48, 0x34, 0x52, 0x0a, 0x61, 0x75, 0x74, + 0x6f, 0x63, 0x65, 0x72, 0x74, 0x43, 0x61, 0x88, 0x01, 0x01, 0x12, 0x2a, 0x0a, 0x0e, 0x61, 0x75, + 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x4d, 0x20, 0x01, + 0x28, 0x09, 0x48, 0x35, 0x52, 0x0d, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x45, 0x6d, + 0x61, 0x69, 0x6c, 0x88, 0x01, 0x01, 0x12, 0x35, 0x0a, 0x14, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, + 0x72, 0x74, 0x5f, 0x75, 0x73, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x18, 0x39, + 0x20, 0x01, 0x28, 0x08, 0x48, 0x36, 0x52, 0x12, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, + 0x55, 0x73, 0x65, 0x53, 0x74, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x88, 0x01, 0x01, 0x12, 0x32, 0x0a, + 0x13, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x65, 0x61, 0x62, 0x5f, 0x6b, 0x65, + 0x79, 0x5f, 0x69, 0x64, 0x18, 0x4e, 0x20, 0x01, 0x28, 0x09, 0x48, 0x37, 0x52, 0x10, 0x61, 0x75, + 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x45, 0x61, 0x62, 0x4b, 0x65, 0x79, 0x49, 0x64, 0x88, 0x01, + 0x01, 0x12, 0x34, 0x0a, 0x14, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x65, 0x61, + 0x62, 0x5f, 0x6d, 0x61, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x4f, 0x20, 0x01, 0x28, 0x09, 0x48, + 0x38, 0x52, 0x11, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x45, 0x61, 0x62, 0x4d, 0x61, + 0x63, 0x4b, 0x65, 0x79, 0x88, 0x01, 0x01, 0x12, 0x35, 0x0a, 0x14, 0x61, 0x75, 0x74, 0x6f, 0x63, + 0x65, 0x72, 0x74, 0x5f, 0x6d, 0x75, 0x73, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x70, 0x6c, 0x65, 0x18, + 0x3a, 0x20, 0x01, 0x28, 0x08, 0x48, 0x39, 0x52, 0x12, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, + 0x74, 0x4d, 0x75, 0x73, 0x74, 0x53, 0x74, 0x61, 0x70, 0x6c, 0x65, 0x88, 0x01, 0x01, 0x12, 0x26, + 0x0a, 0x0c, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x64, 0x69, 0x72, 0x18, 0x3b, + 0x20, 0x01, 0x28, 0x09, 0x48, 0x3a, 0x52, 0x0b, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, + 0x44, 0x69, 0x72, 0x88, 0x01, 0x01, 0x12, 0x33, 0x0a, 0x13, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, + 0x72, 0x74, 0x5f, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x63, 0x61, 0x18, 0x50, 0x20, + 0x01, 0x28, 0x09, 0x48, 0x3b, 0x52, 0x11, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x54, + 0x72, 0x75, 0x73, 0x74, 0x65, 0x64, 0x43, 0x61, 0x88, 0x01, 0x01, 0x12, 0x2b, 0x0a, 0x0f, 0x73, + 0x6b, 0x69, 0x70, 0x5f, 0x78, 0x66, 0x66, 0x5f, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x18, 0x3d, + 0x20, 0x01, 0x28, 0x08, 0x48, 0x3c, 0x52, 0x0d, 0x73, 0x6b, 0x69, 0x70, 0x58, 0x66, 0x66, 0x41, + 0x70, 0x70, 0x65, 0x6e, 0x64, 0x88, 0x01, 0x01, 0x12, 0x34, 0x0a, 0x14, 0x78, 0x66, 0x66, 0x5f, + 0x6e, 0x75, 0x6d, 0x5f, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x68, 0x6f, 0x70, 0x73, + 0x18, 0x46, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x3d, 0x52, 0x11, 0x78, 0x66, 0x66, 0x4e, 0x75, 0x6d, + 0x54, 0x72, 0x75, 0x73, 0x74, 0x65, 0x64, 0x48, 0x6f, 0x70, 0x73, 0x88, 0x01, 0x01, 0x12, 0x41, + 0x0a, 0x1b, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x5f, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x5f, 0x61, 0x63, + 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6c, 0x6f, 0x67, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x6c, 0x20, + 0x01, 0x28, 0x09, 0x48, 0x3e, 0x52, 0x17, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x64, 0x6d, 0x69, + 0x6e, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x6f, 0x67, 0x50, 0x61, 0x74, 0x68, 0x88, 0x01, + 0x01, 0x12, 0x3c, 0x0a, 0x18, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x5f, 0x61, 0x64, 0x6d, 0x69, 0x6e, + 0x5f, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x6d, 0x20, + 0x01, 0x28, 0x09, 0x48, 0x3f, 0x52, 0x15, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x64, 0x6d, 0x69, + 0x6e, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x50, 0x61, 0x74, 0x68, 0x88, 0x01, 0x01, 0x12, + 0x33, 0x0a, 0x13, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x5f, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x5f, 0x61, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x6e, 0x20, 0x01, 0x28, 0x09, 0x48, 0x40, 0x52, 0x11, + 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, + 0x73, 0x88, 0x01, 0x01, 0x12, 0x4b, 0x0a, 0x20, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x5f, 0x62, 0x69, + 0x6e, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x6f, 0x20, 0x01, 0x28, 0x09, 0x48, 0x41, + 0x52, 0x1c, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x42, 0x69, 0x6e, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x88, 0x01, + 0x01, 0x12, 0x40, 0x0a, 0x1a, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x5f, 0x62, 0x69, 0x6e, 0x64, 0x5f, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x66, 0x72, 0x65, 0x65, 0x62, 0x69, 0x6e, 0x64, 0x18, + 0x70, 0x20, 0x01, 0x28, 0x08, 0x48, 0x42, 0x52, 0x17, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x42, 0x69, + 0x6e, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x72, 0x65, 0x65, 0x62, 0x69, 0x6e, 0x64, + 0x88, 0x01, 0x01, 0x12, 0x53, 0x0a, 0x26, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, + 0x74, 0x69, 0x63, 0x5f, 0x72, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x5f, 0x64, 0x6f, 0x6d, + 0x61, 0x69, 0x6e, 0x5f, 0x77, 0x68, 0x69, 0x74, 0x65, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x44, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x23, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x74, 0x69, + 0x63, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x57, + 0x68, 0x69, 0x74, 0x65, 0x6c, 0x69, 0x73, 0x74, 0x12, 0x80, 0x01, 0x0a, 0x0a, 0x63, 0x6f, 0x64, + 0x65, 0x63, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x49, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x5c, 0x2e, + 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, + 0x2e, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, + 0x2e, 0x68, 0x74, 0x74, 0x70, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x76, 0x33, 0x2e, 0x48, 0x74, 0x74, 0x70, + 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x72, 0x2e, 0x43, 0x6f, 0x64, 0x65, 0x63, 0x54, 0x79, 0x70, 0x65, 0x48, 0x43, 0x52, 0x09, 0x63, + 0x6f, 0x64, 0x65, 0x63, 0x54, 0x79, 0x70, 0x65, 0x88, 0x01, 0x01, 0x12, 0x28, 0x0a, 0x0d, 0x70, + 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x63, 0x6f, 0x6c, 0x6f, 0x72, 0x18, 0x55, 0x20, 0x01, + 0x28, 0x09, 0x48, 0x44, 0x52, 0x0c, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x43, 0x6f, 0x6c, + 0x6f, 0x72, 0x88, 0x01, 0x01, 0x12, 0x2c, 0x0a, 0x0f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x61, + 0x72, 0x79, 0x5f, 0x63, 0x6f, 0x6c, 0x6f, 0x72, 0x18, 0x56, 0x20, 0x01, 0x28, 0x09, 0x48, 0x45, + 0x52, 0x0e, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x43, 0x6f, 0x6c, 0x6f, 0x72, + 0x88, 0x01, 0x01, 0x12, 0x39, 0x0a, 0x16, 0x64, 0x61, 0x72, 0x6b, 0x6d, 0x6f, 0x64, 0x65, 0x5f, + 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x63, 0x6f, 0x6c, 0x6f, 0x72, 0x18, 0x57, 0x20, + 0x01, 0x28, 0x09, 0x48, 0x46, 0x52, 0x14, 0x64, 0x61, 0x72, 0x6b, 0x6d, 0x6f, 0x64, 0x65, 0x50, + 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x43, 0x6f, 0x6c, 0x6f, 0x72, 0x88, 0x01, 0x01, 0x12, 0x3d, + 0x0a, 0x18, 0x64, 0x61, 0x72, 0x6b, 0x6d, 0x6f, 0x64, 0x65, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, + 0x64, 0x61, 0x72, 0x79, 0x5f, 0x63, 0x6f, 0x6c, 0x6f, 0x72, 0x18, 0x58, 0x20, 0x01, 0x28, 0x09, + 0x48, 0x47, 0x52, 0x16, 0x64, 0x61, 0x72, 0x6b, 0x6d, 0x6f, 0x64, 0x65, 0x53, 0x65, 0x63, 0x6f, + 0x6e, 0x64, 0x61, 0x72, 0x79, 0x43, 0x6f, 0x6c, 0x6f, 0x72, 0x88, 0x01, 0x01, 0x12, 0x1e, 0x0a, + 0x08, 0x6c, 0x6f, 0x67, 0x6f, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x59, 0x20, 0x01, 0x28, 0x09, 0x48, + 0x48, 0x52, 0x07, 0x6c, 0x6f, 0x67, 0x6f, 0x55, 0x72, 0x6c, 0x88, 0x01, 0x01, 0x12, 0x24, 0x0a, + 0x0b, 0x66, 0x61, 0x76, 0x69, 0x63, 0x6f, 0x6e, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x5a, 0x20, 0x01, + 0x28, 0x09, 0x48, 0x49, 0x52, 0x0a, 0x66, 0x61, 0x76, 0x69, 0x63, 0x6f, 0x6e, 0x55, 0x72, 0x6c, + 0x88, 0x01, 0x01, 0x12, 0x46, 0x0a, 0x1d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x5f, 0x66, 0x69, 0x72, 0x73, 0x74, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x67, + 0x72, 0x61, 0x70, 0x68, 0x18, 0x5b, 0x20, 0x01, 0x28, 0x09, 0x48, 0x4a, 0x52, 0x1a, 0x65, 0x72, + 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x46, 0x69, 0x72, 0x73, 0x74, 0x50, + 0x61, 0x72, 0x61, 0x67, 0x72, 0x61, 0x70, 0x68, 0x88, 0x01, 0x01, 0x12, 0x37, 0x0a, 0x15, 0x70, + 0x61, 0x73, 0x73, 0x5f, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x68, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x73, 0x18, 0x75, 0x20, 0x01, 0x28, 0x08, 0x48, 0x4b, 0x52, 0x13, 0x70, 0x61, + 0x73, 0x73, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, + 0x73, 0x88, 0x01, 0x01, 0x12, 0x50, 0x0a, 0x0d, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5f, + 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x76, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x70, 0x6f, + 0x6d, 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x53, 0x65, + 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x2e, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x46, 0x6c, + 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, + 0x65, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x1a, 0x59, 0x0a, 0x0b, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, + 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x62, 0x79, + 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x65, 0x72, 0x74, 0x42, + 0x79, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x6b, 0x65, 0x79, 0x5f, 0x62, 0x79, 0x74, 0x65, + 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x42, 0x79, 0x74, 0x65, + 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, + 0x64, 0x1a, 0x24, 0x0a, 0x0a, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, 0x74, 0x12, + 0x16, 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x1a, 0x40, 0x0a, 0x12, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x45, 0x0a, 0x17, 0x53, 0x65, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x1a, 0x43, 0x0a, 0x15, 0x4a, 0x77, 0x74, 0x43, 0x6c, 0x61, 0x69, 0x6d, 0x73, 0x48, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3f, 0x0a, 0x11, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, + 0x46, 0x6c, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x69, 0x6e, 0x73, 0x74, 0x61, + 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x6c, + 0x6f, 0x67, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x61, 0x63, 0x63, + 0x65, 0x73, 0x73, 0x5f, 0x6c, 0x6f, 0x67, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x42, 0x17, + 0x0a, 0x15, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x5f, 0x6c, 0x6f, 0x67, + 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x70, 0x72, 0x6f, 0x78, + 0x79, 0x5f, 0x6c, 0x6f, 0x67, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x42, 0x10, 0x0a, 0x0e, 0x5f, + 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x42, 0x0b, 0x0a, + 0x09, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x61, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x69, 0x6e, 0x73, 0x65, 0x63, + 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x64, + 0x6e, 0x73, 0x5f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x66, 0x61, 0x6d, 0x69, 0x6c, 0x79, + 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x68, 0x74, 0x74, 0x70, 0x5f, 0x72, 0x65, 0x64, 0x69, 0x72, 0x65, + 0x63, 0x74, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x42, 0x0f, 0x0a, 0x0d, 0x5f, 0x74, 0x69, 0x6d, 0x65, + 0x6f, 0x75, 0x74, 0x5f, 0x72, 0x65, 0x61, 0x64, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x74, 0x69, 0x6d, + 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x77, 0x72, 0x69, 0x74, 0x65, 0x42, 0x0f, 0x0a, 0x0d, 0x5f, 0x74, + 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x69, 0x64, 0x6c, 0x65, 0x42, 0x1b, 0x0a, 0x19, 0x5f, + 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x5f, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x42, 0x24, 0x0a, 0x22, 0x5f, 0x61, 0x75, 0x74, + 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x42, 0x17, + 0x0a, 0x15, 0x5f, 0x73, 0x69, 0x67, 0x6e, 0x6f, 0x75, 0x74, 0x5f, 0x72, 0x65, 0x64, 0x69, 0x72, + 0x65, 0x63, 0x74, 0x5f, 0x75, 0x72, 0x6c, 0x42, 0x1d, 0x0a, 0x1b, 0x5f, 0x61, 0x75, 0x74, 0x68, + 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, + 0x6b, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x63, 0x6f, 0x6f, 0x6b, 0x69, + 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x63, 0x6f, 0x6f, 0x6b, 0x69, + 0x65, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x63, 0x6f, 0x6f, + 0x6b, 0x69, 0x65, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x63, + 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x5f, 0x68, 0x74, 0x74, 0x70, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x42, + 0x10, 0x0a, 0x0e, 0x5f, 0x63, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x5f, 0x65, 0x78, 0x70, 0x69, 0x72, + 0x65, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x63, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x5f, 0x73, 0x61, 0x6d, + 0x65, 0x5f, 0x73, 0x69, 0x74, 0x65, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x69, 0x64, 0x70, 0x5f, 0x63, + 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x69, 0x64, 0x70, + 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x42, 0x0f, + 0x0a, 0x0d, 0x5f, 0x69, 0x64, 0x70, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x42, + 0x13, 0x0a, 0x11, 0x5f, 0x69, 0x64, 0x70, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, + 0x5f, 0x75, 0x72, 0x6c, 0x42, 0x21, 0x0a, 0x1f, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, + 0x7a, 0x65, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x42, 0x1c, 0x0a, 0x1a, 0x5f, 0x6f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x42, 0x18, 0x0a, 0x16, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, + 0x69, 0x63, 0x61, 0x74, 0x65, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x42, + 0x0d, 0x0a, 0x0b, 0x5f, 0x64, 0x65, 0x72, 0x69, 0x76, 0x65, 0x5f, 0x74, 0x6c, 0x73, 0x42, 0x0e, + 0x0a, 0x0c, 0x5f, 0x73, 0x69, 0x67, 0x6e, 0x69, 0x6e, 0x67, 0x5f, 0x6b, 0x65, 0x79, 0x42, 0x1b, + 0x0a, 0x19, 0x5f, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x75, 0x70, 0x73, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x42, 0x12, 0x0a, 0x10, 0x5f, + 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x42, + 0x15, 0x0a, 0x13, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x62, 0x61, 0x73, 0x69, + 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x73, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x42, 0x14, + 0x0a, 0x12, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x5f, 0x63, 0x61, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x74, 0x72, 0x61, 0x63, 0x69, 0x6e, 0x67, + 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x74, 0x72, + 0x61, 0x63, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x5f, 0x72, 0x61, 0x74, + 0x65, 0x42, 0x18, 0x0a, 0x16, 0x5f, 0x74, 0x72, 0x61, 0x63, 0x69, 0x6e, 0x67, 0x5f, 0x6f, 0x74, + 0x6c, 0x70, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x42, 0x18, 0x0a, 0x16, 0x5f, + 0x74, 0x72, 0x61, 0x63, 0x69, 0x6e, 0x67, 0x5f, 0x6f, 0x74, 0x6c, 0x70, 0x5f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x42, 0x0f, 0x0a, 0x0d, 0x5f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x61, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x67, 0x72, 0x70, 0x63, 0x5f, + 0x69, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x65, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x67, 0x72, 0x70, + 0x63, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, + 0x42, 0x22, 0x0a, 0x20, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x5f, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x5f, 0x75, 0x72, 0x6c, 0x42, 0x1a, 0x0a, 0x18, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, + 0x6b, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, + 0x42, 0x27, 0x0a, 0x25, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x5f, + 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x64, 0x6f, + 0x77, 0x6e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x6d, 0x74, 0x6c, 0x73, 0x42, 0x39, 0x0a, + 0x37, 0x5f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x5f, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x5f, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x6c, 0x65, 0x73, 0x73, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, + 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x5f, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x75, 0x73, 0x65, + 0x5f, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x42, + 0x0b, 0x0a, 0x09, 0x5f, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x42, 0x0e, 0x0a, 0x0c, + 0x5f, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x63, 0x61, 0x42, 0x11, 0x0a, 0x0f, + 0x5f, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x42, + 0x17, 0x0a, 0x15, 0x5f, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x75, 0x73, 0x65, + 0x5f, 0x73, 0x74, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x61, 0x75, 0x74, + 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x65, 0x61, 0x62, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x69, 0x64, + 0x42, 0x17, 0x0a, 0x15, 0x5f, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x65, 0x61, + 0x62, 0x5f, 0x6d, 0x61, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x42, 0x17, 0x0a, 0x15, 0x5f, 0x61, 0x75, + 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x6d, 0x75, 0x73, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x70, + 0x6c, 0x65, 0x42, 0x0f, 0x0a, 0x0d, 0x5f, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, + 0x64, 0x69, 0x72, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, + 0x5f, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x63, 0x61, 0x42, 0x12, 0x0a, 0x10, 0x5f, + 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x78, 0x66, 0x66, 0x5f, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x42, + 0x17, 0x0a, 0x15, 0x5f, 0x78, 0x66, 0x66, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x74, 0x72, 0x75, 0x73, + 0x74, 0x65, 0x64, 0x5f, 0x68, 0x6f, 0x70, 0x73, 0x42, 0x1e, 0x0a, 0x1c, 0x5f, 0x65, 0x6e, 0x76, + 0x6f, 0x79, 0x5f, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, + 0x6c, 0x6f, 0x67, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x42, 0x1b, 0x0a, 0x19, 0x5f, 0x65, 0x6e, 0x76, + 0x6f, 0x79, 0x5f, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, + 0x5f, 0x70, 0x61, 0x74, 0x68, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x5f, + 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x42, 0x23, 0x0a, + 0x21, 0x5f, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x5f, 0x62, 0x69, 0x6e, 0x64, 0x5f, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, + 0x73, 0x73, 0x42, 0x1d, 0x0a, 0x1b, 0x5f, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x5f, 0x62, 0x69, 0x6e, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x66, 0x72, 0x65, 0x65, 0x62, 0x69, 0x6e, - 0x64, 0x18, 0x70, 0x20, 0x01, 0x28, 0x08, 0x48, 0x46, 0x52, 0x17, 0x65, 0x6e, 0x76, 0x6f, 0x79, - 0x42, 0x69, 0x6e, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x72, 0x65, 0x65, 0x62, 0x69, - 0x6e, 0x64, 0x88, 0x01, 0x01, 0x12, 0x53, 0x0a, 0x26, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x61, 0x6d, - 0x6d, 0x61, 0x74, 0x69, 0x63, 0x5f, 0x72, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x5f, 0x64, - 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x77, 0x68, 0x69, 0x74, 0x65, 0x6c, 0x69, 0x73, 0x74, 0x18, - 0x44, 0x20, 0x03, 0x28, 0x09, 0x52, 0x23, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x6d, 0x61, - 0x74, 0x69, 0x63, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x44, 0x6f, 0x6d, 0x61, 0x69, - 0x6e, 0x57, 0x68, 0x69, 0x74, 0x65, 0x6c, 0x69, 0x73, 0x74, 0x12, 0x80, 0x01, 0x0a, 0x0a, 0x63, - 0x6f, 0x64, 0x65, 0x63, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x49, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x5c, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, - 0x6e, 0x73, 0x2e, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, - 0x72, 0x6b, 0x2e, 0x68, 0x74, 0x74, 0x70, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x5f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x76, 0x33, 0x2e, 0x48, 0x74, - 0x74, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x2e, 0x43, 0x6f, 0x64, 0x65, 0x63, 0x54, 0x79, 0x70, 0x65, 0x48, 0x47, 0x52, - 0x09, 0x63, 0x6f, 0x64, 0x65, 0x63, 0x54, 0x79, 0x70, 0x65, 0x88, 0x01, 0x01, 0x12, 0x28, 0x0a, - 0x0d, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x63, 0x6f, 0x6c, 0x6f, 0x72, 0x18, 0x55, - 0x20, 0x01, 0x28, 0x09, 0x48, 0x48, 0x52, 0x0c, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x43, - 0x6f, 0x6c, 0x6f, 0x72, 0x88, 0x01, 0x01, 0x12, 0x2c, 0x0a, 0x0f, 0x73, 0x65, 0x63, 0x6f, 0x6e, - 0x64, 0x61, 0x72, 0x79, 0x5f, 0x63, 0x6f, 0x6c, 0x6f, 0x72, 0x18, 0x56, 0x20, 0x01, 0x28, 0x09, - 0x48, 0x49, 0x52, 0x0e, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x43, 0x6f, 0x6c, - 0x6f, 0x72, 0x88, 0x01, 0x01, 0x12, 0x39, 0x0a, 0x16, 0x64, 0x61, 0x72, 0x6b, 0x6d, 0x6f, 0x64, - 0x65, 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x63, 0x6f, 0x6c, 0x6f, 0x72, 0x18, - 0x57, 0x20, 0x01, 0x28, 0x09, 0x48, 0x4a, 0x52, 0x14, 0x64, 0x61, 0x72, 0x6b, 0x6d, 0x6f, 0x64, - 0x65, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x43, 0x6f, 0x6c, 0x6f, 0x72, 0x88, 0x01, 0x01, - 0x12, 0x3d, 0x0a, 0x18, 0x64, 0x61, 0x72, 0x6b, 0x6d, 0x6f, 0x64, 0x65, 0x5f, 0x73, 0x65, 0x63, - 0x6f, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x5f, 0x63, 0x6f, 0x6c, 0x6f, 0x72, 0x18, 0x58, 0x20, 0x01, - 0x28, 0x09, 0x48, 0x4b, 0x52, 0x16, 0x64, 0x61, 0x72, 0x6b, 0x6d, 0x6f, 0x64, 0x65, 0x53, 0x65, - 0x63, 0x6f, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x43, 0x6f, 0x6c, 0x6f, 0x72, 0x88, 0x01, 0x01, 0x12, - 0x1e, 0x0a, 0x08, 0x6c, 0x6f, 0x67, 0x6f, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x59, 0x20, 0x01, 0x28, - 0x09, 0x48, 0x4c, 0x52, 0x07, 0x6c, 0x6f, 0x67, 0x6f, 0x55, 0x72, 0x6c, 0x88, 0x01, 0x01, 0x12, - 0x24, 0x0a, 0x0b, 0x66, 0x61, 0x76, 0x69, 0x63, 0x6f, 0x6e, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x5a, - 0x20, 0x01, 0x28, 0x09, 0x48, 0x4d, 0x52, 0x0a, 0x66, 0x61, 0x76, 0x69, 0x63, 0x6f, 0x6e, 0x55, - 0x72, 0x6c, 0x88, 0x01, 0x01, 0x12, 0x46, 0x0a, 0x1d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x66, 0x69, 0x72, 0x73, 0x74, 0x5f, 0x70, 0x61, 0x72, - 0x61, 0x67, 0x72, 0x61, 0x70, 0x68, 0x18, 0x5b, 0x20, 0x01, 0x28, 0x09, 0x48, 0x4e, 0x52, 0x1a, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x46, 0x69, 0x72, 0x73, - 0x74, 0x50, 0x61, 0x72, 0x61, 0x67, 0x72, 0x61, 0x70, 0x68, 0x88, 0x01, 0x01, 0x12, 0x37, 0x0a, - 0x15, 0x70, 0x61, 0x73, 0x73, 0x5f, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x68, - 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x75, 0x20, 0x01, 0x28, 0x08, 0x48, 0x4f, 0x52, 0x13, - 0x70, 0x61, 0x73, 0x73, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x48, 0x65, 0x61, 0x64, - 0x65, 0x72, 0x73, 0x88, 0x01, 0x01, 0x12, 0x50, 0x0a, 0x0d, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, - 0x65, 0x5f, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x76, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, - 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, - 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x2e, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, - 0x46, 0x6c, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x72, 0x75, 0x6e, 0x74, - 0x69, 0x6d, 0x65, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x1a, 0x59, 0x0a, 0x0b, 0x43, 0x65, 0x72, 0x74, - 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x65, 0x72, 0x74, 0x5f, - 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x65, 0x72, - 0x74, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x6b, 0x65, 0x79, 0x5f, 0x62, 0x79, - 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x42, 0x79, - 0x74, 0x65, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x02, 0x69, 0x64, 0x1a, 0x24, 0x0a, 0x0a, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, - 0x74, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x1a, 0x40, 0x0a, 0x12, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x45, 0x0a, 0x17, 0x53, - 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x1a, 0x43, 0x0a, 0x15, 0x4a, 0x77, 0x74, 0x43, 0x6c, 0x61, 0x69, 0x6d, 0x73, 0x48, - 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3f, 0x0a, 0x11, 0x52, 0x75, 0x6e, 0x74, 0x69, - 0x6d, 0x65, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x69, 0x6e, 0x73, - 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x42, 0x0c, 0x0a, 0x0a, - 0x5f, 0x6c, 0x6f, 0x67, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x61, - 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6c, 0x6f, 0x67, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, - 0x42, 0x17, 0x0a, 0x15, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x5f, 0x6c, - 0x6f, 0x67, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x70, 0x72, - 0x6f, 0x78, 0x79, 0x5f, 0x6c, 0x6f, 0x67, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x42, 0x10, 0x0a, - 0x0e, 0x5f, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x42, - 0x0b, 0x0a, 0x09, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x42, 0x0a, 0x0a, 0x08, - 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x69, 0x6e, 0x73, - 0x65, 0x63, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x42, 0x14, 0x0a, 0x12, - 0x5f, 0x64, 0x6e, 0x73, 0x5f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x66, 0x61, 0x6d, 0x69, - 0x6c, 0x79, 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x68, 0x74, 0x74, 0x70, 0x5f, 0x72, 0x65, 0x64, 0x69, - 0x72, 0x65, 0x63, 0x74, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x42, 0x0f, 0x0a, 0x0d, 0x5f, 0x74, 0x69, - 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x72, 0x65, 0x61, 0x64, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x74, - 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x77, 0x72, 0x69, 0x74, 0x65, 0x42, 0x0f, 0x0a, 0x0d, - 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x69, 0x64, 0x6c, 0x65, 0x42, 0x1b, 0x0a, - 0x19, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x5f, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x42, 0x24, 0x0a, 0x22, 0x5f, 0x61, - 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x5f, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x75, 0x72, 0x6c, - 0x42, 0x17, 0x0a, 0x15, 0x5f, 0x73, 0x69, 0x67, 0x6e, 0x6f, 0x75, 0x74, 0x5f, 0x72, 0x65, 0x64, - 0x69, 0x72, 0x65, 0x63, 0x74, 0x5f, 0x75, 0x72, 0x6c, 0x42, 0x1d, 0x0a, 0x1b, 0x5f, 0x61, 0x75, - 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x62, - 0x61, 0x63, 0x6b, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x63, 0x6f, 0x6f, - 0x6b, 0x69, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x63, 0x6f, 0x6f, - 0x6b, 0x69, 0x65, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x63, - 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x42, 0x13, 0x0a, 0x11, - 0x5f, 0x63, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x5f, 0x68, 0x74, 0x74, 0x70, 0x5f, 0x6f, 0x6e, 0x6c, - 0x79, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x63, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x5f, 0x65, 0x78, 0x70, - 0x69, 0x72, 0x65, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x63, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x5f, 0x73, - 0x61, 0x6d, 0x65, 0x5f, 0x73, 0x69, 0x74, 0x65, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x69, 0x64, 0x70, - 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x69, - 0x64, 0x70, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, - 0x42, 0x0f, 0x0a, 0x0d, 0x5f, 0x69, 0x64, 0x70, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, - 0x72, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x69, 0x64, 0x70, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, - 0x65, 0x72, 0x5f, 0x75, 0x72, 0x6c, 0x42, 0x21, 0x0a, 0x1f, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x6f, - 0x72, 0x69, 0x7a, 0x65, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x42, 0x1c, 0x0a, 0x1a, 0x5f, 0x6f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, - 0x74, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x42, 0x18, 0x0a, 0x16, 0x5f, 0x63, 0x65, 0x72, 0x74, - 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, - 0x79, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x64, 0x65, 0x72, 0x69, 0x76, 0x65, 0x5f, 0x74, 0x6c, 0x73, - 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x73, 0x69, 0x67, 0x6e, 0x69, 0x6e, 0x67, 0x5f, 0x6b, 0x65, 0x79, - 0x42, 0x1b, 0x0a, 0x19, 0x5f, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x75, 0x70, 0x73, - 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x42, 0x12, 0x0a, - 0x10, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, - 0x73, 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x62, 0x61, - 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x6d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x73, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, - 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x63, 0x6c, 0x69, - 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x61, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x74, 0x72, 0x61, 0x63, 0x69, - 0x6e, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x42, 0x16, 0x0a, 0x14, 0x5f, - 0x74, 0x72, 0x61, 0x63, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x5f, 0x72, - 0x61, 0x74, 0x65, 0x42, 0x18, 0x0a, 0x16, 0x5f, 0x74, 0x72, 0x61, 0x63, 0x69, 0x6e, 0x67, 0x5f, - 0x6f, 0x74, 0x6c, 0x70, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x42, 0x18, 0x0a, - 0x16, 0x5f, 0x74, 0x72, 0x61, 0x63, 0x69, 0x6e, 0x67, 0x5f, 0x6f, 0x74, 0x6c, 0x70, 0x5f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x42, 0x1a, 0x0a, 0x18, 0x5f, 0x74, 0x72, 0x61, 0x63, - 0x69, 0x6e, 0x67, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x5f, 0x61, 0x64, 0x64, 0x72, - 0x65, 0x73, 0x73, 0x42, 0x24, 0x0a, 0x22, 0x5f, 0x74, 0x72, 0x61, 0x63, 0x69, 0x6e, 0x67, 0x5f, - 0x6a, 0x61, 0x65, 0x67, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x42, 0x20, 0x0a, 0x1e, 0x5f, 0x74, 0x72, - 0x61, 0x63, 0x69, 0x6e, 0x67, 0x5f, 0x6a, 0x61, 0x65, 0x67, 0x65, 0x72, 0x5f, 0x61, 0x67, 0x65, - 0x6e, 0x74, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x42, 0x1a, 0x0a, 0x18, 0x5f, - 0x74, 0x72, 0x61, 0x63, 0x69, 0x6e, 0x67, 0x5f, 0x7a, 0x69, 0x70, 0x6b, 0x69, 0x6e, 0x5f, 0x65, - 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x42, 0x0f, 0x0a, 0x0d, 0x5f, 0x67, 0x72, 0x70, 0x63, - 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x67, 0x72, 0x70, - 0x63, 0x5f, 0x69, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x65, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x67, - 0x72, 0x70, 0x63, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, - 0x75, 0x74, 0x42, 0x22, 0x0a, 0x20, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, - 0x72, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x42, 0x1a, 0x0a, 0x18, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x62, - 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x79, - 0x70, 0x65, 0x42, 0x27, 0x0a, 0x25, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, - 0x72, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x42, 0x12, 0x0a, 0x10, 0x5f, - 0x64, 0x6f, 0x77, 0x6e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x6d, 0x74, 0x6c, 0x73, 0x42, - 0x39, 0x0a, 0x37, 0x5f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x5f, 0x63, 0x6c, 0x6f, 0x75, 0x64, - 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x6c, 0x65, 0x73, 0x73, 0x5f, 0x61, 0x75, 0x74, 0x68, - 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x75, - 0x73, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, - 0x6c, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x42, 0x0e, - 0x0a, 0x0c, 0x5f, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x63, 0x61, 0x42, 0x11, - 0x0a, 0x0f, 0x5f, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x65, 0x6d, 0x61, 0x69, - 0x6c, 0x42, 0x17, 0x0a, 0x15, 0x5f, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x75, - 0x73, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x61, - 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x65, 0x61, 0x62, 0x5f, 0x6b, 0x65, 0x79, 0x5f, - 0x69, 0x64, 0x42, 0x17, 0x0a, 0x15, 0x5f, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, - 0x65, 0x61, 0x62, 0x5f, 0x6d, 0x61, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x42, 0x17, 0x0a, 0x15, 0x5f, - 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x6d, 0x75, 0x73, 0x74, 0x5f, 0x73, 0x74, - 0x61, 0x70, 0x6c, 0x65, 0x42, 0x0f, 0x0a, 0x0d, 0x5f, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, 0x72, - 0x74, 0x5f, 0x64, 0x69, 0x72, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x65, - 0x72, 0x74, 0x5f, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x63, 0x61, 0x42, 0x12, 0x0a, - 0x10, 0x5f, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x78, 0x66, 0x66, 0x5f, 0x61, 0x70, 0x70, 0x65, 0x6e, - 0x64, 0x42, 0x17, 0x0a, 0x15, 0x5f, 0x78, 0x66, 0x66, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x74, 0x72, - 0x75, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x68, 0x6f, 0x70, 0x73, 0x42, 0x1e, 0x0a, 0x1c, 0x5f, 0x65, - 0x6e, 0x76, 0x6f, 0x79, 0x5f, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, - 0x73, 0x5f, 0x6c, 0x6f, 0x67, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x42, 0x1b, 0x0a, 0x19, 0x5f, 0x65, - 0x6e, 0x76, 0x6f, 0x79, 0x5f, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x66, 0x69, - 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x65, 0x6e, 0x76, 0x6f, - 0x79, 0x5f, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x42, - 0x23, 0x0a, 0x21, 0x5f, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x5f, 0x62, 0x69, 0x6e, 0x64, 0x5f, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x61, 0x64, 0x64, - 0x72, 0x65, 0x73, 0x73, 0x42, 0x1d, 0x0a, 0x1b, 0x5f, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x5f, 0x62, - 0x69, 0x6e, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x66, 0x72, 0x65, 0x65, 0x62, - 0x69, 0x6e, 0x64, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x63, 0x5f, 0x74, 0x79, - 0x70, 0x65, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x63, - 0x6f, 0x6c, 0x6f, 0x72, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x61, - 0x72, 0x79, 0x5f, 0x63, 0x6f, 0x6c, 0x6f, 0x72, 0x42, 0x19, 0x0a, 0x17, 0x5f, 0x64, 0x61, 0x72, - 0x6b, 0x6d, 0x6f, 0x64, 0x65, 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x63, 0x6f, - 0x6c, 0x6f, 0x72, 0x42, 0x1b, 0x0a, 0x19, 0x5f, 0x64, 0x61, 0x72, 0x6b, 0x6d, 0x6f, 0x64, 0x65, - 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x5f, 0x63, 0x6f, 0x6c, 0x6f, 0x72, - 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x6c, 0x6f, 0x67, 0x6f, 0x5f, 0x75, 0x72, 0x6c, 0x42, 0x0e, 0x0a, - 0x0c, 0x5f, 0x66, 0x61, 0x76, 0x69, 0x63, 0x6f, 0x6e, 0x5f, 0x75, 0x72, 0x6c, 0x42, 0x20, 0x0a, - 0x1e, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, - 0x66, 0x69, 0x72, 0x73, 0x74, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x67, 0x72, 0x61, 0x70, 0x68, 0x42, - 0x18, 0x0a, 0x16, 0x5f, 0x70, 0x61, 0x73, 0x73, 0x5f, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, - 0x79, 0x5f, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x4a, 0x04, 0x08, 0x64, 0x10, 0x65, 0x4a, - 0x04, 0x08, 0x6a, 0x10, 0x6b, 0x22, 0xc8, 0x02, 0x0a, 0x16, 0x44, 0x6f, 0x77, 0x6e, 0x73, 0x74, - 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x74, 0x6c, 0x73, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, - 0x12, 0x13, 0x0a, 0x02, 0x63, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x02, - 0x63, 0x61, 0x88, 0x01, 0x01, 0x12, 0x15, 0x0a, 0x03, 0x63, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x48, 0x01, 0x52, 0x03, 0x63, 0x72, 0x6c, 0x88, 0x01, 0x01, 0x12, 0x4b, 0x0a, 0x0b, - 0x65, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x24, 0x2e, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2e, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x2e, 0x4d, 0x74, 0x6c, 0x73, 0x45, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x4d, 0x6f, 0x64, 0x65, 0x48, 0x02, 0x52, 0x0b, 0x65, 0x6e, 0x66, 0x6f, 0x72, - 0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x88, 0x01, 0x01, 0x12, 0x52, 0x0a, 0x17, 0x6d, 0x61, 0x74, - 0x63, 0x68, 0x5f, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x61, 0x6c, 0x74, 0x5f, 0x6e, - 0x61, 0x6d, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x6f, 0x6d, - 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x53, 0x41, 0x4e, - 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x72, 0x52, 0x14, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x53, 0x75, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x41, 0x6c, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x2d, 0x0a, - 0x10, 0x6d, 0x61, 0x78, 0x5f, 0x76, 0x65, 0x72, 0x69, 0x66, 0x79, 0x5f, 0x64, 0x65, 0x70, 0x74, - 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x03, 0x52, 0x0e, 0x6d, 0x61, 0x78, 0x56, 0x65, - 0x72, 0x69, 0x66, 0x79, 0x44, 0x65, 0x70, 0x74, 0x68, 0x88, 0x01, 0x01, 0x42, 0x05, 0x0a, 0x03, - 0x5f, 0x63, 0x61, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x63, 0x72, 0x6c, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, - 0x65, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x13, 0x0a, 0x11, 0x5f, - 0x6d, 0x61, 0x78, 0x5f, 0x76, 0x65, 0x72, 0x69, 0x66, 0x79, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, - 0x22, 0xd1, 0x01, 0x0a, 0x0a, 0x53, 0x41, 0x4e, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x72, 0x12, - 0x3e, 0x0a, 0x08, 0x73, 0x61, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x23, 0x2e, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2e, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x2e, 0x53, 0x41, 0x4e, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x72, 0x2e, 0x53, - 0x41, 0x4e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x07, 0x73, 0x61, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x18, 0x0a, 0x07, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x22, 0x69, 0x0a, 0x07, 0x53, 0x41, 0x4e, - 0x54, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x41, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, - 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x09, - 0x0a, 0x05, 0x45, 0x4d, 0x41, 0x49, 0x4c, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x44, 0x4e, 0x53, - 0x10, 0x02, 0x12, 0x07, 0x0a, 0x03, 0x55, 0x52, 0x49, 0x10, 0x03, 0x12, 0x0e, 0x0a, 0x0a, 0x49, - 0x50, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, 0x53, 0x10, 0x04, 0x12, 0x17, 0x0a, 0x13, 0x55, - 0x53, 0x45, 0x52, 0x5f, 0x50, 0x52, 0x49, 0x4e, 0x43, 0x49, 0x50, 0x41, 0x4c, 0x5f, 0x4e, 0x41, - 0x4d, 0x45, 0x10, 0x05, 0x2a, 0x31, 0x0a, 0x0c, 0x49, 0x73, 0x73, 0x75, 0x65, 0x72, 0x46, 0x6f, - 0x72, 0x6d, 0x61, 0x74, 0x12, 0x12, 0x0a, 0x0e, 0x49, 0x73, 0x73, 0x75, 0x65, 0x72, 0x48, 0x6f, - 0x73, 0x74, 0x4f, 0x6e, 0x6c, 0x79, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x49, 0x73, 0x73, 0x75, - 0x65, 0x72, 0x55, 0x52, 0x49, 0x10, 0x01, 0x2a, 0x63, 0x0a, 0x13, 0x4d, 0x74, 0x6c, 0x73, 0x45, - 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x0b, - 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x50, - 0x4f, 0x4c, 0x49, 0x43, 0x59, 0x10, 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x50, 0x4f, 0x4c, 0x49, 0x43, - 0x59, 0x5f, 0x57, 0x49, 0x54, 0x48, 0x5f, 0x44, 0x45, 0x46, 0x41, 0x55, 0x4c, 0x54, 0x5f, 0x44, - 0x45, 0x4e, 0x59, 0x10, 0x02, 0x12, 0x15, 0x0a, 0x11, 0x52, 0x45, 0x4a, 0x45, 0x43, 0x54, 0x5f, - 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x03, 0x42, 0x2e, 0x5a, 0x2c, - 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x6f, 0x6d, 0x65, 0x72, - 0x69, 0x75, 0x6d, 0x2f, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2f, 0x70, 0x6b, 0x67, - 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x64, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x63, 0x5f, 0x74, 0x79, 0x70, 0x65, + 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x63, 0x6f, 0x6c, + 0x6f, 0x72, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x61, 0x72, 0x79, + 0x5f, 0x63, 0x6f, 0x6c, 0x6f, 0x72, 0x42, 0x19, 0x0a, 0x17, 0x5f, 0x64, 0x61, 0x72, 0x6b, 0x6d, + 0x6f, 0x64, 0x65, 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x63, 0x6f, 0x6c, 0x6f, + 0x72, 0x42, 0x1b, 0x0a, 0x19, 0x5f, 0x64, 0x61, 0x72, 0x6b, 0x6d, 0x6f, 0x64, 0x65, 0x5f, 0x73, + 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x5f, 0x63, 0x6f, 0x6c, 0x6f, 0x72, 0x42, 0x0b, + 0x0a, 0x09, 0x5f, 0x6c, 0x6f, 0x67, 0x6f, 0x5f, 0x75, 0x72, 0x6c, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, + 0x66, 0x61, 0x76, 0x69, 0x63, 0x6f, 0x6e, 0x5f, 0x75, 0x72, 0x6c, 0x42, 0x20, 0x0a, 0x1e, 0x5f, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x66, 0x69, + 0x72, 0x73, 0x74, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x67, 0x72, 0x61, 0x70, 0x68, 0x42, 0x18, 0x0a, + 0x16, 0x5f, 0x70, 0x61, 0x73, 0x73, 0x5f, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, + 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x4a, 0x04, 0x08, 0x62, 0x10, 0x63, 0x4a, 0x04, 0x08, + 0x2b, 0x10, 0x2c, 0x4a, 0x04, 0x08, 0x2c, 0x10, 0x2d, 0x4a, 0x04, 0x08, 0x2d, 0x10, 0x2e, 0x4a, + 0x04, 0x08, 0x64, 0x10, 0x65, 0x4a, 0x04, 0x08, 0x6a, 0x10, 0x6b, 0x22, 0xc8, 0x02, 0x0a, 0x16, + 0x44, 0x6f, 0x77, 0x6e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x74, 0x6c, 0x73, 0x53, 0x65, + 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x13, 0x0a, 0x02, 0x63, 0x61, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x48, 0x00, 0x52, 0x02, 0x63, 0x61, 0x88, 0x01, 0x01, 0x12, 0x15, 0x0a, 0x03, 0x63, + 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x03, 0x63, 0x72, 0x6c, 0x88, + 0x01, 0x01, 0x12, 0x4b, 0x0a, 0x0b, 0x65, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x24, 0x2e, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, + 0x75, 0x6d, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x4d, 0x74, 0x6c, 0x73, 0x45, 0x6e, + 0x66, 0x6f, 0x72, 0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x4d, 0x6f, 0x64, 0x65, 0x48, 0x02, 0x52, + 0x0b, 0x65, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x88, 0x01, 0x01, 0x12, + 0x52, 0x0a, 0x17, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, + 0x5f, 0x61, 0x6c, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x1b, 0x2e, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2e, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x2e, 0x53, 0x41, 0x4e, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x72, 0x52, 0x14, 0x6d, + 0x61, 0x74, 0x63, 0x68, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x41, 0x6c, 0x74, 0x4e, 0x61, + 0x6d, 0x65, 0x73, 0x12, 0x2d, 0x0a, 0x10, 0x6d, 0x61, 0x78, 0x5f, 0x76, 0x65, 0x72, 0x69, 0x66, + 0x79, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x03, 0x52, + 0x0e, 0x6d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x69, 0x66, 0x79, 0x44, 0x65, 0x70, 0x74, 0x68, 0x88, + 0x01, 0x01, 0x42, 0x05, 0x0a, 0x03, 0x5f, 0x63, 0x61, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x63, 0x72, + 0x6c, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x65, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x6d, 0x61, 0x78, 0x5f, 0x76, 0x65, 0x72, 0x69, 0x66, 0x79, + 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, 0x22, 0xd1, 0x01, 0x0a, 0x0a, 0x53, 0x41, 0x4e, 0x4d, 0x61, + 0x74, 0x63, 0x68, 0x65, 0x72, 0x12, 0x3e, 0x0a, 0x08, 0x73, 0x61, 0x6e, 0x5f, 0x74, 0x79, 0x70, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, + 0x75, 0x6d, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x53, 0x41, 0x4e, 0x4d, 0x61, 0x74, + 0x63, 0x68, 0x65, 0x72, 0x2e, 0x53, 0x41, 0x4e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x07, 0x73, 0x61, + 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x22, + 0x69, 0x0a, 0x07, 0x53, 0x41, 0x4e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x41, + 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, + 0x45, 0x44, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x4d, 0x41, 0x49, 0x4c, 0x10, 0x01, 0x12, + 0x07, 0x0a, 0x03, 0x44, 0x4e, 0x53, 0x10, 0x02, 0x12, 0x07, 0x0a, 0x03, 0x55, 0x52, 0x49, 0x10, + 0x03, 0x12, 0x0e, 0x0a, 0x0a, 0x49, 0x50, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, 0x53, 0x10, + 0x04, 0x12, 0x17, 0x0a, 0x13, 0x55, 0x53, 0x45, 0x52, 0x5f, 0x50, 0x52, 0x49, 0x4e, 0x43, 0x49, + 0x50, 0x41, 0x4c, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x05, 0x2a, 0x31, 0x0a, 0x0c, 0x49, 0x73, + 0x73, 0x75, 0x65, 0x72, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x12, 0x0a, 0x0e, 0x49, 0x73, + 0x73, 0x75, 0x65, 0x72, 0x48, 0x6f, 0x73, 0x74, 0x4f, 0x6e, 0x6c, 0x79, 0x10, 0x00, 0x12, 0x0d, + 0x0a, 0x09, 0x49, 0x73, 0x73, 0x75, 0x65, 0x72, 0x55, 0x52, 0x49, 0x10, 0x01, 0x2a, 0x63, 0x0a, + 0x13, 0x4d, 0x74, 0x6c, 0x73, 0x45, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, + 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x50, 0x4f, 0x4c, 0x49, 0x43, 0x59, 0x10, 0x01, 0x12, 0x1c, 0x0a, + 0x18, 0x50, 0x4f, 0x4c, 0x49, 0x43, 0x59, 0x5f, 0x57, 0x49, 0x54, 0x48, 0x5f, 0x44, 0x45, 0x46, + 0x41, 0x55, 0x4c, 0x54, 0x5f, 0x44, 0x45, 0x4e, 0x59, 0x10, 0x02, 0x12, 0x15, 0x0a, 0x11, 0x52, + 0x45, 0x4a, 0x45, 0x43, 0x54, 0x5f, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, + 0x10, 0x03, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2f, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, + 0x75, 0x6d, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/grpc/config/config.proto b/pkg/grpc/config/config.proto index e2987af2d..818ab0fec 100644 --- a/pkg/grpc/config/config.proto +++ b/pkg/grpc/config/config.proto @@ -212,10 +212,10 @@ message Settings { optional double tracing_sample_rate = 42; optional string tracing_otlp_endpoint = 120; optional string tracing_otlp_protocol = 121; - optional string tracing_datadog_address = 98; - optional string tracing_jaeger_collector_endpoint = 43; - optional string tracing_jaeger_agent_endpoint = 44; - optional string tracing_zipkin_endpoint = 45; + reserved 98; // tracing_datadog_address + reserved 43; // tracing_jaeger_collector_endpoint + reserved 44; // tracing_jaeger_agent_endpoint + reserved 45; // tracing_zipkin_endpoint optional string grpc_address = 46; optional bool grpc_insecure = 47; optional google.protobuf.Duration grpc_client_timeout = 99; diff --git a/pkg/grpc/databroker/sync_test.go b/pkg/grpc/databroker/sync_test.go index f047b3058..2fbc67b5c 100644 --- a/pkg/grpc/databroker/sync_test.go +++ b/pkg/grpc/databroker/sync_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace" grpc "google.golang.org/grpc" "github.com/pomerium/pomerium/internal/databroker" @@ -24,7 +25,7 @@ func Test_SyncLatestRecords(t *testing.T) { defer clearTimeout() cc := testutil.NewGRPCServer(t, func(s *grpc.Server) { - databrokerpb.RegisterDataBrokerServiceServer(s, databroker.New(ctx)) + databrokerpb.RegisterDataBrokerServiceServer(s, databroker.New(ctx, trace.NewNoopTracerProvider())) }) c := databrokerpb.NewDataBrokerServiceClient(cc) diff --git a/pkg/identity/oidc/oidc.go b/pkg/identity/oidc/oidc.go index 00a705d7c..40b5c64cd 100644 --- a/pkg/identity/oidc/oidc.go +++ b/pkg/identity/oidc/oidc.go @@ -12,9 +12,11 @@ import ( "sync" go_oidc "github.com/coreos/go-oidc/v3/oidc" + "go.opentelemetry.io/otel/codes" "golang.org/x/oauth2" "github.com/pomerium/pomerium/internal/httputil" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/urlutil" "github.com/pomerium/pomerium/internal/version" "github.com/pomerium/pomerium/pkg/identity/identity" @@ -103,8 +105,12 @@ func New(ctx context.Context, o *oauth.Options, options ...Option) (*Provider, e // the state query parameter on your redirect callback. // See http://tools.ietf.org/html/rfc6749#section-10.12 for more info. func (p *Provider) SignIn(w http.ResponseWriter, r *http.Request, state string) error { + _, span := trace.Continue(r.Context(), "oidc: sign in") + defer span.End() + oa, err := p.GetOauthConfig() if err != nil { + span.SetStatus(codes.Error, err.Error()) return err } @@ -120,22 +126,14 @@ func (p *Provider) SignIn(w http.ResponseWriter, r *http.Request, state string) // Authenticate converts an authorization code returned from the identity // provider into a token which is then converted into a user session. func (p *Provider) Authenticate(ctx context.Context, code string, v identity.State) (*oauth2.Token, error) { - oa, err := p.GetOauthConfig() + ctx, span := trace.Continue(ctx, "oidc: authenticate") + defer span.End() + + oauth2Token, idToken, err := p.exchange(ctx, code) if err != nil { return nil, err } - // Exchange converts an authorization code into a token. - oauth2Token, err := oa.Exchange(ctx, code) - if err != nil { - return nil, fmt.Errorf("identity/oidc: token exchange failed: %w", err) - } - - idToken, err := p.getIDToken(ctx, oauth2Token) - if err != nil { - return nil, fmt.Errorf("identity/oidc: failed getting id_token: %w", err) - } - if rawIDToken, ok := oauth2Token.Extra("id_token").(string); ok { v.SetRawIDToken(rawIDToken) } @@ -143,21 +141,56 @@ func (p *Provider) Authenticate(ctx context.Context, code string, v identity.Sta // hydrate `v` using claims inside the returned `id_token` // https://openid.net/specs/openid-connect-core-1_0.html#TokenEndpoint if err := idToken.Claims(v); err != nil { - return nil, fmt.Errorf("identity/oidc: couldn't unmarshal extra claims %w", err) + err := fmt.Errorf("identity/oidc: couldn't unmarshal extra claims %w", err) + span.SetStatus(codes.Error, err.Error()) + return nil, err } if err := p.UpdateUserInfo(ctx, oauth2Token, v); err != nil { - return nil, fmt.Errorf("identity/oidc: couldn't update user info %w", err) + err := fmt.Errorf("identity/oidc: couldn't update user info %w", err) + span.SetStatus(codes.Error, err.Error()) + return nil, err } return oauth2Token, nil } +func (p *Provider) exchange(ctx context.Context, code string) (*oauth2.Token, *go_oidc.IDToken, error) { + ctx, span := trace.Continue(ctx, "oidc: token exchange") + defer span.End() + + oa, err := p.GetOauthConfig() + if err != nil { + span.SetStatus(codes.Error, err.Error()) + return nil, nil, err + } + + // Exchange converts an authorization code into a token. + oauth2Token, err := oa.Exchange(ctx, code) + if err != nil { + err := fmt.Errorf("identity/oidc: token exchange failed: %w", err) + span.SetStatus(codes.Error, err.Error()) + return nil, nil, err + } + + idToken, err := p.getIDToken(ctx, oauth2Token) + if err != nil { + err := fmt.Errorf("identity/oidc: failed getting id_token: %w", err) + span.SetStatus(codes.Error, err.Error()) + return nil, nil, err + } + + return oauth2Token, idToken, nil +} + // UpdateUserInfo calls the OIDC (spec required) UserInfo Endpoint as well as any // groups endpoint (non-spec) to populate the rest of the user's information. // // https://openid.net/specs/openid-connect-core-1_0.html#UserInfo func (p *Provider) UpdateUserInfo(ctx context.Context, t *oauth2.Token, v any) error { + ctx, span := trace.Continue(ctx, "oidc: update user info") + defer span.End() + pp, err := p.GetProvider() if err != nil { return err diff --git a/pkg/identity/providers.go b/pkg/identity/providers.go index 34b569cc1..9ddad4ceb 100644 --- a/pkg/identity/providers.go +++ b/pkg/identity/providers.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "golang.org/x/oauth2" "github.com/pomerium/pomerium/pkg/identity/identity" @@ -22,6 +23,7 @@ import ( "github.com/pomerium/pomerium/pkg/identity/oidc/okta" "github.com/pomerium/pomerium/pkg/identity/oidc/onelogin" "github.com/pomerium/pomerium/pkg/identity/oidc/ping" + oteltrace "go.opentelemetry.io/otel/trace" ) // State is the identity state. @@ -64,13 +66,20 @@ func init() { } // NewAuthenticator returns a new identity provider based on its name. -func NewAuthenticator(o oauth.Options) (a Authenticator, err error) { - ctx := context.Background() - +func NewAuthenticator(ctx context.Context, tracerProvider oteltrace.TracerProvider, o oauth.Options) (a Authenticator, err error) { if o.ProviderName == "" { return nil, fmt.Errorf("identity: provider is not defined") } + ctx = context.WithValue(ctx, oauth2.HTTPClient, &http.Client{ + Transport: otelhttp.NewTransport(nil, + otelhttp.WithTracerProvider(tracerProvider), + otelhttp.WithSpanNameFormatter(func(_ string, r *http.Request) string { + return fmt.Sprintf("OAuth2 Client: %s %s", r.Method, r.URL.Path) + }), + ), + }) + ctor, ok := registry[o.ProviderName] if !ok { return nil, fmt.Errorf("identity: unknown provider: %s", o.ProviderName) diff --git a/proxy/data_test.go b/proxy/data_test.go index 2e74036dc..b15a02539 100644 --- a/proxy/data_test.go +++ b/proxy/data_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" @@ -32,7 +33,7 @@ func Test_getUserInfoData(t *testing.T) { defer clearTimeout() cc := testutil.NewGRPCServer(t, func(srv *grpc.Server) { - databrokerpb.RegisterDataBrokerServiceServer(srv, databroker.New(ctx)) + databrokerpb.RegisterDataBrokerServiceServer(srv, databroker.New(ctx, trace.NewNoopTracerProvider())) }) t.Cleanup(func() { cc.Close() }) diff --git a/proxy/handlers.go b/proxy/handlers.go index 42fa01953..db5dd2179 100644 --- a/proxy/handlers.go +++ b/proxy/handlers.go @@ -9,11 +9,13 @@ import ( "github.com/go-jose/go-jose/v3/jwt" "github.com/gorilla/mux" + "go.opentelemetry.io/otel" "github.com/pomerium/pomerium/config" "github.com/pomerium/pomerium/internal/handlers" "github.com/pomerium/pomerium/internal/httputil" "github.com/pomerium/pomerium/internal/middleware" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/urlutil" ) @@ -72,6 +74,7 @@ func (p *Proxy) SignOut(w http.ResponseWriter, r *http.Request) error { if redirectURL != nil { q.Set(urlutil.QueryRedirectURI, redirectURL.String()) } + otel.GetTextMapPropagator().Inject(r.Context(), trace.PomeriumURLQueryCarrier(q)) dashboardURL.RawQuery = q.Encode() state.sessionStore.ClearSession(w, r) diff --git a/proxy/handlers_test.go b/proxy/handlers_test.go index 2817b22d0..b86ce1cb0 100644 --- a/proxy/handlers_test.go +++ b/proxy/handlers_test.go @@ -225,9 +225,8 @@ func TestProxy_jsonUserInfo(t *testing.T) { // The /.pomerium/jwt endpoint should be registered only if explicitly enabled. func TestProxy_registerDashboardHandlers_jwtEndpoint(t *testing.T) { - proxy := &Proxy{ - state: atomicutil.NewValue(&proxyState{}), - } + proxy, err := New(context.Background(), &config.Config{Options: config.NewDefaultOptions()}) + require.NoError(t, err) req := httptest.NewRequest(http.MethodGet, "/.pomerium/jwt", nil) rawJWT := "eyJ0eXAiOiJKV1QiLCJhbGciOiJub25lIn0.eyJzdWIiOiIxMjM0NTY3ODkwIn0." req.Header.Set("X-Pomerium-Jwt-Assertion", rawJWT) diff --git a/proxy/proxy.go b/proxy/proxy.go index 58f795860..ea58c63a6 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -10,6 +10,8 @@ import ( "net/http" "github.com/gorilla/mux" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + oteltrace "go.opentelemetry.io/otel/trace" "github.com/pomerium/pomerium/config" "github.com/pomerium/pomerium/internal/atomicutil" @@ -17,6 +19,7 @@ import ( "github.com/pomerium/pomerium/internal/httputil" "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/telemetry/metrics" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/pkg/cryptutil" ) @@ -56,17 +59,20 @@ type Proxy struct { currentOptions *atomicutil.Value[*config.Options] currentRouter *atomicutil.Value[*mux.Router] webauthn *webauthn.Handler + tracerProvider oteltrace.TracerProvider } // New takes a Proxy service from options and a validation function. // Function returns an error if options fail to validate. func New(ctx context.Context, cfg *config.Config) (*Proxy, error) { - state, err := newProxyStateFromConfig(ctx, cfg) + tracerProvider := trace.NewTracerProvider(ctx, "Proxy") + state, err := newProxyStateFromConfig(ctx, tracerProvider, cfg) if err != nil { return nil, err } p := &Proxy{ + tracerProvider: tracerProvider, state: atomicutil.NewValue(state), currentOptions: config.NewAtomicOptions(), currentRouter: atomicutil.NewValue(httputil.NewRouter()), @@ -96,7 +102,7 @@ func (p *Proxy) OnConfigChange(ctx context.Context, cfg *config.Config) { if err := p.setHandlers(ctx, cfg.Options); err != nil { log.Ctx(ctx).Error().Err(err).Msg("proxy: failed to update proxy handlers from configuration settings") } - if state, err := newProxyStateFromConfig(ctx, cfg); err != nil { + if state, err := newProxyStateFromConfig(ctx, p.tracerProvider, cfg); err != nil { log.Ctx(ctx).Error().Err(err).Msg("proxy: failed to update proxy state from configuration settings") } else { p.state.Store(state) @@ -115,6 +121,7 @@ func (p *Proxy) setHandlers(ctx context.Context, opts *config.Options) error { r.StrictSlash(true) // dashboard handlers are registered to all routes r = p.registerDashboardHandlers(r, opts) + r.Use(trace.NewHTTPMiddleware(otelhttp.WithTracerProvider(p.tracerProvider))) p.currentRouter.Store(r) return nil diff --git a/proxy/state.go b/proxy/state.go index 5a7727e13..c07e1c51b 100644 --- a/proxy/state.go +++ b/proxy/state.go @@ -9,6 +9,9 @@ import ( "github.com/pomerium/pomerium/internal/authenticateflow" "github.com/pomerium/pomerium/pkg/grpc" "github.com/pomerium/pomerium/pkg/grpc/databroker" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + oteltrace "go.opentelemetry.io/otel/trace" + googlegrpc "google.golang.org/grpc" ) var outboundGRPCConnection = new(grpc.CachedOutboundGRPClientConn) @@ -31,7 +34,7 @@ type proxyState struct { authenticateFlow authenticateFlow } -func newProxyStateFromConfig(ctx context.Context, cfg *config.Config) (*proxyState, error) { +func newProxyStateFromConfig(ctx context.Context, tracerProvider oteltrace.TracerProvider, cfg *config.Config) (*proxyState, error) { err := ValidateOptions(cfg.Options) if err != nil { return nil, err @@ -62,7 +65,7 @@ func newProxyStateFromConfig(ctx context.Context, cfg *config.Config) (*proxySta InstallationID: cfg.Options.InstallationID, ServiceName: cfg.Options.Services, SignedJWTKey: state.sharedKey, - }) + }, googlegrpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(tracerProvider)))) if err != nil { return nil, err } @@ -71,10 +74,10 @@ func newProxyStateFromConfig(ctx context.Context, cfg *config.Config) (*proxySta state.programmaticRedirectDomainWhitelist = cfg.Options.ProgrammaticRedirectDomainWhitelist if cfg.Options.UseStatelessAuthenticateFlow() { - state.authenticateFlow, err = authenticateflow.NewStateless(ctx, + state.authenticateFlow, err = authenticateflow.NewStateless(ctx, tracerProvider, cfg, state.sessionStore, nil, nil, nil) } else { - state.authenticateFlow, err = authenticateflow.NewStateful(ctx, cfg, state.sessionStore) + state.authenticateFlow, err = authenticateflow.NewStateful(ctx, tracerProvider, cfg, state.sessionStore) } if err != nil { return nil, err