diff --git a/authenticate/authenticate.go b/authenticate/authenticate.go index 79c6fdf01..7cf18f0ab 100644 --- a/authenticate/authenticate.go +++ b/authenticate/authenticate.go @@ -10,7 +10,9 @@ import ( "github.com/pomerium/pomerium/config" "github.com/pomerium/pomerium/internal/atomicutil" "github.com/pomerium/pomerium/internal/log" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/pkg/cryptutil" + oteltrace "go.opentelemetry.io/otel/trace" ) // ValidateOptions checks that configuration are complete and valid. @@ -38,23 +40,31 @@ func ValidateOptions(o *config.Options) error { // Authenticate contains data required to run the authenticate service. type Authenticate struct { - cfg *authenticateConfig - options *atomicutil.Value[*config.Options] - state *atomicutil.Value[*authenticateState] + cfg *authenticateConfig + options *atomicutil.Value[*config.Options] + state *atomicutil.Value[*authenticateState] + tracerProvider oteltrace.TracerProvider + tracer oteltrace.Tracer } // New validates and creates a new authenticate service from a set of Options. func New(ctx context.Context, cfg *config.Config, options ...Option) (*Authenticate, error) { authenticateConfig := getAuthenticateConfig(options...) + + tracerProvider := trace.NewTracerProvider(ctx, "Authenticate") + tracer := tracerProvider.Tracer(trace.PomeriumCoreTracer) + a := &Authenticate{ - cfg: authenticateConfig, - options: config.NewAtomicOptions(), - state: atomicutil.NewValue(newAuthenticateState()), + cfg: authenticateConfig, + options: config.NewAtomicOptions(), + state: atomicutil.NewValue(newAuthenticateState()), + tracerProvider: tracerProvider, + tracer: tracer, } a.options.Store(cfg.Options) - state, err := newAuthenticateStateFromConfig(ctx, cfg, authenticateConfig) + state, err := newAuthenticateStateFromConfig(ctx, tracerProvider, cfg, authenticateConfig) if err != nil { return nil, err } @@ -70,7 +80,7 @@ func (a *Authenticate) OnConfigChange(ctx context.Context, cfg *config.Config) { } a.options.Store(cfg.Options) - if state, err := newAuthenticateStateFromConfig(ctx, cfg, a.cfg); err != nil { + if state, err := newAuthenticateStateFromConfig(ctx, a.tracerProvider, cfg, a.cfg); err != nil { log.Ctx(ctx).Error().Err(err).Msg("authenticate: failed to update state") } else { a.state.Store(state) diff --git a/authenticate/config.go b/authenticate/config.go index ef293808b..11ce67bee 100644 --- a/authenticate/config.go +++ b/authenticate/config.go @@ -1,14 +1,17 @@ package authenticate import ( + "context" + "github.com/pomerium/pomerium/authenticate/events" "github.com/pomerium/pomerium/config" identitypb "github.com/pomerium/pomerium/pkg/grpc/identity" "github.com/pomerium/pomerium/pkg/identity" + oteltrace "go.opentelemetry.io/otel/trace" ) type authenticateConfig struct { - getIdentityProvider func(options *config.Options, idpID string) (identity.Authenticator, error) + getIdentityProvider func(ctx context.Context, tracerProvider oteltrace.TracerProvider, options *config.Options, idpID string) (identity.Authenticator, error) profileTrimFn func(*identitypb.Profile) authEventFn events.AuthEventFn } @@ -26,7 +29,7 @@ func getAuthenticateConfig(options ...Option) *authenticateConfig { } // WithGetIdentityProvider sets the getIdentityProvider function in the config. -func WithGetIdentityProvider(getIdentityProvider func(options *config.Options, idpID string) (identity.Authenticator, error)) Option { +func WithGetIdentityProvider(getIdentityProvider func(ctx context.Context, tracerProvider oteltrace.TracerProvider, options *config.Options, idpID string) (identity.Authenticator, error)) Option { return func(cfg *authenticateConfig) { cfg.getIdentityProvider = getIdentityProvider } diff --git a/authenticate/handlers.go b/authenticate/handlers.go index f394b3f1b..5f3418f6c 100644 --- a/authenticate/handlers.go +++ b/authenticate/handlers.go @@ -21,7 +21,6 @@ import ( "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/middleware" "github.com/pomerium/pomerium/internal/sessions" - "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/urlutil" "github.com/pomerium/pomerium/pkg/cryptutil" "github.com/pomerium/pomerium/pkg/identity" @@ -114,7 +113,7 @@ func (a *Authenticate) RetrieveSession(next http.Handler) http.Handler { // session state is attached to the users's request context. func (a *Authenticate) VerifySession(next http.Handler) http.Handler { return httputil.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { - ctx, span := trace.StartSpan(r.Context(), "authenticate.VerifySession") + ctx, span := a.tracer.Start(r.Context(), "authenticate.VerifySession") defer span.End() state := a.state.Load() @@ -160,7 +159,7 @@ func (a *Authenticate) RobotsTxt(w http.ResponseWriter, _ *http.Request) { // SignIn handles authenticating a user. func (a *Authenticate) SignIn(w http.ResponseWriter, r *http.Request) error { - ctx, span := trace.StartSpan(r.Context(), "authenticate.SignIn") + ctx, span := a.tracer.Start(r.Context(), "authenticate.SignIn") defer span.End() state := a.state.Load() @@ -197,13 +196,13 @@ func (a *Authenticate) SignOut(w http.ResponseWriter, r *http.Request) error { } func (a *Authenticate) signOutRedirect(w http.ResponseWriter, r *http.Request) error { - ctx, span := trace.StartSpan(r.Context(), "authenticate.SignOut") + ctx, span := a.tracer.Start(r.Context(), "authenticate.SignOut") defer span.End() options := a.options.Load() idpID := a.getIdentityProviderIDForRequest(r) - authenticator, err := a.cfg.getIdentityProvider(options, idpID) + authenticator, err := a.cfg.getIdentityProvider(ctx, a.tracerProvider, options, idpID) if err != nil { return err } @@ -274,7 +273,7 @@ func (a *Authenticate) reauthenticateOrFail(w http.ResponseWriter, r *http.Reque options := a.options.Load() idpID := a.getIdentityProviderIDForRequest(r) - authenticator, err := a.cfg.getIdentityProvider(options, idpID) + authenticator, err := a.cfg.getIdentityProvider(r.Context(), a.tracerProvider, options, idpID) if err != nil { return err } @@ -307,6 +306,10 @@ func (a *Authenticate) OAuthCallback(w http.ResponseWriter, r *http.Request) err if err != nil { return fmt.Errorf("authenticate.OAuthCallback: %w", err) } + q := redirect.Query() + if traceparent := q.Get(urlutil.QueryTraceparent); traceparent != "" { + w.Header().Set("X-Pomerium-Traceparent", traceparent) + } httputil.Redirect(w, r, redirect.String(), http.StatusFound) return nil } @@ -321,7 +324,7 @@ func (a *Authenticate) statusForErrorCode(errorCode string) int { } func (a *Authenticate) getOAuthCallback(w http.ResponseWriter, r *http.Request) (*url.URL, error) { - ctx, span := trace.StartSpan(r.Context(), "authenticate.getOAuthCallback") + ctx, span := a.tracer.Start(r.Context(), "authenticate.getOAuthCallback") defer span.End() state := a.state.Load() @@ -380,7 +383,7 @@ Or contact your administrator. idpID := state.flow.GetIdentityProviderIDForURLValues(redirectURL.Query()) - authenticator, err := a.cfg.getIdentityProvider(options, idpID) + authenticator, err := a.cfg.getIdentityProvider(ctx, a.tracerProvider, options, idpID) if err != nil { return nil, err } @@ -432,7 +435,7 @@ func (a *Authenticate) getSessionFromCtx(ctx context.Context) (*sessions.State, } func (a *Authenticate) userInfo(w http.ResponseWriter, r *http.Request) error { - ctx, span := trace.StartSpan(r.Context(), "authenticate.userInfo") + ctx, span := a.tracer.Start(r.Context(), "authenticate.userInfo") defer span.End() options := a.options.Load() @@ -484,7 +487,7 @@ func (a *Authenticate) revokeSession(ctx context.Context, w http.ResponseWriter, idpID := r.FormValue(urlutil.QueryIdentityProviderID) - authenticator, err := a.cfg.getIdentityProvider(options, idpID) + authenticator, err := a.cfg.getIdentityProvider(ctx, a.tracerProvider, options, idpID) if err != nil { return "" } diff --git a/authenticate/handlers_test.go b/authenticate/handlers_test.go index c8f5342a5..4dacd372a 100644 --- a/authenticate/handlers_test.go +++ b/authenticate/handlers_test.go @@ -16,6 +16,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + oteltrace "go.opentelemetry.io/otel/trace" "go.uber.org/mock/gomock" "golang.org/x/crypto/chacha20poly1305" "golang.org/x/oauth2" @@ -39,23 +40,21 @@ import ( "github.com/pomerium/pomerium/pkg/identity/oidc" ) -func testAuthenticate() *Authenticate { - redirectURL, _ := url.Parse("https://auth.example.com/oauth/callback") - var auth Authenticate - auth.state = atomicutil.NewValue(&authenticateState{ - redirectURL: redirectURL, - cookieSecret: cryptutil.NewKey(), - flow: new(stubFlow), +func testAuthenticate(t *testing.T) *Authenticate { + opts := newTestOptions(t) + opts.AuthenticateURLString = "https://auth.example.com/oauth/callback" + auth, err := New(context.Background(), &config.Config{ + Options: opts, }) - auth.options = config.NewAtomicOptions() - auth.options.Store(&config.Options{ - SharedKey: cryptutil.NewBase64Key(), - }) - return &auth + if err != nil { + panic(err) + } + auth.state.Load().flow = new(stubFlow) + return auth } func TestAuthenticate_RobotsTxt(t *testing.T) { - auth := testAuthenticate() + auth := testAuthenticate(t) req, err := http.NewRequest(http.MethodGet, "/robots.txt", nil) if err != nil { t.Fatal(err) @@ -73,7 +72,7 @@ func TestAuthenticate_RobotsTxt(t *testing.T) { } func TestAuthenticate_Handler(t *testing.T) { - auth := testAuthenticate() + auth := testAuthenticate(t) h := auth.Handler() if h == nil { @@ -224,17 +223,16 @@ func TestAuthenticate_SignOut(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - a := &Authenticate{ - cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ *config.Options, _ string) (identity.Authenticator, error) { - return tt.provider, nil - })), - state: atomicutil.NewValue(&authenticateState{ - sessionStore: tt.sessionStore, - sharedEncoder: mock.Encoder{}, - flow: new(stubFlow), - }), - options: config.NewAtomicOptions(), - } + a := testAuthenticate(t) + a.cfg = getAuthenticateConfig(WithGetIdentityProvider(func(_ context.Context, _ oteltrace.TracerProvider, _ *config.Options, _ string) (identity.Authenticator, error) { + return tt.provider, nil + })) + a.state = atomicutil.NewValue(&authenticateState{ + sessionStore: tt.sessionStore, + sharedEncoder: mock.Encoder{}, + flow: new(stubFlow), + }) + a.options = config.NewAtomicOptions() if tt.signoutRedirectURL != "" { opts := a.options.Load() opts.SignOutRedirectURLString = tt.signoutRedirectURL @@ -280,7 +278,7 @@ func TestAuthenticate_SignOutDoesNotRequireSession(t *testing.T) { sessionStore := &mstore.Store{LoadError: errors.New("no session")} a := &Authenticate{ - cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ *config.Options, _ string) (identity.Authenticator, error) { + cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ context.Context, _ oteltrace.TracerProvider, _ *config.Options, _ string) (identity.Authenticator, error) { return identity.MockProvider{}, nil })), state: atomicutil.NewValue(&authenticateState{ @@ -354,18 +352,17 @@ func TestAuthenticate_OAuthCallback(t *testing.T) { t.Fatal(err) } authURL, _ := url.Parse(tt.authenticateURL) - a := &Authenticate{ - cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ *config.Options, _ string) (identity.Authenticator, error) { - return tt.provider, nil - })), - state: atomicutil.NewValue(&authenticateState{ - redirectURL: authURL, - sessionStore: tt.session, - cookieCipher: aead, - flow: new(stubFlow), - }), - options: config.NewAtomicOptions(), - } + a := testAuthenticate(t) + a.cfg = getAuthenticateConfig(WithGetIdentityProvider(func(_ context.Context, _ oteltrace.TracerProvider, _ *config.Options, _ string) (identity.Authenticator, error) { + return tt.provider, nil + })) + a.state = atomicutil.NewValue(&authenticateState{ + redirectURL: authURL, + sessionStore: tt.session, + cookieCipher: aead, + flow: new(stubFlow), + }) + a.options = config.NewAtomicOptions() u, _ := url.Parse("/oauthGet") params, _ := url.ParseQuery(u.RawQuery) params.Add("error", tt.paramErr) @@ -466,20 +463,19 @@ func TestAuthenticate_SessionValidatorMiddleware(t *testing.T) { if err != nil { t.Fatal(err) } - a := &Authenticate{ - cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ *config.Options, _ string) (identity.Authenticator, error) { - return tt.provider, nil - })), - state: atomicutil.NewValue(&authenticateState{ - cookieSecret: cryptutil.NewKey(), - redirectURL: uriParseHelper("https://authenticate.corp.beyondperimeter.com"), - sessionStore: tt.session, - cookieCipher: aead, - sharedEncoder: signer, - flow: new(stubFlow), - }), - options: config.NewAtomicOptions(), - } + a := testAuthenticate(t) + a.cfg = getAuthenticateConfig(WithGetIdentityProvider(func(_ context.Context, _ oteltrace.TracerProvider, _ *config.Options, _ string) (identity.Authenticator, error) { + return tt.provider, nil + })) + a.state = atomicutil.NewValue(&authenticateState{ + cookieSecret: cryptutil.NewKey(), + redirectURL: uriParseHelper("https://authenticate.corp.beyondperimeter.com"), + sessionStore: tt.session, + cookieCipher: aead, + sharedEncoder: signer, + flow: new(stubFlow), + }) + a.options = config.NewAtomicOptions() r := httptest.NewRequest(http.MethodGet, "/", nil) state, err := tt.session.LoadSession(r) if err != nil { @@ -512,7 +508,7 @@ func TestAuthenticate_userInfo(t *testing.T) { t.Run("cookie-redirect-uri", func(t *testing.T) { w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "https://authenticate.service.cluster.local/.pomerium/?pomerium_redirect_uri=https://www.example.com", nil) - var a Authenticate + a := testAuthenticate(t) a.state = atomicutil.NewValue(&authenticateState{ cookieSecret: cryptutil.NewKey(), flow: new(stubFlow), @@ -577,14 +573,13 @@ func TestAuthenticate_userInfo(t *testing.T) { if !tt.validSignature { f.verifySignatureErr = errors.New("bad signature") } - a := &Authenticate{ - options: o, - state: atomicutil.NewValue(&authenticateState{ - sessionStore: tt.sessionStore, - sharedEncoder: signer, - flow: f, - }), - } + a := testAuthenticate(t) + a.options = o + a.state = atomicutil.NewValue(&authenticateState{ + sessionStore: tt.sessionStore, + sharedEncoder: signer, + flow: f, + }) r := httptest.NewRequest(http.MethodGet, tt.url, nil) state, err := tt.sessionStore.LoadSession(r) if err != nil { @@ -606,7 +601,7 @@ func TestAuthenticate_userInfo(t *testing.T) { func TestAuthenticate_CORS(t *testing.T) { f := new(stubFlow) - auth := testAuthenticate() + auth := testAuthenticate(t) state := auth.state.Load() state.sessionLoader = &mstore.Store{Session: &sessions.State{}} state.sharedEncoder = mock.Encoder{} @@ -645,7 +640,7 @@ func TestAuthenticate_CORS(t *testing.T) { func TestSignOutBranding(t *testing.T) { t.Parallel() - auth := testAuthenticate() + auth := testAuthenticate(t) auth.state.Load().flow.(*stubFlow).verifySignatureErr = errors.New("unsigned URL") auth.options.Store(&config.Options{ BrandingOptions: &configproto.Settings{ diff --git a/authenticate/identity.go b/authenticate/identity.go index 8ea432e15..1ff0efe79 100644 --- a/authenticate/identity.go +++ b/authenticate/identity.go @@ -1,13 +1,16 @@ package authenticate import ( + "context" + "github.com/pomerium/pomerium/config" "github.com/pomerium/pomerium/internal/urlutil" "github.com/pomerium/pomerium/pkg/identity" "github.com/pomerium/pomerium/pkg/identity/oauth" + oteltrace "go.opentelemetry.io/otel/trace" ) -func defaultGetIdentityProvider(options *config.Options, idpID string) (identity.Authenticator, error) { +func defaultGetIdentityProvider(ctx context.Context, tracerProvider oteltrace.TracerProvider, options *config.Options, idpID string) (identity.Authenticator, error) { authenticateURL, err := options.GetAuthenticateURL() if err != nil { return nil, err @@ -23,7 +26,7 @@ func defaultGetIdentityProvider(options *config.Options, idpID string) (identity if err != nil { return nil, err } - return identity.NewAuthenticator(oauth.Options{ + return identity.NewAuthenticator(ctx, tracerProvider, oauth.Options{ RedirectURL: redirectURL, ProviderName: idp.GetType(), ProviderURL: idp.GetUrl(), diff --git a/authenticate/state.go b/authenticate/state.go index 3680f0f46..522a0e842 100644 --- a/authenticate/state.go +++ b/authenticate/state.go @@ -8,6 +8,7 @@ import ( "net/url" "github.com/go-jose/go-jose/v3" + oteltrace "go.opentelemetry.io/otel/trace" "golang.org/x/oauth2" "github.com/pomerium/pomerium/config" @@ -65,7 +66,9 @@ func newAuthenticateState() *authenticateState { func newAuthenticateStateFromConfig( ctx context.Context, - cfg *config.Config, authenticateConfig *authenticateConfig, + tracerProvider oteltrace.TracerProvider, + cfg *config.Config, + authenticateConfig *authenticateConfig, ) (*authenticateState, error) { err := ValidateOptions(cfg.Options) if err != nil { @@ -147,6 +150,7 @@ func newAuthenticateStateFromConfig( if cfg.Options.UseStatelessAuthenticateFlow() { state.flow, err = authenticateflow.NewStateless(ctx, + tracerProvider, cfg, cookieStore, authenticateConfig.getIdentityProvider, @@ -154,7 +158,7 @@ func newAuthenticateStateFromConfig( authenticateConfig.authEventFn, ) } else { - state.flow, err = authenticateflow.NewStateful(ctx, cfg, cookieStore) + state.flow, err = authenticateflow.NewStateful(ctx, tracerProvider, cfg, cookieStore) } if err != nil { return nil, err diff --git a/authorize/authorize.go b/authorize/authorize.go index 5c608b1cb..4cd3dd496 100644 --- a/authorize/authorize.go +++ b/authorize/authorize.go @@ -21,6 +21,7 @@ import ( "github.com/pomerium/pomerium/pkg/cryptutil" "github.com/pomerium/pomerium/pkg/grpc/databroker" "github.com/pomerium/pomerium/pkg/storage" + oteltrace "go.opentelemetry.io/otel/trace" ) // Authorize struct holds @@ -35,18 +36,25 @@ type Authorize struct { // This should provide a consistent view of the data at a given server/record version and // avoid partial updates. stateLock sync.RWMutex + + tracerProvider oteltrace.TracerProvider + tracer oteltrace.Tracer } // New validates and creates a new Authorize service from a set of config options. func New(ctx context.Context, cfg *config.Config) (*Authorize, error) { + tracerProvider := trace.NewTracerProvider(ctx, "Authorize") + tracer := tracerProvider.Tracer(trace.PomeriumCoreTracer) a := &Authorize{ currentOptions: config.NewAtomicOptions(), store: store.New(), globalCache: storage.NewGlobalCache(time.Minute), + tracerProvider: tracerProvider, + tracer: tracer, } a.accessTracker = NewAccessTracker(a, accessTrackerMaxSize, accessTrackerDebouncePeriod) - state, err := newAuthorizeStateFromConfig(ctx, cfg, a.store, nil) + state, err := newAuthorizeStateFromConfig(ctx, tracerProvider, cfg, a.store, nil) if err != nil { return nil, err } @@ -88,7 +96,7 @@ func newPolicyEvaluator( ctx = log.WithContext(ctx, func(c zerolog.Context) zerolog.Context { return c.Str("service", "authorize") }) - ctx, span := trace.StartSpan(ctx, "authorize.newPolicyEvaluator") + ctx, span := trace.Continue(ctx, "authorize.newPolicyEvaluator") defer span.End() clientCA, err := opts.DownstreamMTLS.GetCA() @@ -142,7 +150,7 @@ func newPolicyEvaluator( func (a *Authorize) OnConfigChange(ctx context.Context, cfg *config.Config) { currentState := a.state.Load() a.currentOptions.Store(cfg.Options) - if state, err := newAuthorizeStateFromConfig(ctx, cfg, a.store, currentState.evaluator); err != nil { + if state, err := newAuthorizeStateFromConfig(ctx, a.tracerProvider, cfg, a.store, currentState.evaluator); err != nil { log.Ctx(ctx).Error().Err(err).Msg("authorize: error updating state") } else { a.state.Store(state) diff --git a/authorize/check_response.go b/authorize/check_response.go index 6a0b8b25d..816a6bb3b 100644 --- a/authorize/check_response.go +++ b/authorize/check_response.go @@ -228,16 +228,24 @@ func (a *Authorize) requireLoginResponse( // always assume https scheme checkRequestURL := getCheckRequestURL(in) checkRequestURL.Scheme = "https" + var signInURLQuery url.Values + headers := http.Header{} + if id := in.GetAttributes().GetRequest().GetHttp().GetHeaders()["traceparent"]; id != "" { + headers["X-Pomerium-Traceparent"] = []string{id} + headers["X-Pomerium-Tracestate"] = []string{"pomerium.traceparent=" + id} // TODO: this might not be necessary anymore + signInURLQuery = url.Values{} + signInURLQuery.Add("pomerium_traceparent", id) + signInURLQuery.Add("pomerium_tracestate", "pomerium.traceparent="+id) + } redirectTo, err := state.authenticateFlow.AuthenticateSignInURL( - ctx, nil, &checkRequestURL, idp.GetId()) + ctx, signInURLQuery, &checkRequestURL, idp.GetId()) if err != nil { return nil, err } + headers["Location"] = []string{redirectTo} - return a.deniedResponse(ctx, in, http.StatusFound, "Login", http.Header{ - "Location": {redirectTo}, - }) + return a.deniedResponse(ctx, in, http.StatusFound, "Login", headers) } func (a *Authorize) requireWebAuthnResponse( diff --git a/authorize/databroker.go b/authorize/databroker.go index de7535752..e82550522 100644 --- a/authorize/databroker.go +++ b/authorize/databroker.go @@ -3,7 +3,6 @@ package authorize import ( "context" - "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/pkg/grpc/databroker" "github.com/pomerium/pomerium/pkg/grpc/session" "github.com/pomerium/pomerium/pkg/grpc/user" @@ -63,7 +62,7 @@ func (a *Authorize) getDataBrokerSessionOrServiceAccount( sessionID string, dataBrokerRecordVersion uint64, ) (s sessionOrServiceAccount, err error) { - ctx, span := trace.StartSpan(ctx, "authorize.getDataBrokerSessionOrServiceAccount") + ctx, span := a.tracer.Start(ctx, "authorize.getDataBrokerSessionOrServiceAccount") defer span.End() record, err := getDataBrokerRecord(ctx, grpcutil.GetTypeURL(new(session.Session)), sessionID, dataBrokerRecordVersion) @@ -96,7 +95,7 @@ func (a *Authorize) getDataBrokerUser( ctx context.Context, userID string, ) (*user.User, error) { - ctx, span := trace.StartSpan(ctx, "authorize.getDataBrokerUser") + ctx, span := a.tracer.Start(ctx, "authorize.getDataBrokerUser") defer span.End() record, err := getDataBrokerRecord(ctx, grpcutil.GetTypeURL(new(user.User)), userID, 0) diff --git a/authorize/evaluator/evaluator.go b/authorize/evaluator/evaluator.go index 726bd8b8f..1523cf04c 100644 --- a/authorize/evaluator/evaluator.go +++ b/authorize/evaluator/evaluator.go @@ -199,7 +199,7 @@ func getOrCreatePolicyEvaluators( // Evaluate evaluates the rego for the given policy and generates the identity headers. func (e *Evaluator) Evaluate(ctx context.Context, req *Request) (*Result, error) { - ctx, span := trace.StartSpan(ctx, "authorize.Evaluator.Evaluate") + ctx, span := trace.Continue(ctx, "authorize.Evaluator.Evaluate") defer span.End() eg, ctx := errgroup.WithContext(ctx) diff --git a/authorize/evaluator/headers_evaluator.go b/authorize/evaluator/headers_evaluator.go index e5a706917..7ed1baeb6 100644 --- a/authorize/evaluator/headers_evaluator.go +++ b/authorize/evaluator/headers_evaluator.go @@ -30,7 +30,7 @@ func NewHeadersEvaluator(store *store.Store) *HeadersEvaluator { // Evaluate evaluates the headers.rego script. func (e *HeadersEvaluator) Evaluate(ctx context.Context, req *Request, options ...rego.EvalOption) (*HeadersResponse, error) { - ctx, span := trace.StartSpan(ctx, "authorize.HeadersEvaluator.Evaluate") + ctx, span := trace.Continue(ctx, "authorize.HeadersEvaluator.Evaluate") defer span.End() ectx := new(rego.EvalContext) diff --git a/authorize/evaluator/policy_evaluator.go b/authorize/evaluator/policy_evaluator.go index a0b7fef96..56a4f228d 100644 --- a/authorize/evaluator/policy_evaluator.go +++ b/authorize/evaluator/policy_evaluator.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/open-policy-agent/opa/rego" - octrace "go.opencensus.io/trace" + "go.opentelemetry.io/otel/attribute" "github.com/pomerium/pomerium/authorize/internal/store" "github.com/pomerium/pomerium/config" @@ -209,9 +209,9 @@ func (e *PolicyEvaluator) Evaluate(ctx context.Context, req *PolicyRequest) (*Po } func (e *PolicyEvaluator) evaluateQuery(ctx context.Context, req *PolicyRequest, query policyQuery) (*PolicyResponse, error) { - ctx, span := trace.StartSpan(ctx, "authorize.PolicyEvaluator.evaluateQuery") + ctx, span := trace.Continue(ctx, "authorize.PolicyEvaluator.evaluateQuery") defer span.End() - span.AddAttributes(octrace.StringAttribute("script_checksum", query.checksum())) + span.SetAttributes(attribute.String("script_checksum", query.checksum())) rs, err := safeEval(ctx, query.PreparedEvalQuery, rego.EvalInput(req), diff --git a/authorize/grpc.go b/authorize/grpc.go index 3324a2177..8172226f9 100644 --- a/authorize/grpc.go +++ b/authorize/grpc.go @@ -19,7 +19,6 @@ import ( "github.com/pomerium/pomerium/internal/httputil" "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/sessions" - "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/urlutil" "github.com/pomerium/pomerium/pkg/contextutil" "github.com/pomerium/pomerium/pkg/grpc/user" @@ -29,7 +28,7 @@ import ( // Check implements the envoy auth server gRPC endpoint. func (a *Authorize) Check(ctx context.Context, in *envoy_service_auth_v3.CheckRequest) (*envoy_service_auth_v3.CheckResponse, error) { - ctx, span := trace.StartSpan(ctx, "authorize.grpc.Check") + ctx, span := a.tracer.Start(ctx, "authorize.grpc.Check") defer span.End() querier := storage.NewTracingQuerier( diff --git a/authorize/internal/store/store.go b/authorize/internal/store/store.go index 5c32257a7..6f651e645 100644 --- a/authorize/internal/store/store.go +++ b/authorize/internal/store/store.go @@ -14,7 +14,7 @@ import ( opastorage "github.com/open-policy-agent/opa/storage" "github.com/open-policy-agent/opa/storage/inmem" "github.com/open-policy-agent/opa/types" - octrace "go.opencensus.io/trace" + "go.opentelemetry.io/otel/attribute" "google.golang.org/grpc" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" @@ -145,20 +145,20 @@ func (s *Store) GetDataBrokerRecordOption() func(*rego.Rego) { types.NewObject(nil, types.NewDynamicProperty(types.S, types.S)), ), }, func(bctx rego.BuiltinContext, op1 *ast.Term, op2 *ast.Term) (*ast.Term, error) { - ctx, span := trace.StartSpan(bctx.Context, "rego.get_databroker_record") + ctx, span := trace.Continue(bctx.Context, "rego.get_databroker_record") defer span.End() recordType, ok := op1.Value.(ast.String) if !ok { return nil, fmt.Errorf("invalid record type: %T", op1) } - span.AddAttributes(octrace.StringAttribute("record_type", recordType.String())) + span.SetAttributes(attribute.String("record_type", recordType.String())) recordIDOrIndex, ok := op2.Value.(ast.String) if !ok { return nil, fmt.Errorf("invalid record id: %T", op2) } - span.AddAttributes(octrace.StringAttribute("record_id", recordIDOrIndex.String())) + span.SetAttributes(attribute.String("record_id", recordIDOrIndex.String())) msg := s.GetDataBrokerRecord(ctx, string(recordType), string(recordIDOrIndex)) if msg == nil { diff --git a/authorize/log.go b/authorize/log.go index 594689245..3e5088262 100644 --- a/authorize/log.go +++ b/authorize/log.go @@ -7,10 +7,10 @@ import ( envoy_service_auth_v3 "github.com/envoyproxy/go-control-plane/envoy/service/auth/v3" "github.com/go-jose/go-jose/v3/jwt" "github.com/rs/zerolog" + "go.opentelemetry.io/otel/attribute" "github.com/pomerium/pomerium/authorize/evaluator" "github.com/pomerium/pomerium/internal/log" - "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/pkg/grpc/databroker" "github.com/pomerium/pomerium/pkg/grpc/session" "github.com/pomerium/pomerium/pkg/grpc/user" @@ -24,7 +24,7 @@ func (a *Authorize) logAuthorizeCheck( in *envoy_service_auth_v3.CheckRequest, res *evaluator.Result, s sessionOrServiceAccount, u *user.User, ) { - ctx, span := trace.StartSpan(ctx, "authorize.grpc.LogAuthorizeCheck") + ctx, span := a.tracer.Start(ctx, "authorize.grpc.LogAuthorizeCheck") defer span.End() hdrs := getCheckRequestHeaders(in) @@ -39,17 +39,24 @@ func (a *Authorize) logAuthorizeCheck( // result if res != nil { + span.SetAttributes(attribute.Bool("result.allow", res.Allow.Value)) evt = evt.Bool("allow", res.Allow.Value) + allowReasons := res.Allow.Reasons.Strings() if res.Allow.Value { - evt = evt.Strs("allow-why-true", res.Allow.Reasons.Strings()) + span.SetAttributes(attribute.StringSlice("result.allow-why-true", allowReasons)) + evt = evt.Strs("allow-why-true", allowReasons) } else { - evt = evt.Strs("allow-why-false", res.Allow.Reasons.Strings()) + span.SetAttributes(attribute.StringSlice("result.allow-why-false", allowReasons)) + evt = evt.Strs("allow-why-false", allowReasons) } evt = evt.Bool("deny", res.Deny.Value) + denyReasons := res.Deny.Reasons.Strings() if res.Deny.Value { - evt = evt.Strs("deny-why-true", res.Deny.Reasons.Strings()) + span.SetAttributes(attribute.StringSlice("result.deny-why-true", denyReasons)) + evt = evt.Strs("deny-why-true", denyReasons) } else { - evt = evt.Strs("deny-why-false", res.Deny.Reasons.Strings()) + span.SetAttributes(attribute.StringSlice("result.deny-why-false", denyReasons)) + evt = evt.Strs("deny-why-false", denyReasons) } } diff --git a/authorize/state.go b/authorize/state.go index ec82e8c57..10cfb4195 100644 --- a/authorize/state.go +++ b/authorize/state.go @@ -5,6 +5,8 @@ import ( "fmt" "net/url" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + oteltrace "go.opentelemetry.io/otel/trace" googlegrpc "google.golang.org/grpc" "github.com/pomerium/pomerium/authorize/evaluator" @@ -32,7 +34,10 @@ type authorizeState struct { func newAuthorizeStateFromConfig( ctx context.Context, - cfg *config.Config, store *store.Store, previousPolicyEvaluator *evaluator.Evaluator, + tracerProvider oteltrace.TracerProvider, + cfg *config.Config, + store *store.Store, + previousPolicyEvaluator *evaluator.Evaluator, ) (*authorizeState, error) { if err := validateOptions(cfg.Options); err != nil { return nil, fmt.Errorf("authorize: bad options: %w", err) @@ -62,7 +67,7 @@ func newAuthorizeStateFromConfig( InstallationID: cfg.Options.InstallationID, ServiceName: cfg.Options.Services, SignedJWTKey: sharedKey, - }) + }, googlegrpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(tracerProvider)))) if err != nil { return nil, fmt.Errorf("authorize: error creating databroker connection: %w", err) } @@ -75,9 +80,9 @@ func newAuthorizeStateFromConfig( } if cfg.Options.UseStatelessAuthenticateFlow() { - state.authenticateFlow, err = authenticateflow.NewStateless(ctx, cfg, nil, nil, nil, nil) + state.authenticateFlow, err = authenticateflow.NewStateless(ctx, tracerProvider, cfg, nil, nil, nil, nil) } else { - state.authenticateFlow, err = authenticateflow.NewStateful(ctx, cfg, nil) + state.authenticateFlow, err = authenticateflow.NewStateful(ctx, tracerProvider, cfg, nil) } if err != nil { return nil, err diff --git a/cmd/pomerium/main.go b/cmd/pomerium/main.go index 9c035e8fa..0d2869bea 100644 --- a/cmd/pomerium/main.go +++ b/cmd/pomerium/main.go @@ -12,6 +12,7 @@ import ( "github.com/pomerium/pomerium/config" "github.com/pomerium/pomerium/internal/log" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/version" _ "github.com/pomerium/pomerium/internal/zero/bootstrap/writers/filesystem" _ "github.com/pomerium/pomerium/internal/zero/bootstrap/writers/k8s" @@ -30,9 +31,15 @@ func main() { } root.AddCommand(zero_cmd.BuildRootCmd()) root.PersistentFlags().StringVar(&configFile, "config", "", "Specify configuration file location") - - ctx := context.Background() log.SetLevel(zerolog.InfoLevel) + ctx := trace.Options{ + RemoteClient: trace.NewSyncClient(trace.NewRemoteClientFromEnv()), + }.NewContext(context.Background()) + defer func() { + if err := trace.ShutdownContext(ctx); err != nil { + log.Error().Err(err).Send() + } + }() runFn := run if zero_cmd.IsManagedMode(configFile) { runFn = zero_cmd.Run diff --git a/config/envoyconfig/bootstrap.go b/config/envoyconfig/bootstrap.go index d1dbbd8f9..665f216fb 100644 --- a/config/envoyconfig/bootstrap.go +++ b/config/envoyconfig/bootstrap.go @@ -3,6 +3,7 @@ package envoyconfig import ( "context" "fmt" + "math" "os" "path/filepath" @@ -37,7 +38,7 @@ func (b *Builder) BuildBootstrap( cfg *config.Config, fullyStatic bool, ) (bootstrap *envoy_config_bootstrap_v3.Bootstrap, err error) { - ctx, span := trace.StartSpan(ctx, "envoyconfig.Builder.BuildBootstrap") + ctx, span := trace.Continue(ctx, "envoyconfig.Builder.BuildBootstrap") defer span.End() bootstrap = new(envoy_config_bootstrap_v3.Bootstrap) @@ -52,7 +53,7 @@ func (b *Builder) BuildBootstrap( return nil, fmt.Errorf("error building bootstrap dynamic resources: %w", err) } - bootstrap.LayeredRuntime, err = b.BuildBootstrapLayeredRuntime() + bootstrap.LayeredRuntime, err = b.BuildBootstrapLayeredRuntime(ctx) if err != nil { return nil, fmt.Errorf("error building bootstrap layered runtime: %w", err) } @@ -148,7 +149,13 @@ func (b *Builder) BuildBootstrapDynamicResources( } // BuildBootstrapLayeredRuntime builds the layered runtime for the envoy bootstrap. -func (b *Builder) BuildBootstrapLayeredRuntime() (*envoy_config_bootstrap_v3.LayeredRuntime, error) { +func (b *Builder) BuildBootstrapLayeredRuntime(ctx context.Context) (*envoy_config_bootstrap_v3.LayeredRuntime, error) { + flushIntervalMs := 5000 + minFlushSpans := 3 + if trace.DebugFlagsFromContext(ctx).Check(trace.EnvoyFlushEverySpan) { + minFlushSpans = 1 + flushIntervalMs = math.MaxInt32 + } layer, err := structpb.NewStruct(map[string]any{ "re2": map[string]any{ "max_program_size": map[string]any{ @@ -156,6 +163,21 @@ func (b *Builder) BuildBootstrapLayeredRuntime() (*envoy_config_bootstrap_v3.Lay "warn_level": 1024, }, }, + "tracing": map[string]any{ + "opentelemetry": map[string]any{ + "flush_interval_ms": flushIntervalMs, + // For most requests, envoy generates 3 spans: + // - ingress (downstream->envoy) + // - ext_authz check request (envoy->pomerium) + // - egress (envoy->upstream) + // The default value is 5, which usually leads to delayed exports. + // This can be set lower, e.g. 1 to have envoy export every span + // individually (useful for testing), but 3 is a reasonable default. + // If set to 1, also set flush_interval_ms to a very large number to + // effectively disable it. + "min_flush_spans": minFlushSpans, + }, + }, }) if err != nil { return nil, fmt.Errorf("envoyconfig: failed to create layered runtime layer: %w", err) @@ -180,7 +202,7 @@ func (b *Builder) BuildBootstrapStaticResources( cfg *config.Config, fullyStatic bool, ) (staticResources *envoy_config_bootstrap_v3.Bootstrap_StaticResources, err error) { - ctx, span := trace.StartSpan(ctx, "envoyconfig.Builder.BuildBootstrapStaticResources") + ctx, span := trace.Continue(ctx, "envoyconfig.Builder.BuildBootstrapStaticResources") defer span.End() staticResources = new(envoy_config_bootstrap_v3.Bootstrap_StaticResources) diff --git a/config/envoyconfig/bootstrap_test.go b/config/envoyconfig/bootstrap_test.go index bdceb7945..ec27b7fe7 100644 --- a/config/envoyconfig/bootstrap_test.go +++ b/config/envoyconfig/bootstrap_test.go @@ -36,7 +36,7 @@ func TestBuilder_BuildBootstrapAdmin(t *testing.T) { func TestBuilder_BuildBootstrapLayeredRuntime(t *testing.T) { b := New("localhost:1111", "localhost:2222", "localhost:3333", filemgr.NewManager(), nil) - staticCfg, err := b.BuildBootstrapLayeredRuntime() + staticCfg, err := b.BuildBootstrapLayeredRuntime(context.Background()) assert.NoError(t, err) testutil.AssertProtoJSONEqual(t, ` { "layers": [{ @@ -47,6 +47,12 @@ func TestBuilder_BuildBootstrapLayeredRuntime(t *testing.T) { "error_level": 1048576, "warn_level": 1024 } + }, + "tracing": { + "opentelemetry": { + "flush_interval_ms": 5000, + "min_flush_spans": 3 + } } } }] } diff --git a/config/envoyconfig/clusters.go b/config/envoyconfig/clusters.go index d0d1fd7f9..3fa9d2f8f 100644 --- a/config/envoyconfig/clusters.go +++ b/config/envoyconfig/clusters.go @@ -26,7 +26,7 @@ import ( // BuildClusters builds envoy clusters from the given config. func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*envoy_config_cluster_v3.Cluster, error) { - ctx, span := trace.StartSpan(ctx, "envoyconfig.Builder.BuildClusters") + ctx, span := trace.Continue(ctx, "envoyconfig.Builder.BuildClusters") defer span.End() grpcURLs := []*url.URL{{ @@ -104,13 +104,6 @@ func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*env envoyAdminCluster, } - tracingCluster, err := buildTracingCluster(cfg.Options) - if err != nil { - return nil, err - } else if tracingCluster != nil { - clusters = append(clusters, tracingCluster) - } - if config.IsProxy(cfg.Options.Services) { for policy := range cfg.Options.GetAllPolicies() { if len(policy.To) > 0 { @@ -444,6 +437,16 @@ func grpcHealthChecks(name string) []*envoy_config_core_v3.HealthCheck { ServiceName: name, }, }, + // EventLogger: []*envoy_config_core_v3.TypedExtensionConfig{ + // { + // Name: "envoy.health_check.event_sink.file", + // TypedConfig: marshalAny(&envoy_extensions_eventsinks_file_v3.HealthCheckEventFileSink{ + // EventLogPath: "/tmp/healthchecks", + // }), + // }, + // }, + // AlwaysLogHealthCheckFailures: true, + // AlwaysLogHealthCheckSuccess: true, }} } diff --git a/config/envoyconfig/extensions/doc.go b/config/envoyconfig/extensions/doc.go new file mode 100644 index 000000000..24b6b60a6 --- /dev/null +++ b/config/envoyconfig/extensions/doc.go @@ -0,0 +1,5 @@ +// Package extensions contains the protobuf config definitions for Pomerium's +// custom envoy extensions. Because the required configuration is minimal, +// identical copies of the proto files are stored separately in envoy-custom. +// Be sure to keep these in sync if the definitions change. +package extensions diff --git a/config/envoyconfig/extensions/trace_context.pb.go b/config/envoyconfig/extensions/trace_context.pb.go new file mode 100644 index 000000000..d70fdbb32 --- /dev/null +++ b/config/envoyconfig/extensions/trace_context.pb.go @@ -0,0 +1,125 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.35.2 +// protoc (unknown) +// source: github.com/pomerium/pomerium/config/envoyconfig/extensions/trace_context.proto + +package extensions + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type TraceContext struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *TraceContext) Reset() { + *x = TraceContext{} + mi := &file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TraceContext) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TraceContext) ProtoMessage() {} + +func (x *TraceContext) ProtoReflect() protoreflect.Message { + mi := &file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TraceContext.ProtoReflect.Descriptor instead. +func (*TraceContext) Descriptor() ([]byte, []int) { + return file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_rawDescGZIP(), []int{0} +} + +var File_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto protoreflect.FileDescriptor + +var file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_rawDesc = []byte{ + 0x0a, 0x4e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x6f, 0x6d, + 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2f, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2f, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x2f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x2f, 0x74, 0x72, 0x61, + 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x13, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x6e, + 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x0e, 0x0a, 0x0c, 0x54, 0x72, 0x61, 0x63, 0x65, 0x43, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x42, 0x3c, 0x5a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2f, 0x70, 0x6f, 0x6d, + 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x65, 0x6e, 0x76, + 0x6f, 0x79, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, + 0x6f, 0x6e, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_rawDescOnce sync.Once + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_rawDescData = file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_rawDesc +) + +func file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_rawDescGZIP() []byte { + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_rawDescOnce.Do(func() { + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_rawDescData = protoimpl.X.CompressGZIP(file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_rawDescData) + }) + return file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_rawDescData +} + +var file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_goTypes = []any{ + (*TraceContext)(nil), // 0: pomerium.extensions.TraceContext +} +var file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_init() +} +func file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_init() { + if File_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_goTypes, + DependencyIndexes: file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_depIdxs, + MessageInfos: file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_msgTypes, + }.Build() + File_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto = out.File + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_rawDesc = nil + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_goTypes = nil + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_trace_context_proto_depIdxs = nil +} diff --git a/config/envoyconfig/extensions/trace_context.proto b/config/envoyconfig/extensions/trace_context.proto new file mode 100644 index 000000000..c336342ef --- /dev/null +++ b/config/envoyconfig/extensions/trace_context.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +package pomerium.extensions; + +option go_package = "github.com/pomerium/pomerium/config/envoyconfig/extensions"; + +message TraceContext {} diff --git a/config/envoyconfig/extensions/uuidx.pb.go b/config/envoyconfig/extensions/uuidx.pb.go new file mode 100644 index 000000000..eba0c29b1 --- /dev/null +++ b/config/envoyconfig/extensions/uuidx.pb.go @@ -0,0 +1,157 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.35.2 +// protoc (unknown) +// source: github.com/pomerium/pomerium/config/envoyconfig/extensions/uuidx.proto + +package extensions + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + wrapperspb "google.golang.org/protobuf/types/known/wrapperspb" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type UuidxRequestIdConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + PackTraceReason *wrapperspb.BoolValue `protobuf:"bytes,1,opt,name=pack_trace_reason,json=packTraceReason,proto3" json:"pack_trace_reason,omitempty"` + UseRequestIdForTraceSampling *wrapperspb.BoolValue `protobuf:"bytes,2,opt,name=use_request_id_for_trace_sampling,json=useRequestIdForTraceSampling,proto3" json:"use_request_id_for_trace_sampling,omitempty"` +} + +func (x *UuidxRequestIdConfig) Reset() { + *x = UuidxRequestIdConfig{} + mi := &file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UuidxRequestIdConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UuidxRequestIdConfig) ProtoMessage() {} + +func (x *UuidxRequestIdConfig) ProtoReflect() protoreflect.Message { + mi := &file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UuidxRequestIdConfig.ProtoReflect.Descriptor instead. +func (*UuidxRequestIdConfig) Descriptor() ([]byte, []int) { + return file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_rawDescGZIP(), []int{0} +} + +func (x *UuidxRequestIdConfig) GetPackTraceReason() *wrapperspb.BoolValue { + if x != nil { + return x.PackTraceReason + } + return nil +} + +func (x *UuidxRequestIdConfig) GetUseRequestIdForTraceSampling() *wrapperspb.BoolValue { + if x != nil { + return x.UseRequestIdForTraceSampling + } + return nil +} + +var File_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto protoreflect.FileDescriptor + +var file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_rawDesc = []byte{ + 0x0a, 0x46, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x6f, 0x6d, + 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2f, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2f, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x2f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x2f, 0x75, 0x75, 0x69, + 0x64, 0x78, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x13, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, + 0x75, 0x6d, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x1e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x77, + 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xc3, 0x01, + 0x0a, 0x14, 0x55, 0x75, 0x69, 0x64, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x46, 0x0a, 0x11, 0x70, 0x61, 0x63, 0x6b, 0x5f, 0x74, + 0x72, 0x61, 0x63, 0x65, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0f, 0x70, + 0x61, 0x63, 0x6b, 0x54, 0x72, 0x61, 0x63, 0x65, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x63, + 0x0a, 0x21, 0x75, 0x73, 0x65, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, + 0x5f, 0x66, 0x6f, 0x72, 0x5f, 0x74, 0x72, 0x61, 0x63, 0x65, 0x5f, 0x73, 0x61, 0x6d, 0x70, 0x6c, + 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x1c, 0x75, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x49, 0x64, 0x46, 0x6f, 0x72, 0x54, 0x72, 0x61, 0x63, 0x65, 0x53, 0x61, 0x6d, 0x70, 0x6c, + 0x69, 0x6e, 0x67, 0x42, 0x3c, 0x5a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2f, 0x70, 0x6f, 0x6d, 0x65, 0x72, + 0x69, 0x75, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x65, 0x6e, 0x76, 0x6f, 0x79, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, + 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_rawDescOnce sync.Once + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_rawDescData = file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_rawDesc +) + +func file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_rawDescGZIP() []byte { + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_rawDescOnce.Do(func() { + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_rawDescData = protoimpl.X.CompressGZIP(file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_rawDescData) + }) + return file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_rawDescData +} + +var file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_goTypes = []any{ + (*UuidxRequestIdConfig)(nil), // 0: pomerium.extensions.UuidxRequestIdConfig + (*wrapperspb.BoolValue)(nil), // 1: google.protobuf.BoolValue +} +var file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_depIdxs = []int32{ + 1, // 0: pomerium.extensions.UuidxRequestIdConfig.pack_trace_reason:type_name -> google.protobuf.BoolValue + 1, // 1: pomerium.extensions.UuidxRequestIdConfig.use_request_id_for_trace_sampling:type_name -> google.protobuf.BoolValue + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_init() } +func file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_init() { + if File_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_goTypes, + DependencyIndexes: file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_depIdxs, + MessageInfos: file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_msgTypes, + }.Build() + File_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto = out.File + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_rawDesc = nil + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_goTypes = nil + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_uuidx_proto_depIdxs = nil +} diff --git a/config/envoyconfig/extensions/uuidx.proto b/config/envoyconfig/extensions/uuidx.proto new file mode 100644 index 000000000..c0d413964 --- /dev/null +++ b/config/envoyconfig/extensions/uuidx.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package pomerium.extensions; + +import "google/protobuf/wrappers.proto"; + +option go_package = "github.com/pomerium/pomerium/config/envoyconfig/extensions"; + +message UuidxRequestIdConfig { + google.protobuf.BoolValue pack_trace_reason = 1; + google.protobuf.BoolValue use_request_id_for_trace_sampling = 2; +} diff --git a/config/envoyconfig/filters.go b/config/envoyconfig/filters.go index 2292f73e4..fda1984cb 100644 --- a/config/envoyconfig/filters.go +++ b/config/envoyconfig/filters.go @@ -34,6 +34,16 @@ func ExtAuthzFilter(grpcClientTimeout *durationpb.Duration) *envoy_extensions_fi ClusterName: "pomerium-authorize", }, }, + InitialMetadata: []*envoy_config_core_v3.HeaderValue{ + { + Key: "x-pomerium-traceparent", + Value: `%DYNAMIC_METADATA(pomerium.internal:traceparent)%`, + }, + { + Key: "x-pomerium-tracestate", + Value: `%DYNAMIC_METADATA(pomerium.internal:tracestate)%`, + }, + }, }, }, MetadataContextNamespaces: []string{"com.pomerium.client-certificate-info"}, diff --git a/config/envoyconfig/http_connection_manager.go b/config/envoyconfig/http_connection_manager.go index 5d956b373..2b7791a9b 100644 --- a/config/envoyconfig/http_connection_manager.go +++ b/config/envoyconfig/http_connection_manager.go @@ -39,6 +39,22 @@ func (b *Builder) buildVirtualHost( return nil, err } vh.Routes = append(vh.Routes, rs...) + vh.RequestHeadersToAdd = []*envoy_config_core_v3.HeaderValueOption{ + { + Header: &envoy_config_core_v3.HeaderValue{ + Key: "x-pomerium-traceparent", + Value: `%DYNAMIC_METADATA(pomerium.internal:traceparent)%`, + }, + AppendAction: envoy_config_core_v3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, + }, + { + Header: &envoy_config_core_v3.HeaderValue{ + Key: "x-pomerium-tracestate", + Value: `%DYNAMIC_METADATA(pomerium.internal:tracestate)%`, + }, + AppendAction: envoy_config_core_v3.HeaderValueOption_APPEND_IF_EXISTS_OR_ADD, + }, + } return vh, nil } diff --git a/config/envoyconfig/listeners.go b/config/envoyconfig/listeners.go index f017c0834..e02b39e3e 100644 --- a/config/envoyconfig/listeners.go +++ b/config/envoyconfig/listeners.go @@ -20,7 +20,7 @@ func (b *Builder) BuildListeners( cfg *config.Config, fullyStatic bool, ) ([]*envoy_config_listener_v3.Listener, error) { - ctx, span := trace.StartSpan(ctx, "envoyconfig.Builder.BuildListeners") + ctx, span := trace.Continue(ctx, "envoyconfig.Builder.BuildListeners") defer span.End() var listeners []*envoy_config_listener_v3.Listener diff --git a/config/envoyconfig/listeners_grpc.go b/config/envoyconfig/listeners_grpc.go index 802947cf1..2f92b3de1 100644 --- a/config/envoyconfig/listeners_grpc.go +++ b/config/envoyconfig/listeners_grpc.go @@ -73,6 +73,9 @@ func (b *Builder) buildGRPCHTTPConnectionManagerFilter() *envoy_config_listener_ PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: fmt.Sprintf("/%s/", svc)}, Grpc: &envoy_config_route_v3.RouteMatch_GrpcRouteMatchOptions{}, }, + Decorator: &envoy_config_route_v3.Decorator{ + Operation: fmt.Sprintf("pomerium-control-plane-grpc %s", svc), + }, Action: &envoy_config_route_v3.Route_Route{ Route: &envoy_config_route_v3.RouteAction{ ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{ diff --git a/config/envoyconfig/listeners_main.go b/config/envoyconfig/listeners_main.go index b72f760f3..fd29dd455 100644 --- a/config/envoyconfig/listeners_main.go +++ b/config/envoyconfig/listeners_main.go @@ -10,7 +10,6 @@ import ( envoy_config_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" envoy_extensions_access_loggers_grpc_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/grpc/v3" envoy_extensions_filters_network_http_connection_manager "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" - envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/wrapperspb" @@ -177,11 +176,6 @@ func (b *Builder) buildMainHTTPConnectionManagerFilter( maxStreamDuration = durationpb.New(cfg.Options.WriteTimeout) } - tracingProvider, err := buildTracingHTTP(cfg.Options) - if err != nil { - return nil, err - } - localReply, err := b.buildLocalReplyConfig(cfg.Options) if err != nil { return nil, err @@ -198,10 +192,6 @@ func (b *Builder) buildMainHTTPConnectionManagerFilter( }, HttpProtocolOptions: http1ProtocolOptions, RequestTimeout: durationpb.New(cfg.Options.ReadTimeout), - Tracing: &envoy_extensions_filters_network_http_connection_manager.HttpConnectionManager_Tracing{ - RandomSampling: &envoy_type_v3.Percent{Value: cfg.Options.TracingSampleRate * 100}, - Provider: tracingProvider, - }, // See https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for UseRemoteAddress: &wrapperspb.BoolValue{Value: true}, SkipXffAppend: cfg.Options.SkipXffAppend, @@ -224,6 +214,8 @@ func (b *Builder) buildMainHTTPConnectionManagerFilter( mgr.CodecType = cfg.Options.GetCodecType().ToEnvoy() } + applyTracingConfig(mgr, cfg.Options) + if fullyStatic { routeConfiguration, err := b.buildMainRouteConfiguration(ctx, cfg) if err != nil { diff --git a/config/envoyconfig/listeners_test.go b/config/envoyconfig/listeners_test.go index bd7d9baa9..95a1eba54 100644 --- a/config/envoyconfig/listeners_test.go +++ b/config/envoyconfig/listeners_test.go @@ -103,7 +103,7 @@ func TestBuildListeners(t *testing.T) { }] } } - }`, httpConfig.Get("httpFilters.6").String(), + }`, httpConfig.Get("httpFilters.7").String(), "should add alt-svc header") case "quic-ingress": hasQUIC = true @@ -151,5 +151,6 @@ func Test_buildMainHTTPConnectionManagerFilter(t *testing.T) { options.AuthenticateURLString = "https://authenticate.example.com" filter, err := b.buildMainHTTPConnectionManagerFilter(context.Background(), &config.Config{Options: options}, false, false) require.NoError(t, err) + testutil.AssertProtoJSONEqual(t, testData(t, "main_http_connection_manager_filter.json", nil), filter) } diff --git a/config/envoyconfig/outbound.go b/config/envoyconfig/outbound.go index bcf0b93f7..bcb518684 100644 --- a/config/envoyconfig/outbound.go +++ b/config/envoyconfig/outbound.go @@ -108,6 +108,9 @@ func (b *Builder) buildOutboundRoutes() []*envoy_config_route_v3.Route { PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: prefix}, Grpc: &envoy_config_route_v3.RouteMatch_GrpcRouteMatchOptions{}, }, + Decorator: &envoy_config_route_v3.Decorator{ + Operation: fmt.Sprintf("Outbound (grpc): %s %s", def.Cluster, prefix), + }, Action: &envoy_config_route_v3.Route_Route{ Route: &envoy_config_route_v3.RouteAction{ ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{ @@ -130,6 +133,9 @@ func (b *Builder) buildOutboundRoutes() []*envoy_config_route_v3.Route { Match: &envoy_config_route_v3.RouteMatch{ PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: "/envoy/stats/prometheus"}, }, + Decorator: &envoy_config_route_v3.Decorator{ + Operation: "Outbound: envoy-metrics /envoy/stats/prometheus/*", + }, Action: &envoy_config_route_v3.Route_Route{ Route: &envoy_config_route_v3.RouteAction{ ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{ diff --git a/config/envoyconfig/outbound_test.go b/config/envoyconfig/outbound_test.go index 2a12488c2..3c7233f7a 100644 --- a/config/envoyconfig/outbound_test.go +++ b/config/envoyconfig/outbound_test.go @@ -15,6 +15,9 @@ func Test_buildOutboundRoutes(t *testing.T) { "grpc": {}, "prefix": "/envoy.service.auth.v3.Authorization/" }, + "decorator": { + "operation": "Outbound (grpc): pomerium-authorize /envoy.service.auth.v3.Authorization/" + }, "name": "pomerium-authorize", "route": { "autoHostRewrite": true, @@ -28,6 +31,9 @@ func Test_buildOutboundRoutes(t *testing.T) { "grpc": {}, "prefix": "/databroker.DataBrokerService/" }, + "decorator": { + "operation": "Outbound (grpc): pomerium-databroker /databroker.DataBrokerService/" + }, "name": "pomerium-databroker", "route": { "autoHostRewrite": true, @@ -41,6 +47,9 @@ func Test_buildOutboundRoutes(t *testing.T) { "grpc": {}, "prefix": "/registry.Registry/" }, + "decorator": { + "operation": "Outbound (grpc): pomerium-databroker /registry.Registry/" + }, "name": "pomerium-databroker", "route": { "autoHostRewrite": true, @@ -54,6 +63,9 @@ func Test_buildOutboundRoutes(t *testing.T) { "grpc": {}, "prefix": "/" }, + "decorator": { + "operation": "Outbound (grpc): pomerium-control-plane-grpc /" + }, "name": "pomerium-control-plane-grpc", "route": { "autoHostRewrite": true, @@ -66,6 +78,9 @@ func Test_buildOutboundRoutes(t *testing.T) { "match": { "prefix": "/envoy/stats/prometheus" }, + "decorator": { + "operation": "Outbound: envoy-metrics /envoy/stats/prometheus/*" + }, "name": "envoy-metrics", "route": { "cluster": "pomerium-envoy-admin", diff --git a/config/envoyconfig/route_configurations.go b/config/envoyconfig/route_configurations.go index 4757b1a74..9881e7322 100644 --- a/config/envoyconfig/route_configurations.go +++ b/config/envoyconfig/route_configurations.go @@ -20,7 +20,7 @@ func (b *Builder) BuildRouteConfigurations( ctx context.Context, cfg *config.Config, ) ([]*envoy_config_route_v3.RouteConfiguration, error) { - ctx, span := trace.StartSpan(ctx, "envoyconfig.Builder.BuildRouteConfigurations") + ctx, span := trace.Continue(ctx, "envoyconfig.Builder.BuildRouteConfigurations") defer span.End() var routeConfigurations []*envoy_config_route_v3.RouteConfiguration diff --git a/config/envoyconfig/route_configurations_test.go b/config/envoyconfig/route_configurations_test.go index d7e97eb6a..4cc103068 100644 --- a/config/envoyconfig/route_configurations_test.go +++ b/config/envoyconfig/route_configurations_test.go @@ -42,6 +42,21 @@ func TestBuilder_buildMainRouteConfiguration(t *testing.T) { { "name": "catch-all", "domains": ["*"], + "requestHeadersToAdd": [ + { + "appendAction": "OVERWRITE_IF_EXISTS_OR_ADD", + "header": { + "key": "x-pomerium-traceparent", + "value": "%DYNAMIC_METADATA(pomerium.internal:traceparent)%" + } + }, + { + "header": { + "key": "x-pomerium-tracestate", + "value": "%DYNAMIC_METADATA(pomerium.internal:tracestate)%" + } + } + ], "routes": [ `+protojson.Format(b.buildControlPlanePathRoute(cfg.Options, "/ping"))+`, `+protojson.Format(b.buildControlPlanePathRoute(cfg.Options, "/healthz"))+`, @@ -51,6 +66,10 @@ func TestBuilder_buildMainRouteConfiguration(t *testing.T) { `+protojson.Format(b.buildControlPlanePrefixRoute(cfg.Options, "/.well-known/pomerium/"))+`, { "name": "policy-0", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "headers": [ { "name": ":authority", "stringMatch": { "safeRegex": { "regex": "^(.*)\\.example\\.com$" } }} @@ -104,6 +123,10 @@ func TestBuilder_buildMainRouteConfiguration(t *testing.T) { }, { "name": "policy-0", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "headers": [ { "name": ":authority", "stringMatch": { "safeRegex": { "regex": "^(.*)\\.example\\.com:443$" } }} diff --git a/config/envoyconfig/routes.go b/config/envoyconfig/routes.go index 850976c1e..4bbb10263 100644 --- a/config/envoyconfig/routes.go +++ b/config/envoyconfig/routes.go @@ -114,6 +114,9 @@ func (b *Builder) buildControlPlanePathRoute( Match: &envoy_config_route_v3.RouteMatch{ PathSpecifier: &envoy_config_route_v3.RouteMatch_Path{Path: path}, }, + Decorator: &envoy_config_route_v3.Decorator{ + Operation: "internal: ${method} ${host}${path}", + }, Action: &envoy_config_route_v3.Route_Route{ Route: &envoy_config_route_v3.RouteAction{ ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{ @@ -138,6 +141,9 @@ func (b *Builder) buildControlPlanePrefixRoute( Match: &envoy_config_route_v3.RouteMatch{ PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: prefix}, }, + Decorator: &envoy_config_route_v3.Decorator{ + Operation: "internal: ${method} ${host}${path}", + }, Action: &envoy_config_route_v3.Route_Route{ Route: &envoy_config_route_v3.RouteAction{ ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{ @@ -269,8 +275,12 @@ func (b *Builder) buildRouteForPolicyAndMatch( } route := &envoy_config_route_v3.Route{ - Name: name, - Match: match, + Name: name, + Match: match, + Decorator: &envoy_config_route_v3.Decorator{ + Operation: "ingress: ${method} ${host}${path}", + Propagate: wrapperspb.Bool(false), + }, Metadata: &envoy_config_core_v3.Metadata{}, RequestHeadersToRemove: getRequestHeadersToRemove(cfg.Options, policy), ResponseHeadersToAdd: toEnvoyHeaders(cfg.Options.GetSetResponseHeadersForPolicy(policy)), diff --git a/config/envoyconfig/routes_test.go b/config/envoyconfig/routes_test.go index 86825413a..3c9ea98ce 100644 --- a/config/envoyconfig/routes_test.go +++ b/config/envoyconfig/routes_test.go @@ -59,6 +59,9 @@ func Test_buildPomeriumHTTPRoutes(t *testing.T) { routeString := func(typ, name string) string { str := `{ "name": "pomerium-` + typ + `-` + name + `", + "decorator": { + "operation": "internal: ${method} ${host}${path}" + }, "match": { "` + typ + `": "` + name + `" }, @@ -135,6 +138,9 @@ func Test_buildControlPlanePathRoute(t *testing.T) { testutil.AssertProtoJSONEqual(t, ` { "name": "pomerium-path-/hello/world", + "decorator": { + "operation": "internal: ${method} ${host}${path}" + }, "match": { "path": "/hello/world" }, @@ -179,6 +185,9 @@ func Test_buildControlPlanePrefixRoute(t *testing.T) { testutil.AssertProtoJSONEqual(t, ` { "name": "pomerium-prefix-/hello/world/", + "decorator": { + "operation": "internal: ${method} ${host}${path}" + }, "match": { "prefix": "/hello/world/" }, @@ -406,6 +415,10 @@ func Test_buildPolicyRoutes(t *testing.T) { [ { "name": "policy-1", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/" }, @@ -476,6 +489,10 @@ func Test_buildPolicyRoutes(t *testing.T) { }, { "name": "policy-2", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "path": "/some/path" }, @@ -547,6 +564,10 @@ func Test_buildPolicyRoutes(t *testing.T) { }, { "name": "policy-3", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/some/prefix/" }, @@ -617,6 +638,10 @@ func Test_buildPolicyRoutes(t *testing.T) { }, { "name": "policy-4", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "safeRegex": { "regex": "^/[a]+$" @@ -689,6 +714,10 @@ func Test_buildPolicyRoutes(t *testing.T) { }, { "name": "policy-5", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/some/prefix/" }, @@ -760,6 +789,10 @@ func Test_buildPolicyRoutes(t *testing.T) { }, { "name": "policy-6", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "path": "/some/path" }, @@ -830,6 +863,10 @@ func Test_buildPolicyRoutes(t *testing.T) { }, { "name": "policy-7", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "path": "/some/path" }, @@ -901,6 +938,10 @@ func Test_buildPolicyRoutes(t *testing.T) { }, { "name": "policy-8", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "path": "/websocket-timeout" }, @@ -994,6 +1035,10 @@ func Test_buildPolicyRoutes(t *testing.T) { [ { "name": "policy-0", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/" }, @@ -1083,6 +1128,10 @@ func Test_buildPolicyRoutes(t *testing.T) { [ { "name": "policy-0", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "connectMatcher": {} }, @@ -1155,6 +1204,10 @@ func Test_buildPolicyRoutes(t *testing.T) { }, { "name": "policy-1", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "connectMatcher": {} }, @@ -1248,6 +1301,10 @@ func Test_buildPolicyRoutes(t *testing.T) { [ { "name": "policy-0", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "connectMatcher": {} }, @@ -1345,6 +1402,10 @@ func Test_buildPolicyRoutes(t *testing.T) { [ { "name": "policy-0", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/" }, @@ -1440,6 +1501,10 @@ func Test_buildPolicyRoutes(t *testing.T) { [ { "name": "policy-0", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/" }, @@ -1588,6 +1653,10 @@ func Test_buildPolicyRoutesRewrite(t *testing.T) { [ { "name": "policy-0", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/" }, @@ -1659,6 +1728,10 @@ func Test_buildPolicyRoutesRewrite(t *testing.T) { }, { "name": "policy-1", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/" }, @@ -1730,6 +1803,10 @@ func Test_buildPolicyRoutesRewrite(t *testing.T) { }, { "name": "policy-2", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/" }, @@ -1806,6 +1883,10 @@ func Test_buildPolicyRoutesRewrite(t *testing.T) { }, { "name": "policy-3", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/" }, @@ -1877,6 +1958,10 @@ func Test_buildPolicyRoutesRewrite(t *testing.T) { }, { "name": "policy-4", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/" }, @@ -1948,6 +2033,10 @@ func Test_buildPolicyRoutesRewrite(t *testing.T) { }, { "name": "policy-5", + "decorator": { + "operation": "ingress: ${method} ${host}${path}", + "propagate": false + }, "match": { "prefix": "/" }, diff --git a/config/envoyconfig/testdata/main_http_connection_manager_filter.json b/config/envoyconfig/testdata/main_http_connection_manager_filter.json index b67fa0d06..f286d777a 100644 --- a/config/envoyconfig/testdata/main_http_connection_manager_filter.json +++ b/config/envoyconfig/testdata/main_http_connection_manager_filter.json @@ -23,7 +23,69 @@ "commonHttpProtocolOptions": { "idleTimeout": "300s" }, + "earlyHeaderMutationExtensions": [ + { + "name": "envoy.http.early_header_mutation.trace_context", + "typedConfig": { + "@type": "type.googleapis.com/pomerium.extensions.TraceContext" + } + } + ], "httpFilters": [ + { + "name": "envoy.filters.http.header_to_metadata", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.header_to_metadata.v3.Config", + "requestRules": [ + { + "header": "x-pomerium-traceparent", + "onHeaderPresent": { + "metadataNamespace": "pomerium.internal", + "key": "traceparent" + } + }, + { + "header": "x-pomerium-tracestate", + "onHeaderPresent": { + "metadataNamespace": "pomerium.internal", + "key": "tracestate" + } + }, + { + "header": "x-pomerium-external-parent-span", + "onHeaderPresent": { + "key": "external-parent-span", + "metadataNamespace": "pomerium.internal" + }, + "remove": true + }, + { + "header": "x-pomerium-sampling-decision", + "onHeaderPresent": { + "metadataNamespace": "pomerium.internal", + "key": "sampling-decision" + }, + "remove": true + } + ], + "responseRules": [ + { + "header": "x-pomerium-traceparent", + "onHeaderPresent": { + "metadataNamespace": "pomerium.internal", + "key": "traceparent" + } + }, + { + "header": "x-pomerium-tracestate", + "onHeaderPresent": { + "metadataNamespace": "pomerium.internal", + "key": "tracestate" + } + } + ] + } + }, { "name": "envoy.filters.http.lua", "typedConfig": { @@ -50,13 +112,25 @@ "envoyGrpc": { "clusterName": "pomerium-authorize" }, - "timeout": "10s" + "timeout": "10s", + "initialMetadata": [ + { + "key": "x-pomerium-traceparent", + "value": "%DYNAMIC_METADATA(pomerium.internal:traceparent)%" + }, + { + "key": "x-pomerium-tracestate", + "value": "%DYNAMIC_METADATA(pomerium.internal:tracestate)%" + } + ] }, - "metadataContextNamespaces": ["com.pomerium.client-certificate-info"], + "transportApiVersion": "V3", "statusOnError": { "code": "InternalServerError" }, - "transportApiVersion": "V3" + "metadataContextNamespaces": [ + "com.pomerium.client-certificate-info" + ] } }, { @@ -73,7 +147,7 @@ "typedConfig": { "@type": "type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua", "defaultSourceCode": { - "inlineString": "function has_prefix(str, prefix)\n return str ~= nil and str:sub(1, #prefix) == prefix\nend\n\nfunction remove_pomerium_cookie(cookie_name, cookie)\n local result = \"\"\n for c in cookie:gmatch(\"([^;]+)\") do\n c = c:gsub(\"^ +\",\"\")\n local name = c:match(\"^([^=]+)\")\n if name ~= cookie_name then\n if string.len(result) \u003e 0 then\n result = result .. \"; \" .. c\n else\n result = result .. c\n end\n end\n end\n return result\nend\n\nfunction envoy_on_request(request_handle)\n local headers = request_handle:headers()\n local metadata = request_handle:metadata()\n\n local remove_cookie_name = metadata:get(\"remove_pomerium_cookie\")\n if remove_cookie_name then\n local cookie = headers:get(\"cookie\")\n if cookie ~= nil then\n local newcookie = remove_pomerium_cookie(remove_cookie_name, cookie)\n headers:replace(\"cookie\", newcookie)\n end\n end\n\n local remove_authorization = metadata:get(\"remove_pomerium_authorization\")\n if remove_authorization then\n local authorization = headers:get(\"authorization\")\n local authorization_prefix = \"Pomerium \"\n if has_prefix(authorization, authorization_prefix) then\n headers:remove(\"authorization\")\n end\n\n headers:remove('x-pomerium-authorization')\n end\nend\n\nfunction envoy_on_response(response_handle) end\n" + "inlineString": "function has_prefix(str, prefix)\n return str ~= nil and str:sub(1, #prefix) == prefix\nend\n\nfunction remove_pomerium_cookie(cookie_name, cookie)\n local result = \"\"\n for c in cookie:gmatch(\"([^;]+)\") do\n c = c:gsub(\"^ +\",\"\")\n local name = c:match(\"^([^=]+)\")\n if name ~= cookie_name then\n if string.len(result) > 0 then\n result = result .. \"; \" .. c\n else\n result = result .. c\n end\n end\n end\n return result\nend\n\nfunction envoy_on_request(request_handle)\n local headers = request_handle:headers()\n local metadata = request_handle:metadata()\n\n local remove_cookie_name = metadata:get(\"remove_pomerium_cookie\")\n if remove_cookie_name then\n local cookie = headers:get(\"cookie\")\n if cookie ~= nil then\n local newcookie = remove_pomerium_cookie(remove_cookie_name, cookie)\n headers:replace(\"cookie\", newcookie)\n end\n end\n\n local remove_authorization = metadata:get(\"remove_pomerium_authorization\")\n if remove_authorization then\n local authorization = headers:get(\"authorization\")\n local authorization_prefix = \"Pomerium \"\n if has_prefix(authorization, authorization_prefix) then\n headers:remove(\"authorization\")\n end\n\n headers:remove('x-pomerium-authorization')\n end\nend\n\nfunction envoy_on_response(response_handle) end\n" } } }, @@ -167,6 +241,13 @@ } ] }, + "requestIdExtension": { + "typedConfig": { + "@type": "type.googleapis.com/pomerium.extensions.UuidxRequestIdConfig", + "packTraceReason": true, + "useRequestIdForTraceSampling": true + } + }, "requestTimeout": "30s", "normalizePath": true, "rds": { @@ -181,7 +262,83 @@ "tracing": { "randomSampling": { "value": 0.01 - } + }, + "verbose": true, + "maxPathTagLength": 1024, + "customTags": [ + { + "tag": "pomerium.traceparent", + "metadata": { + "kind": { + "request": {} + }, + "metadataKey": { + "key": "pomerium.internal", + "path": [ + { + "key": "traceparent" + } + ] + } + } + }, + { + "tag": "pomerium.tracestate", + "metadata": { + "kind": { + "request": {} + }, + "metadataKey": { + "key": "pomerium.internal", + "path": [ + { + "key": "tracestate" + } + ] + } + } + }, + { + "metadata": { + "kind": { + "request": {} + }, + "metadataKey": { + "key": "pomerium.internal", + "path": [ + { + "key": "external-parent-span" + } + ] + } + }, + "tag": "pomerium.external-parent-span" + } + ], + "provider": { + "name": "envoy.tracers.opentelemetry", + "typedConfig": { + "@type": "type.googleapis.com/envoy.config.trace.v3.OpenTelemetryConfig", + "grpcService": { + "envoyGrpc": { + "clusterName": "pomerium-control-plane-grpc" + } + }, + "serviceName": "Envoy", + "resourceDetectors": [ + { + "name": "envoy.tracers.opentelemetry.resource_detectors.static_config", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.tracers.opentelemetry.resource_detectors.v3.StaticConfigResourceDetectorConfig", + "attributes": { + "pomerium.envoy": "true" + } + } + } + ] + } + }, + "spawnUpstreamSpan": true }, "useRemoteAddress": true, "xffNumTrustedHops": 1 diff --git a/config/envoyconfig/tracing.go b/config/envoyconfig/tracing.go index 60eb69f9d..a47ea356a 100644 --- a/config/envoyconfig/tracing.go +++ b/config/envoyconfig/tracing.go @@ -2,133 +2,160 @@ package envoyconfig import ( "fmt" - "net" - - envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" - envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" - envoy_config_trace_v3 "github.com/envoyproxy/go-control-plane/envoy/config/trace/v3" - "google.golang.org/protobuf/types/known/durationpb" + "os" + "strconv" + envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + tracev3 "github.com/envoyproxy/go-control-plane/envoy/config/trace/v3" + envoy_extensions_filters_http_header_to_metadata "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/header_to_metadata/v3" + envoy_extensions_filters_network_http_connection_manager "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + envoy_extensions_tracers_otel "github.com/envoyproxy/go-control-plane/envoy/extensions/tracers/opentelemetry/resource_detectors/v3" + metadatav3 "github.com/envoyproxy/go-control-plane/envoy/type/metadata/v3" + envoy_tracing_v3 "github.com/envoyproxy/go-control-plane/envoy/type/tracing/v3" + envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3" "github.com/pomerium/pomerium/config" - "github.com/pomerium/pomerium/internal/telemetry/trace" - "github.com/pomerium/pomerium/pkg/protoutil" + "github.com/pomerium/pomerium/config/envoyconfig/extensions" + "google.golang.org/protobuf/types/known/wrapperspb" ) -func buildTracingCluster(options *config.Options) (*envoy_config_cluster_v3.Cluster, error) { - tracingOptions, err := config.NewTracingOptions(options) - if err != nil { - return nil, fmt.Errorf("envoyconfig: invalid tracing config: %w", err) +func applyTracingConfig( + mgr *envoy_extensions_filters_network_http_connection_manager.HttpConnectionManager, + opts *config.Options, +) { + mgr.HttpFilters = append([]*envoy_extensions_filters_network_http_connection_manager.HttpFilter{ + tracingMetadataFilter(), + }, mgr.HttpFilters...) + + mgr.EarlyHeaderMutationExtensions = []*envoy_config_core_v3.TypedExtensionConfig{ + { + Name: "envoy.http.early_header_mutation.trace_context", + TypedConfig: marshalAny(&extensions.TraceContext{}), + }, + } + mgr.RequestIdExtension = &envoy_extensions_filters_network_http_connection_manager.RequestIDExtension{ + TypedConfig: marshalAny(&extensions.UuidxRequestIdConfig{ + PackTraceReason: wrapperspb.Bool(true), + UseRequestIdForTraceSampling: wrapperspb.Bool(true), + }), } - switch tracingOptions.Provider { - case trace.DatadogTracingProviderName: - addr, _ := parseAddress("127.0.0.1:8126") - - if options.TracingDatadogAddress != "" { - addr, err = parseAddress(options.TracingDatadogAddress) - if err != nil { - return nil, fmt.Errorf("envoyconfig: invalid tracing datadog address: %w", err) - } + maxPathTagLength := uint32(1024) + if value, ok := os.LookupEnv("OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT"); ok { + if num, err := strconv.ParseUint(value, 10, 32); err == nil { + maxPathTagLength = max(64, uint32(num)) } - - endpoints := []*envoy_config_endpoint_v3.LbEndpoint{{ - HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{ - Endpoint: &envoy_config_endpoint_v3.Endpoint{ - Address: addr, + } + requestTag := func(key string) *envoy_tracing_v3.CustomTag { + return &envoy_tracing_v3.CustomTag{ + Tag: fmt.Sprintf("pomerium.%s", key), + Type: &envoy_tracing_v3.CustomTag_Metadata_{ + Metadata: &envoy_tracing_v3.CustomTag_Metadata{ + Kind: &metadatav3.MetadataKind{ + Kind: &metadatav3.MetadataKind_Request_{ + Request: &metadatav3.MetadataKind_Request{}, + }, + }, + MetadataKey: &metadatav3.MetadataKey{ + Key: "pomerium.internal", + Path: []*metadatav3.MetadataKey_PathSegment{ + { + Segment: &metadatav3.MetadataKey_PathSegment_Key{ + Key: key, + }, + }, + }, + }, }, }, - }} - - return &envoy_config_cluster_v3.Cluster{ - Name: "datadog-apm", - ConnectTimeout: &durationpb.Duration{ - Seconds: 5, - }, - ClusterDiscoveryType: getClusterDiscoveryType(endpoints), - LbPolicy: envoy_config_cluster_v3.Cluster_ROUND_ROBIN, - LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{ - ClusterName: "datadog-apm", - Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{{ - LbEndpoints: endpoints, - }}, - }, - }, nil - case trace.ZipkinTracingProviderName: - host := tracingOptions.ZipkinEndpoint.Host - if _, port, _ := net.SplitHostPort(host); port == "" { - if tracingOptions.ZipkinEndpoint.Scheme == "https" { - host = net.JoinHostPort(host, "443") - } else { - host = net.JoinHostPort(host, "80") - } } - - addr, err := parseAddress(host) - if err != nil { - return nil, fmt.Errorf("envoyconfig: invalid tracing zipkin address: %w", err) - } - - endpoints := []*envoy_config_endpoint_v3.LbEndpoint{{ - HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{ - Endpoint: &envoy_config_endpoint_v3.Endpoint{ - Address: addr, - }, + } + mgr.Tracing = &envoy_extensions_filters_network_http_connection_manager.HttpConnectionManager_Tracing{ + RandomSampling: &envoy_type_v3.Percent{Value: opts.TracingSampleRate * 100}, + Verbose: true, + SpawnUpstreamSpan: wrapperspb.Bool(true), + Provider: &tracev3.Tracing_Http{ + Name: "envoy.tracers.opentelemetry", + ConfigType: &tracev3.Tracing_Http_TypedConfig{ + TypedConfig: marshalAny(&tracev3.OpenTelemetryConfig{ + GrpcService: &envoy_config_core_v3.GrpcService{ + TargetSpecifier: &envoy_config_core_v3.GrpcService_EnvoyGrpc_{ + EnvoyGrpc: &envoy_config_core_v3.GrpcService_EnvoyGrpc{ + ClusterName: "pomerium-control-plane-grpc", + }, + }, + }, + ServiceName: "Envoy", + ResourceDetectors: []*envoy_config_core_v3.TypedExtensionConfig{ + { + Name: "envoy.tracers.opentelemetry.resource_detectors.static_config", + TypedConfig: marshalAny(&envoy_extensions_tracers_otel.StaticConfigResourceDetectorConfig{ + Attributes: map[string]string{ + "pomerium.envoy": "true", + }, + }), + }, + }, + }), }, - }} - return &envoy_config_cluster_v3.Cluster{ - Name: "zipkin", - ConnectTimeout: &durationpb.Duration{ - Seconds: 5, - }, - ClusterDiscoveryType: getClusterDiscoveryType(endpoints), - LbPolicy: envoy_config_cluster_v3.Cluster_ROUND_ROBIN, - LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{ - ClusterName: "zipkin", - Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{{ - LbEndpoints: endpoints, - }}, - }, - }, nil - default: - return nil, nil + }, + // this allows full URLs to be displayed in traces, they are otherwise truncated + MaxPathTagLength: wrapperspb.UInt32(maxPathTagLength), + CustomTags: []*envoy_tracing_v3.CustomTag{ + requestTag("traceparent"), + requestTag("tracestate"), + requestTag("external-parent-span"), + }, } } -func buildTracingHTTP(options *config.Options) (*envoy_config_trace_v3.Tracing_Http, error) { - tracingOptions, err := config.NewTracingOptions(options) - if err != nil { - return nil, fmt.Errorf("invalid tracing config: %w", err) +func tracingMetadataFilter() *envoy_extensions_filters_network_http_connection_manager.HttpFilter { + traceparentRule := &envoy_extensions_filters_http_header_to_metadata.Config_Rule{ + Header: "x-pomerium-traceparent", + OnHeaderPresent: &envoy_extensions_filters_http_header_to_metadata.Config_KeyValuePair{ + MetadataNamespace: "pomerium.internal", + Key: "traceparent", + }, + Remove: false, } - - switch tracingOptions.Provider { - case trace.DatadogTracingProviderName: - tracingTC := protoutil.NewAny(&envoy_config_trace_v3.DatadogConfig{ - CollectorCluster: "datadog-apm", - ServiceName: tracingOptions.Service, - }) - return &envoy_config_trace_v3.Tracing_Http{ - Name: "envoy.tracers.datadog", - ConfigType: &envoy_config_trace_v3.Tracing_Http_TypedConfig{ - TypedConfig: tracingTC, - }, - }, nil - case trace.ZipkinTracingProviderName: - path := tracingOptions.ZipkinEndpoint.Path - if path == "" { - path = "/" - } - tracingTC := protoutil.NewAny(&envoy_config_trace_v3.ZipkinConfig{ - CollectorCluster: "zipkin", - CollectorEndpoint: path, - CollectorEndpointVersion: envoy_config_trace_v3.ZipkinConfig_HTTP_JSON, - }) - return &envoy_config_trace_v3.Tracing_Http{ - Name: "envoy.tracers.zipkin", - ConfigType: &envoy_config_trace_v3.Tracing_Http_TypedConfig{ - TypedConfig: tracingTC, - }, - }, nil - default: - return nil, nil + tracestateRule := &envoy_extensions_filters_http_header_to_metadata.Config_Rule{ + Header: "x-pomerium-tracestate", + OnHeaderPresent: &envoy_extensions_filters_http_header_to_metadata.Config_KeyValuePair{ + MetadataNamespace: "pomerium.internal", + Key: "tracestate", + }, + Remove: false, + } + externalParentSpanRule := &envoy_extensions_filters_http_header_to_metadata.Config_Rule{ + Header: "x-pomerium-external-parent-span", + OnHeaderPresent: &envoy_extensions_filters_http_header_to_metadata.Config_KeyValuePair{ + MetadataNamespace: "pomerium.internal", + Key: "external-parent-span", + }, + Remove: true, + } + samplingDecisionRule := &envoy_extensions_filters_http_header_to_metadata.Config_Rule{ + Header: "x-pomerium-sampling-decision", + OnHeaderPresent: &envoy_extensions_filters_http_header_to_metadata.Config_KeyValuePair{ + MetadataNamespace: "pomerium.internal", + Key: "sampling-decision", + }, + Remove: true, + } + return &envoy_extensions_filters_network_http_connection_manager.HttpFilter{ + Name: "envoy.filters.http.header_to_metadata", + ConfigType: &envoy_extensions_filters_network_http_connection_manager.HttpFilter_TypedConfig{ + TypedConfig: marshalAny(&envoy_extensions_filters_http_header_to_metadata.Config{ + RequestRules: []*envoy_extensions_filters_http_header_to_metadata.Config_Rule{ + traceparentRule, + tracestateRule, + externalParentSpanRule, + samplingDecisionRule, + }, + ResponseRules: []*envoy_extensions_filters_http_header_to_metadata.Config_Rule{ + traceparentRule, + tracestateRule, + }, + }), + }, } } diff --git a/config/envoyconfig/tracing_test.go b/config/envoyconfig/tracing_test.go deleted file mode 100644 index d9eadec4a..000000000 --- a/config/envoyconfig/tracing_test.go +++ /dev/null @@ -1,135 +0,0 @@ -package envoyconfig - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/pomerium/pomerium/config" - "github.com/pomerium/pomerium/internal/testutil" -) - -func TestBuildTracingCluster(t *testing.T) { - t.Run("datadog", func(t *testing.T) { - c, err := buildTracingCluster(&config.Options{ - TracingProvider: "datadog", - }) - require.NoError(t, err) - testutil.AssertProtoJSONEqual(t, ` - { - "name": "datadog-apm", - "type": "STATIC", - "connectTimeout": "5s", - "loadAssignment": { - "clusterName": "datadog-apm", - "endpoints": [{ - "lbEndpoints": [{ - "endpoint": { - "address": { - "socketAddress": { - "address": "127.0.0.1", - "portValue": 8126 - } - } - } - }] - }] - } - } - `, c) - - c, err = buildTracingCluster(&config.Options{ - TracingProvider: "datadog", - TracingDatadogAddress: "example.com:8126", - }) - require.NoError(t, err) - testutil.AssertProtoJSONEqual(t, ` - { - "name": "datadog-apm", - "type": "STRICT_DNS", - "connectTimeout": "5s", - "loadAssignment": { - "clusterName": "datadog-apm", - "endpoints": [{ - "lbEndpoints": [{ - "endpoint": { - "address": { - "socketAddress": { - "address": "example.com", - "portValue": 8126 - } - } - } - }] - }] - } - } - `, c) - }) - t.Run("zipkin", func(t *testing.T) { - c, err := buildTracingCluster(&config.Options{ - TracingProvider: "zipkin", - ZipkinEndpoint: "https://example.com/api/v2/spans", - }) - require.NoError(t, err) - testutil.AssertProtoJSONEqual(t, ` - { - "name": "zipkin", - "type": "STRICT_DNS", - "connectTimeout": "5s", - "loadAssignment": { - "clusterName": "zipkin", - "endpoints": [{ - "lbEndpoints": [{ - "endpoint": { - "address": { - "socketAddress": { - "address": "example.com", - "portValue": 443 - } - } - } - }] - }] - } - } - `, c) - }) -} - -func TestBuildTracingHTTP(t *testing.T) { - t.Run("datadog", func(t *testing.T) { - h, err := buildTracingHTTP(&config.Options{ - TracingProvider: "datadog", - }) - require.NoError(t, err) - testutil.AssertProtoJSONEqual(t, ` - { - "name": "envoy.tracers.datadog", - "typedConfig": { - "@type": "type.googleapis.com/envoy.config.trace.v3.DatadogConfig", - "collectorCluster": "datadog-apm", - "serviceName": "pomerium" - } - } - `, h) - }) - t.Run("zipkin", func(t *testing.T) { - h, err := buildTracingHTTP(&config.Options{ - TracingProvider: "zipkin", - ZipkinEndpoint: "https://example.com/api/v2/spans", - }) - require.NoError(t, err) - testutil.AssertProtoJSONEqual(t, ` - { - "name": "envoy.tracers.zipkin", - "typedConfig": { - "@type": "type.googleapis.com/envoy.config.trace.v3.ZipkinConfig", - "collectorCluster": "zipkin", - "collectorEndpoint": "/api/v2/spans", - "collectorEndpointVersion": "HTTP_JSON" - } - } - `, h) - }) -} diff --git a/config/log.go b/config/log.go index 0b86e6ca4..545a5e69a 100644 --- a/config/log.go +++ b/config/log.go @@ -20,11 +20,6 @@ func NewLogManager(ctx context.Context, src Source) *LogManager { return mgr } -// Close closes the log manager. -func (mgr *LogManager) Close() error { - return nil -} - // OnConfigChange is called whenever configuration changes. func (mgr *LogManager) OnConfigChange(_ context.Context, cfg *Config) { if cfg == nil || cfg.Options == nil { diff --git a/config/metrics.go b/config/metrics.go index 695f9598b..39746dfb3 100644 --- a/config/metrics.go +++ b/config/metrics.go @@ -45,11 +45,6 @@ func NewMetricsManager(ctx context.Context, src Source) *MetricsManager { return mgr } -// Close closes any underlying http server. -func (mgr *MetricsManager) Close() error { - return nil -} - // OnConfigChange updates the metrics manager when configuration is changed. func (mgr *MetricsManager) OnConfigChange(ctx context.Context, cfg *Config) { mgr.mu.Lock() diff --git a/config/trace.go b/config/trace.go deleted file mode 100644 index 1c8830494..000000000 --- a/config/trace.go +++ /dev/null @@ -1,125 +0,0 @@ -package config - -import ( - "context" - "fmt" - "reflect" - "sync" - - "github.com/rs/zerolog" - - "github.com/pomerium/pomerium/internal/log" - "github.com/pomerium/pomerium/internal/telemetry" - "github.com/pomerium/pomerium/internal/telemetry/trace" - "github.com/pomerium/pomerium/internal/urlutil" -) - -// TracingOptions are the options for tracing. -type TracingOptions = trace.TracingOptions - -// NewTracingOptions builds a new TracingOptions from core Options -func NewTracingOptions(o *Options) (*TracingOptions, error) { - tracingOpts := TracingOptions{ - Provider: o.TracingProvider, - Service: telemetry.ServiceName(o.Services), - JaegerAgentEndpoint: o.TracingJaegerAgentEndpoint, - SampleRate: o.TracingSampleRate, - } - - switch o.TracingProvider { - case trace.DatadogTracingProviderName: - tracingOpts.DatadogAddress = o.TracingDatadogAddress - case trace.JaegerTracingProviderName: - if o.TracingJaegerCollectorEndpoint != "" { - jaegerCollectorEndpoint, err := urlutil.ParseAndValidateURL(o.TracingJaegerCollectorEndpoint) - if err != nil { - return nil, fmt.Errorf("config: invalid jaeger endpoint url: %w", err) - } - tracingOpts.JaegerCollectorEndpoint = jaegerCollectorEndpoint - tracingOpts.JaegerAgentEndpoint = o.TracingJaegerAgentEndpoint - } - case trace.ZipkinTracingProviderName: - zipkinEndpoint, err := urlutil.ParseAndValidateURL(o.ZipkinEndpoint) - if err != nil { - return nil, fmt.Errorf("config: invalid zipkin endpoint url: %w", err) - } - tracingOpts.ZipkinEndpoint = zipkinEndpoint - case "": - return &TracingOptions{}, nil - default: - return nil, fmt.Errorf("config: provider %s unknown", o.TracingProvider) - } - - return &tracingOpts, nil -} - -// A TraceManager manages setting up a trace exporter based on configuration options. -type TraceManager struct { - mu sync.Mutex - traceOpts *TracingOptions - provider trace.Provider -} - -// NewTraceManager creates a new TraceManager. -func NewTraceManager(ctx context.Context, src Source) *TraceManager { - ctx = log.WithContext(ctx, func(c zerolog.Context) zerolog.Context { - return c.Str("service", "trace_manager") - }) - mgr := &TraceManager{} - src.OnConfigChange(ctx, mgr.OnConfigChange) - mgr.OnConfigChange(ctx, src.GetConfig()) - return mgr -} - -// Close closes any underlying trace exporter. -func (mgr *TraceManager) Close() error { - mgr.mu.Lock() - defer mgr.mu.Unlock() - - var err error - if mgr.provider != nil { - err = mgr.provider.Unregister() - } - return err -} - -// OnConfigChange updates the manager whenever the configuration is changed. -func (mgr *TraceManager) OnConfigChange(ctx context.Context, cfg *Config) { - mgr.mu.Lock() - defer mgr.mu.Unlock() - - traceOpts, err := NewTracingOptions(cfg.Options) - if err != nil { - log.Ctx(ctx).Error().Err(err).Msg("trace: failed to build tracing options") - return - } - - if reflect.DeepEqual(traceOpts, mgr.traceOpts) { - log.Ctx(ctx).Debug().Msg("no change detected in trace options") - return - } - mgr.traceOpts = traceOpts - - if mgr.provider != nil { - _ = mgr.provider.Unregister() - mgr.provider = nil - } - - if !traceOpts.Enabled() { - return - } - - log.Ctx(ctx).Info().Interface("options", traceOpts).Msg("trace: starting exporter") - - mgr.provider, err = trace.GetProvider(traceOpts) - if err != nil { - log.Ctx(ctx).Error().Err(err).Msg("trace: failed to register exporter") - return - } - - err = mgr.provider.Register(traceOpts) - if err != nil { - log.Ctx(ctx).Error().Err(err).Msg("trace: failed to register exporter") - return - } -} diff --git a/config/trace_client.go b/config/trace_client.go new file mode 100644 index 000000000..9a8dfa858 --- /dev/null +++ b/config/trace_client.go @@ -0,0 +1,44 @@ +package config + +import ( + "errors" + "fmt" + "strings" + + "github.com/pomerium/pomerium/internal/telemetry/trace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" +) + +var ErrNoTracingConfig = errors.New("no tracing config") + +func NewTraceClientFromOptions(opts *Options) (otlptrace.Client, error) { + switch opts.TracingProvider { + case "otlp": + endpoint := opts.TracingOTLPEndpoint + protocol := opts.TracingOTLPProtocol + if protocol == "" && endpoint != "" { + // treat this field as equivalent to OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + protocol = trace.BestEffortProtocolFromOTLPEndpoint(opts.TracingOTLPEndpoint, true) + } + switch strings.ToLower(strings.TrimSpace(protocol)) { + case "grpc": + return otlptracegrpc.NewClient( + otlptracegrpc.WithEndpointURL(endpoint), + ), nil + case "http/protobuf", "": + return otlptracehttp.NewClient( + otlptracehttp.WithEndpointURL(endpoint), + ), nil + default: + return nil, fmt.Errorf(`unknown otlp trace exporter protocol %q, expected "grpc" or "http/protobuf"\n`, protocol) + } + case "none", "noop": + return trace.NoopClient{}, nil + case "": + return nil, ErrNoTracingConfig + default: + return nil, fmt.Errorf(`unknown tracing provider %q, expected one of ["otlp"]`, opts.TracingProvider) + } +} diff --git a/config/trace_test.go b/config/trace_test.go deleted file mode 100644 index af87ac955..000000000 --- a/config/trace_test.go +++ /dev/null @@ -1,161 +0,0 @@ -package config - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "net/url" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "github.com/stretchr/testify/assert" - - "github.com/pomerium/pomerium/internal/telemetry/trace" -) - -func Test_NewTracingOptions(t *testing.T) { - tests := []struct { - name string - opts *Options - want *TracingOptions - wantErr bool - }{ - { - "datadog_good", - &Options{TracingProvider: "datadog"}, - &TracingOptions{Provider: "datadog", Service: "pomerium"}, - false, - }, - { - "jaeger_good", - &Options{TracingProvider: "jaeger", TracingJaegerAgentEndpoint: "foo", TracingJaegerCollectorEndpoint: "http://foo", Services: ServiceAll}, - &TracingOptions{Provider: "jaeger", JaegerAgentEndpoint: "foo", JaegerCollectorEndpoint: &url.URL{Scheme: "http", Host: "foo"}, Service: "pomerium"}, - false, - }, - { - "jaeger_bad", - &Options{TracingProvider: "jaeger", TracingJaegerAgentEndpoint: "foo", TracingJaegerCollectorEndpoint: "badurl"}, - nil, - true, - }, - { - "zipkin_good", - &Options{TracingProvider: "zipkin", ZipkinEndpoint: "https://foo/api/v1/spans", Services: ServiceAuthorize}, - &TracingOptions{Provider: "zipkin", ZipkinEndpoint: &url.URL{Scheme: "https", Host: "foo", Path: "/api/v1/spans"}, Service: "pomerium-authorize"}, - false, - }, - { - "zipkin_bad", - &Options{TracingProvider: "zipkin", ZipkinEndpoint: "notaurl"}, - nil, - true, - }, - { - "noprovider", - &Options{}, - &TracingOptions{}, - false, - }, - { - "fakeprovider", - &Options{TracingProvider: "fake"}, - nil, - true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := NewTracingOptions(tt.opts) - assert.NotEqual(t, err == nil, tt.wantErr, "unexpected error value") - assert.Empty(t, cmp.Diff(tt.want, got)) - }) - } -} - -func Test_TracingEnabled(t *testing.T) { - tests := []struct { - name string - opts *TracingOptions - want bool - }{ - {"enabled", &TracingOptions{Provider: "zipkin"}, true}, - {"not enabled", &TracingOptions{}, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, tt.opts.Enabled(), "unexpected tracing state") - }) - } -} - -func TestTraceManager(t *testing.T) { - ctx, clearTimeout := context.WithTimeout(context.Background(), time.Second*30) - defer clearTimeout() - - type Request struct { - URL string - Name string - } - - incoming := make(chan Request, 100) - - h := http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { - var objs []struct { - Name string - } - json.NewDecoder(r.Body).Decode(&objs) - for _, obj := range objs { - incoming <- Request{Name: obj.Name, URL: r.Host} - } - }) - - srv1 := httptest.NewServer(h) - defer srv1.Close() - srv2 := httptest.NewServer(h) - defer srv2.Close() - - src := NewStaticSource(&Config{Options: &Options{ - TracingProvider: "zipkin", - ZipkinEndpoint: srv1.URL, - TracingSampleRate: 1, - }}) - - _ = NewTraceManager(ctx, src) - - _, span := trace.StartSpan(ctx, "Example") - span.End() - - src.SetConfig(ctx, &Config{Options: &Options{ - TracingProvider: "zipkin", - ZipkinEndpoint: srv2.URL, - TracingSampleRate: 1, - }}) - - _, span = trace.StartSpan(ctx, "Example") - span.End() - - expect := map[Request]struct{}{ - {Name: "example", URL: srv1.Listener.Addr().String()}: {}, - {Name: "example", URL: srv2.Listener.Addr().String()}: {}, - } - - for len(expect) > 0 { - var req Request - select { - case <-ctx.Done(): - t.Error("timeout waiting for requests") - return - case req = <-incoming: - } - - if _, ok := expect[req]; ok { - delete(expect, req) - } else { - t.Error("unexpected request", req) - return - } - } -} diff --git a/databroker/cache.go b/databroker/cache.go index 8105ed932..78f8db5e4 100644 --- a/databroker/cache.go +++ b/databroker/cache.go @@ -10,6 +10,7 @@ import ( "time" "github.com/rs/zerolog" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/metadata" @@ -18,7 +19,7 @@ import ( "github.com/pomerium/pomerium/internal/atomicutil" "github.com/pomerium/pomerium/internal/events" "github.com/pomerium/pomerium/internal/log" - "github.com/pomerium/pomerium/internal/telemetry" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/version" "github.com/pomerium/pomerium/pkg/cryptutil" "github.com/pomerium/pomerium/pkg/envoy/files" @@ -28,6 +29,7 @@ import ( "github.com/pomerium/pomerium/pkg/identity" "github.com/pomerium/pomerium/pkg/identity/legacymanager" "github.com/pomerium/pomerium/pkg/identity/manager" + oteltrace "go.opentelemetry.io/otel/trace" ) // DataBroker represents the databroker service. The databroker service is a simple interface @@ -43,6 +45,8 @@ type DataBroker struct { localGRPCServer *grpc.Server localGRPCConnection *grpc.ClientConn sharedKey *atomicutil.Value[[]byte] + tracerProvider oteltrace.TracerProvider + tracer oteltrace.Tracer } type Options struct { @@ -87,9 +91,12 @@ func New(ctx context.Context, cfg *config.Config, eventsMgr *events.Manager, opt ), ) + tracerProvider := trace.NewTracerProvider(ctx, "Data Broker") + tracer := tracerProvider.Tracer(trace.PomeriumCoreTracer) // No metrics handler because we have one in the control plane. Add one // if we no longer register with that grpc Server localGRPCServer := grpc.NewServer( + grpc.StatsHandler(trace.NewServerStatsHandler(otelgrpc.NewServerHandler(otelgrpc.WithTracerProvider(tracerProvider)))), grpc.ChainStreamInterceptor(log.StreamServerInterceptor(log.Ctx(ctx)), si), grpc.ChainUnaryInterceptor(log.UnaryServerInterceptor(log.Ctx(ctx)), ui), ) @@ -100,12 +107,11 @@ func New(ctx context.Context, cfg *config.Config, eventsMgr *events.Manager, opt } sharedKeyValue := atomicutil.NewValue(sharedKey) - clientStatsHandler := telemetry.NewGRPCClientStatsHandler(cfg.Options.Services) clientDialOptions := []grpc.DialOption{ grpc.WithInsecure(), - grpc.WithChainUnaryInterceptor(clientStatsHandler.UnaryInterceptor, grpcutil.WithUnarySignedJWT(sharedKeyValue.Load)), + grpc.WithChainUnaryInterceptor(grpcutil.WithUnarySignedJWT(sharedKeyValue.Load)), grpc.WithChainStreamInterceptor(grpcutil.WithStreamSignedJWT(sharedKeyValue.Load)), - grpc.WithStatsHandler(clientStatsHandler.Handler), + grpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(tracerProvider))), } ctx = log.WithContext(ctx, func(c zerolog.Context) zerolog.Context { @@ -120,7 +126,7 @@ func New(ctx context.Context, cfg *config.Config, eventsMgr *events.Manager, opt return nil, err } - dataBrokerServer, err := newDataBrokerServer(ctx, cfg) + dataBrokerServer, err := newDataBrokerServer(ctx, tracerProvider, cfg) if err != nil { return nil, err } @@ -133,6 +139,8 @@ func New(ctx context.Context, cfg *config.Config, eventsMgr *events.Manager, opt localGRPCConnection: localGRPCConnection, sharedKey: sharedKeyValue, eventsMgr: eventsMgr, + tracerProvider: tracerProvider, + tracer: tracer, } c.Register(c.localGRPCServer) @@ -202,7 +210,7 @@ func (c *DataBroker) update(ctx context.Context, cfg *config.Config) error { }, c.legacyManagerOptions...) if cfg.Options.SupportsUserRefresh() { - authenticator, err := identity.NewAuthenticator(oauthOptions) + authenticator, err := identity.NewAuthenticator(ctx, c.tracerProvider, oauthOptions) if err != nil { log.Ctx(ctx).Error().Err(err).Msg("databroker: failed to create authenticator") } else { diff --git a/databroker/databroker.go b/databroker/databroker.go index 7817f403d..94087a8a4 100644 --- a/databroker/databroker.go +++ b/databroker/databroker.go @@ -5,6 +5,7 @@ import ( "context" "fmt" + oteltrace "go.opentelemetry.io/otel/trace" "google.golang.org/protobuf/types/known/emptypb" "github.com/pomerium/pomerium/config" @@ -23,7 +24,7 @@ type dataBrokerServer struct { } // newDataBrokerServer creates a new databroker service server. -func newDataBrokerServer(ctx context.Context, cfg *config.Config) (*dataBrokerServer, error) { +func newDataBrokerServer(ctx context.Context, tracerProvider oteltrace.TracerProvider, cfg *config.Config) (*dataBrokerServer, error) { srv := &dataBrokerServer{ sharedKey: atomicutil.NewValue([]byte{}), } @@ -33,7 +34,7 @@ func newDataBrokerServer(ctx context.Context, cfg *config.Config) (*dataBrokerSe return nil, err } - srv.server = databroker.New(ctx, opts...) + srv.server = databroker.New(ctx, tracerProvider, opts...) srv.setKey(cfg) return srv, nil } diff --git a/databroker/databroker_test.go b/databroker/databroker_test.go index 1d1fa46bf..5e379aafb 100644 --- a/databroker/databroker_test.go +++ b/databroker/databroker_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -29,7 +30,7 @@ var lis *bufconn.Listener func init() { lis = bufconn.Listen(bufSize) s := grpc.NewServer() - internalSrv := internal_databroker.New(context.Background()) + internalSrv := internal_databroker.New(context.Background(), trace.NewNoopTracerProvider()) srv := &dataBrokerServer{server: internalSrv, sharedKey: atomicutil.NewValue([]byte{})} databroker.RegisterDataBrokerServiceServer(s, srv) diff --git a/go.mod b/go.mod index 66406d8a2..67ee5fdfe 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,8 @@ go 1.23.0 require ( cloud.google.com/go/storage v1.49.0 - contrib.go.opencensus.io/exporter/jaeger v0.2.1 contrib.go.opencensus.io/exporter/prometheus v0.4.2 - contrib.go.opencensus.io/exporter/zipkin v0.1.2 github.com/CAFxX/httpcompression v0.0.9 - github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20200406135749-5c268882acf0 github.com/VictoriaMetrics/fastcache v1.12.2 github.com/aws/aws-sdk-go-v2 v1.32.7 github.com/aws/aws-sdk-go-v2/config v1.28.7 @@ -46,7 +43,6 @@ require ( github.com/natefinch/atomic v1.0.1 github.com/oapi-codegen/runtime v1.1.1 github.com/open-policy-agent/opa v1.0.0 - github.com/openzipkin/zipkin-go v0.4.3 github.com/peterbourgon/ff/v3 v3.4.0 github.com/pires/go-proxyproto v0.8.0 github.com/pomerium/csrf v1.7.0 @@ -70,15 +66,20 @@ require ( github.com/volatiletech/null/v9 v9.0.0 github.com/yuin/gopher-lua v1.1.1 go.opencensus.io v0.24.0 + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.57.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 + go.opentelemetry.io/contrib/propagators/autoprop v0.57.0 go.opentelemetry.io/otel v1.33.0 - go.opentelemetry.io/otel/bridge/opencensus v1.33.0 - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.33.0 + go.opentelemetry.io/otel/bridge/opencensus v1.32.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0 go.opentelemetry.io/otel/metric v1.33.0 go.opentelemetry.io/otel/sdk v1.33.0 - go.opentelemetry.io/otel/sdk/metric v1.33.0 + go.opentelemetry.io/otel/sdk/metric v1.32.0 go.opentelemetry.io/otel/trace v1.33.0 + go.opentelemetry.io/proto/otlp v1.4.0 go.uber.org/automaxprocs v1.6.0 go.uber.org/mock v0.5.0 go.uber.org/zap v1.27.0 @@ -107,7 +108,6 @@ require ( cloud.google.com/go/monitoring v1.21.2 // indirect dario.cat/mergo v1.0.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect - github.com/DataDog/datadog-go v3.5.0+incompatible // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect @@ -195,7 +195,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect - github.com/philhofer/fwd v1.1.2 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -218,10 +218,8 @@ require ( github.com/tchap/go-patricia/v2 v2.3.1 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect - github.com/tinylib/msgp v1.1.8 // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect - github.com/uber/jaeger-client-go v2.25.0+incompatible // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect @@ -232,10 +230,10 @@ require ( github.com/zeebo/xxh3 v1.0.2 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.31.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 // indirect - go.opentelemetry.io/proto/otlp v1.4.0 // indirect + go.opentelemetry.io/contrib/propagators/aws v1.32.0 // indirect + go.opentelemetry.io/contrib/propagators/b3 v1.32.0 // indirect + go.opentelemetry.io/contrib/propagators/jaeger v1.32.0 // indirect + go.opentelemetry.io/contrib/propagators/ot v1.32.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect golang.org/x/mod v0.20.0 // indirect @@ -243,7 +241,6 @@ require ( golang.org/x/tools v0.24.0 // indirect google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect - gopkg.in/DataDog/dd-trace-go.v1 v1.22.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 962193b06..3a256d5f8 100644 --- a/go.sum +++ b/go.sum @@ -52,12 +52,8 @@ cloud.google.com/go/storage v1.49.0 h1:zenOPBOWHCnojRd9aJZAyQXBYqkJkdQS42dxL55CI cloud.google.com/go/storage v1.49.0/go.mod h1:k1eHhhpLvrPjVGfo0mOUPEJ4Y2+a/Hv5PiwehZI9qGU= cloud.google.com/go/trace v1.11.2 h1:4ZmaBdL8Ng/ajrgKqY5jfvzqMXbrDcBsUGXOT9aqTtI= cloud.google.com/go/trace v1.11.2/go.mod h1:bn7OwXd4pd5rFuAnTrzBuoZ4ax2XQeG3qNgYmfCy0Io= -contrib.go.opencensus.io/exporter/jaeger v0.2.1 h1:yGBYzYMewVL0yO9qqJv3Z5+IRhPdU7e9o/2oKpX4YvI= -contrib.go.opencensus.io/exporter/jaeger v0.2.1/go.mod h1:Y8IsLgdxqh1QxYxPC5IgXVmBaeLUeQFfBeBi9PbeZd0= contrib.go.opencensus.io/exporter/prometheus v0.4.2 h1:sqfsYl5GIY/L570iT+l93ehxaWJs2/OwXtiWwew3oAg= contrib.go.opencensus.io/exporter/prometheus v0.4.2/go.mod h1:dvEHbiKmgvbr5pjaF9fpw1KeYcjrnC1J8B+JKjsZyRQ= -contrib.go.opencensus.io/exporter/zipkin v0.1.2 h1:YqE293IZrKtqPnpwDPH/lOqTWD/s3Iwabycam74JV3g= -contrib.go.opencensus.io/exporter/zipkin v0.1.2/go.mod h1:mP5xM3rrgOjpn79MM8fZbj3gsxcuytSqtH0dxSWW1RE= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -69,10 +65,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CAFxX/httpcompression v0.0.9 h1:0ue2X8dOLEpxTm8tt+OdHcgA+gbDge0OqFQWGKSqgrg= github.com/CAFxX/httpcompression v0.0.9/go.mod h1:XX8oPZA+4IDcfZ0A71Hz0mZsv/YJOgYygkFhizVPilM= -github.com/DataDog/datadog-go v3.5.0+incompatible h1:AShr9cqkF+taHjyQgcBcQUt/ZNK+iPq4ROaZwSX5c/U= -github.com/DataDog/datadog-go v3.5.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20200406135749-5c268882acf0 h1:Y6HFfo8UuntPOpfmUmLb0o3MNYKfUuH2aNmvypsDbY4= -github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20200406135749-5c268882acf0/go.mod h1:/VV3EFO/hTNQZHAqaj+CPGy2+ioFrP4EX3iRwozubhQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 h1:3c8yed4lgqTt+oTQ+JNMDo+F4xprBf+O/il4ZC0nRLw= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0/go.mod h1:obipzmGjfSjam60XLwGfqUkJsfiheAl+TUjG+4yzyPM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 h1:UQ0AhxogsIRZDkElkblfnwjc3IaltCm2HUMvezQaL7s= @@ -86,8 +78,6 @@ github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= github.com/agnivade/levenshtein v1.2.0 h1:U9L4IOT0Y3i0TIlUIDJ7rVUziKi/zPbrJGaFrtYH3SY= @@ -206,10 +196,6 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -277,9 +263,7 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -316,7 +300,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/brotli/go/cbrotli v0.0.0-20230829110029-ed738e842d2f h1:jopqB+UTSdJGEJT8tEqYyE29zN91fi2827oLET8tl7k= @@ -372,8 +355,6 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.14.0 h1:f+jMrjBPl+DL9nI4IQzLUxMq7XrAqFYB7hBPqMNIe8o= github.com/googleapis/gax-go/v2 v2.14.0/go.mod h1:lhBCnjdLrWRaPvLWhmc8IS24m9mr07qSYnHncrgo+zk= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= @@ -450,7 +431,6 @@ github.com/libdns/libdns v0.2.2 h1:O6ws7bAfRPaBsgAYt8MDe2HcNBGC29hkZ9MX2eUSX3s= github.com/libdns/libdns v0.2.2/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ= github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae h1:dIZY4ULFcto4tAFlj1FYZl8ztUZ13bdq+PLY+NOfbyI= github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= -github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/martinlindhe/base36 v1.1.1 h1:1F1MZ5MGghBXDZ2KJ3QfxmiydlWOGB8HCEtkap5NkVg= @@ -505,13 +485,11 @@ github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+ github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.19.1 h1:QXgq3Z8Crl5EL1WBAC98A5sEBHARrAJNzAmMxzLcRF0= github.com/onsi/ginkgo/v2 v2.19.1/go.mod h1:O3DtEWQkPa/F7fBMgmZQKKsluAy8pd3rEQdrjkPb9zA= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= @@ -522,18 +500,10 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/openzipkin/zipkin-go v0.4.3 h1:9EGwpqkgnwdEIJ+Od7QVSEIH+ocmm5nPat0G7sjsSdg= -github.com/openzipkin/zipkin-go v0.4.3/go.mod h1:M9wCJZFWCo2RiY+o1eBCEMe0Dp2S5LDHcMZmk3RmK7c= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/peterbourgon/ff/v3 v3.4.0 h1:QBvM/rizZM1cB0p0lGMdmR7HxZeI/ZrBWB4DqLkMUBc= github.com/peterbourgon/ff/v3 v3.4.0/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ= -github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= -github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= -github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= -github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1 h1:VGcrWe3yk6o+t7BdVNy5UDPWa4OZuDWtE1W1ZbS7Kyw= -github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -543,7 +513,6 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -598,7 +567,6 @@ github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= github.com/quic-go/quic-go v0.48.2 h1:wsKXZPeGWpMpCGSWqOcqpW2wZYic/8T3aqiOID0/KWE= github.com/quic-go/quic-go v0.48.2/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -644,7 +612,6 @@ github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+ github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/sryoya/protorand v0.0.0-20240429201223-e7440656b2a4 h1:/jKH9ivHOUkahZs3zPfJfOmkXDFB6OdsHZ4W8gyDb/c= github.com/sryoya/protorand v0.0.0-20240429201223-e7440656b2a4/go.mod h1:9a23nlv6vzBeVlQq6JQCjljZ6sfzsB6aha1m5Ly1W2Y= -github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -677,17 +644,12 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= -github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= -github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= github.com/tniswong/go.rfcx v0.0.0-20181019234604-07783c52761f h1:C43EMGXFtvYf/zunHR6ivZV7Z6ytg73t0GXwYyicXMQ= github.com/tniswong/go.rfcx v0.0.0-20181019234604-07783c52761f/go.mod h1:N+sR0vLSCTtI6o06PMWsjMB4TVqqDttKNq4iC9wvxVY= -github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U= -github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/valyala/gozstd v1.20.1 h1:xPnnnvjmaDDitMFfDxmQ4vpx0+3CdTg2o3lALvXTU/g= github.com/valyala/gozstd v1.20.1/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= @@ -730,30 +692,40 @@ go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJyS go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/contrib/detectors/gcp v1.31.0 h1:G1JQOreVrfhRkner+l4mrGxmfqYCAuy76asTDAo0xsA= go.opentelemetry.io/contrib/detectors/gcp v1.31.0/go.mod h1:tzQL6E1l+iV44YFTkcAeNQqzXUiekSYP9jjJjXwEd00= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.57.0 h1:qtFISDHKolvIxzSs0gIaiPUPR0Cucb0F2coHC7ZLdps= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.57.0/go.mod h1:Y+Pop1Q6hCOnETWTW4NROK/q1hv50hM7yDaUTjG8lp8= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= +go.opentelemetry.io/contrib/propagators/autoprop v0.57.0 h1:bNPJOdT5154XxzeFmrh8R+PXnV4t3TZEczy8gHEpcpg= +go.opentelemetry.io/contrib/propagators/autoprop v0.57.0/go.mod h1:Tb0j0mK+QatKdCxCKPN7CSzc7kx/q34/KaohJx/N96s= +go.opentelemetry.io/contrib/propagators/aws v1.32.0 h1:NELzr8bW7a7aHVZj5gaep1PfkvoSCGx+1qNGZx/uhhU= +go.opentelemetry.io/contrib/propagators/aws v1.32.0/go.mod h1:XKMrzHNka3eOA+nGEcNKYVL9s77TAhkwQEynYuaRFnQ= +go.opentelemetry.io/contrib/propagators/b3 v1.32.0 h1:MazJBz2Zf6HTN/nK/s3Ru1qme+VhWU5hm83QxEP+dvw= +go.opentelemetry.io/contrib/propagators/b3 v1.32.0/go.mod h1:B0s70QHYPrJwPOwD1o3V/R8vETNOG9N3qZf4LDYvA30= +go.opentelemetry.io/contrib/propagators/jaeger v1.32.0 h1:K/fOyTMD6GELKTIJBaJ9k3ppF2Njt8MeUGBOwfaWXXA= +go.opentelemetry.io/contrib/propagators/jaeger v1.32.0/go.mod h1:ISE6hda//MTWvtngG7p4et3OCngsrTVfl7c6DjN17f8= +go.opentelemetry.io/contrib/propagators/ot v1.32.0 h1:Poy02A4wOZubHyd2hpHPDgZW+rn6EIq0vCwTZJ6Lmu8= +go.opentelemetry.io/contrib/propagators/ot v1.32.0/go.mod h1:cbhaURV+VR3NIMarzDYZU1RDEkXG1fNd1WMP1XCcGkY= go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= -go.opentelemetry.io/otel/bridge/opencensus v1.33.0 h1:sGcK0Wif7sPG3GZG9z8b+tpRZiUHwv27WgmsaZ1wgzM= -go.opentelemetry.io/otel/bridge/opencensus v1.33.0/go.mod h1:LXJy68HiJRu+2yJmVnbDn/F9JS9Kxfsj5WpA5t5NfRY= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.33.0 h1:7F29RDmnlqk6B5d+sUqemt8TBfDqxryYW5gX6L74RFA= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.33.0/go.mod h1:ZiGDq7xwDMKmWDrN1XsXAj0iC7hns+2DhxBFSncNHSE= +go.opentelemetry.io/otel/bridge/opencensus v1.32.0 h1:OVbbFgPG60UolI8ZUs+Z75NnKiO0C9QltXBrqUDImS0= +go.opentelemetry.io/otel/bridge/opencensus v1.32.0/go.mod h1:J5SEiJNu6zzqpcA6+AVpxUKzxNocUMsefgHRpS8zdW8= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0 h1:j7ZSD+5yn+lo3sGV69nW04rRR0jhYnBwjuX3r0HvnK0= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0/go.mod h1:WXbYJTUaZXAbYd8lbgGuvih0yuCfOFC5RJoYnoLcGz8= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 h1:j9+03ymgYhPKmeXGk5Zu+cIZOlVzd9Zv7QIiyItjFBU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0/go.mod h1:Y5+XiUG4Emn1hTfciPzGPJaSI+RpDts6BnCIir0SLqk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0 h1:cMyu9O88joYEaI47CnQkxO1XZdpoTF9fEnW2duIddhw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0/go.mod h1:6Am3rn7P9TVVeXYG+wtcGE7IE1tsQ+bP3AuWcKt/gOI= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0 h1:WDdP9acbMYjbKIyJUhTvtzj601sVJOqgWdUxSdR/Ysc= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I= go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= -go.opentelemetry.io/otel/sdk/metric v1.33.0 h1:Gs5VK9/WUJhNXZgn8MR6ITatvAmKeIuCtNbsP3JkNqU= -go.opentelemetry.io/otel/sdk/metric v1.33.0/go.mod h1:dL5ykHZmm1B1nVRk9dDjChwDmt81MjVp3gLkQRwKf/Q= +go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= +go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= @@ -811,7 +783,6 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= @@ -852,7 +823,6 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= @@ -935,7 +905,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -946,7 +915,6 @@ golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= @@ -959,7 +927,6 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= @@ -1014,7 +981,6 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= @@ -1082,7 +1048,6 @@ google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY= google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= @@ -1113,8 +1078,6 @@ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= -gopkg.in/DataDog/dd-trace-go.v1 v1.22.0 h1:gpWsqqkwUldNZXGJqT69NU9MdEDhLboK1C4nMgR0MWw= -gopkg.in/DataDog/dd-trace-go.v1 v1.22.0/go.mod h1:DVp8HmDh8PuTu2Z0fVVlBsyWaC++fzwVCaGWylTe3tg= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/atomicutil/value.go b/internal/atomicutil/value.go index 665ff3241..279b7ad32 100644 --- a/internal/atomicutil/value.go +++ b/internal/atomicutil/value.go @@ -33,3 +33,14 @@ func (v *Value[T]) Load() T { func (v *Value[T]) Store(val T) { v.value.Store(val) } + +// Swap swaps the value atomically. +func (v *Value[T]) Swap(val T) T { + old, _ := v.value.Swap(val).(T) + return old +} + +// Swap swaps the value atomically. +func (v *Value[T]) CompareAndSwap(old, new T) bool { + return v.value.CompareAndSwap(old, new) +} diff --git a/internal/authenticateflow/authenticateflow.go b/internal/authenticateflow/authenticateflow.go index bdd7f8be8..2fd7edafe 100644 --- a/internal/authenticateflow/authenticateflow.go +++ b/internal/authenticateflow/authenticateflow.go @@ -4,11 +4,17 @@ package authenticateflow import ( + "context" "fmt" "time" + oteltrace "go.opentelemetry.io/otel/trace" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/structpb" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/pkg/grpc" "github.com/pomerium/pomerium/pkg/grpc/user" "github.com/pomerium/pomerium/pkg/identity" @@ -33,3 +39,23 @@ func populateUserFromClaims(u *user.User, claims map[string]any) { u.Claims[k] = vs } } + +var outboundDatabrokerTraceClientOpts = []trace.ClientStatsHandlerOption{ + trace.WithStatsInterceptor(ignoreNotFoundErrors), +} + +func ignoreNotFoundErrors(ctx context.Context, rs stats.RPCStats) stats.RPCStats { + if end, ok := rs.(*stats.End); ok && end.IsClient() { + if status.Code(end.Error) == codes.NotFound { + oteltrace.SpanFromContext(ctx).AddEvent("status code: NotFound") + return &stats.End{ + Client: end.Client, + BeginTime: end.BeginTime, + EndTime: end.EndTime, + Trailer: end.Trailer, + Error: nil, + } + } + } + return rs +} diff --git a/internal/authenticateflow/stateful.go b/internal/authenticateflow/stateful.go index 44050325d..58d62d516 100644 --- a/internal/authenticateflow/stateful.go +++ b/internal/authenticateflow/stateful.go @@ -9,7 +9,11 @@ import ( "net/url" "time" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "go.opentelemetry.io/otel" + oteltrace "go.opentelemetry.io/otel/trace" "golang.org/x/oauth2" + googlegrpc "google.golang.org/grpc" "google.golang.org/protobuf/types/known/timestamppb" "github.com/pomerium/pomerium/config" @@ -19,6 +23,7 @@ import ( "github.com/pomerium/pomerium/internal/httputil" "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/sessions" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/urlutil" "github.com/pomerium/pomerium/pkg/cryptutil" "github.com/pomerium/pomerium/pkg/grpc" @@ -56,7 +61,7 @@ type Stateful struct { // NewStateful initializes the authentication flow for the given configuration // and session store. -func NewStateful(ctx context.Context, cfg *config.Config, sessionStore sessions.SessionStore) (*Stateful, error) { +func NewStateful(ctx context.Context, tracerProvider oteltrace.TracerProvider, cfg *config.Config, sessionStore sessions.SessionStore) (*Stateful, error) { s := &Stateful{ sessionDuration: cfg.Options.CookieExpire, sessionStore: sessionStore, @@ -94,7 +99,10 @@ func NewStateful(ctx context.Context, cfg *config.Config, sessionStore sessions. InstallationID: cfg.Options.InstallationID, ServiceName: cfg.Options.Services, SignedJWTKey: s.sharedKey, - }) + }, googlegrpc.WithStatsHandler(trace.NewClientStatsHandler( + otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(tracerProvider)), + outboundDatabrokerTraceClientOpts..., + ))) if err != nil { return nil, err } @@ -316,7 +324,7 @@ func (s *Stateful) LogAuthenticateEvent(*http.Request) {} // AuthenticateSignInURL returns a URL to redirect the user to the authenticate // domain. func (s *Stateful) AuthenticateSignInURL( - _ context.Context, queryParams url.Values, redirectURL *url.URL, idpID string, + ctx context.Context, queryParams url.Values, redirectURL *url.URL, idpID string, ) (string, error) { signinURL := s.authenticateURL.ResolveReference(&url.URL{ Path: "/.pomerium/sign_in", @@ -327,6 +335,7 @@ func (s *Stateful) AuthenticateSignInURL( } queryParams.Set(urlutil.QueryRedirectURI, redirectURL.String()) queryParams.Set(urlutil.QueryIdentityProviderID, idpID) + otel.GetTextMapPropagator().Inject(ctx, trace.PomeriumURLQueryCarrier(queryParams)) signinURL.RawQuery = queryParams.Encode() redirectTo := urlutil.NewSignedURL(s.sharedKey, signinURL).String() diff --git a/internal/authenticateflow/stateful_test.go b/internal/authenticateflow/stateful_test.go index 173374410..aa6e85841 100644 --- a/internal/authenticateflow/stateful_test.go +++ b/internal/authenticateflow/stateful_test.go @@ -15,6 +15,7 @@ import ( "github.com/go-jose/go-jose/v3/jwt" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace" "go.uber.org/mock/gomock" "golang.org/x/oauth2" "google.golang.org/grpc" @@ -69,7 +70,7 @@ func TestStatefulSignIn(t *testing.T) { tt := tt t.Run(tt.name, func(t *testing.T) { sessionStore := &mstore.Store{SaveError: tt.saveError} - flow, err := NewStateful(context.Background(), &config.Config{Options: opts}, sessionStore) + flow, err := NewStateful(context.Background(), trace.NewNoopTracerProvider(), &config.Config{Options: opts}, sessionStore) if err != nil { t.Fatal(err) } @@ -123,12 +124,12 @@ func TestStatefulAuthenticateSignInURL(t *testing.T) { opts.AuthenticateURLString = "https://authenticate.example.com" key := cryptutil.NewKey() opts.SharedKey = base64.StdEncoding.EncodeToString(key) - flow, err := NewStateful(context.Background(), &config.Config{Options: opts}, nil) + flow, err := NewStateful(context.Background(), trace.NewNoopTracerProvider(), &config.Config{Options: opts}, nil) require.NoError(t, err) t.Run("NilQueryParams", func(t *testing.T) { redirectURL := &url.URL{Scheme: "https", Host: "example.com"} - u, err := flow.AuthenticateSignInURL(nil, nil, redirectURL, "fake-idp-id") + u, err := flow.AuthenticateSignInURL(context.Background(), nil, redirectURL, "fake-idp-id") assert.NoError(t, err) parsed, _ := url.Parse(u) assert.NoError(t, urlutil.NewSignedURL(key, parsed).Validate()) @@ -143,7 +144,7 @@ func TestStatefulAuthenticateSignInURL(t *testing.T) { redirectURL := &url.URL{Scheme: "https", Host: "example.com"} q := url.Values{} q.Set("foo", "bar") - u, err := flow.AuthenticateSignInURL(nil, q, redirectURL, "fake-idp-id") + u, err := flow.AuthenticateSignInURL(context.Background(), q, redirectURL, "fake-idp-id") assert.NoError(t, err) parsed, _ := url.Parse(u) assert.NoError(t, urlutil.NewSignedURL(key, parsed).Validate()) @@ -238,7 +239,7 @@ func TestStatefulCallback(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - flow, err := NewStateful(context.Background(), &config.Config{Options: opts}, tt.sessionStore) + flow, err := NewStateful(context.Background(), trace.NewNoopTracerProvider(), &config.Config{Options: opts}, tt.sessionStore) if err != nil { t.Fatal(err) } @@ -289,7 +290,7 @@ func TestStatefulCallback(t *testing.T) { func TestStatefulRevokeSession(t *testing.T) { opts := config.NewDefaultOptions() - flow, err := NewStateful(context.Background(), &config.Config{Options: opts}, nil) + flow, err := NewStateful(context.Background(), trace.NewNoopTracerProvider(), &config.Config{Options: opts}, nil) require.NoError(t, err) ctrl := gomock.NewController(t) @@ -367,7 +368,7 @@ func TestPersistSession(t *testing.T) { opts := config.NewDefaultOptions() opts.CookieExpire = 4 * time.Hour - flow, err := NewStateful(context.Background(), &config.Config{Options: opts}, nil) + flow, err := NewStateful(context.Background(), trace.NewNoopTracerProvider(), &config.Config{Options: opts}, nil) require.NoError(t, err) ctrl := gomock.NewController(t) diff --git a/internal/authenticateflow/stateless.go b/internal/authenticateflow/stateless.go index e89e8199a..ef188eae9 100644 --- a/internal/authenticateflow/stateless.go +++ b/internal/authenticateflow/stateless.go @@ -10,6 +10,7 @@ import ( "github.com/go-jose/go-jose/v3" "golang.org/x/oauth2" + googlegrpc "google.golang.org/grpc" "google.golang.org/protobuf/encoding/protojson" "github.com/pomerium/pomerium/authenticate/events" @@ -20,6 +21,7 @@ import ( "github.com/pomerium/pomerium/internal/httputil" "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/sessions" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/urlutil" "github.com/pomerium/pomerium/pkg/cryptutil" "github.com/pomerium/pomerium/pkg/grpc" @@ -29,6 +31,9 @@ import ( "github.com/pomerium/pomerium/pkg/grpc/user" "github.com/pomerium/pomerium/pkg/hpke" "github.com/pomerium/pomerium/pkg/identity" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "go.opentelemetry.io/otel" + oteltrace "go.opentelemetry.io/otel/trace" ) // Stateless implements the stateless authentication flow. In this flow, the @@ -56,18 +61,21 @@ type Stateless struct { dataBrokerClient databroker.DataBrokerServiceClient - getIdentityProvider func(options *config.Options, idpID string) (identity.Authenticator, error) + getIdentityProvider func(ctx context.Context, tracerProvider oteltrace.TracerProvider, options *config.Options, idpID string) (identity.Authenticator, error) profileTrimFn func(*identitypb.Profile) authEventFn events.AuthEventFn + + tracerProvider oteltrace.TracerProvider } // NewStateless initializes the authentication flow for the given // configuration, session store, and additional options. func NewStateless( ctx context.Context, + tracerProvider oteltrace.TracerProvider, cfg *config.Config, sessionStore sessions.SessionStore, - getIdentityProvider func(options *config.Options, idpID string) (identity.Authenticator, error), + getIdentityProvider func(ctx context.Context, tracerProvider oteltrace.TracerProvider, options *config.Options, idpID string) (identity.Authenticator, error), profileTrimFn func(*identitypb.Profile), authEventFn events.AuthEventFn, ) (*Stateless, error) { @@ -77,6 +85,7 @@ func NewStateless( getIdentityProvider: getIdentityProvider, profileTrimFn: profileTrimFn, authEventFn: authEventFn, + tracerProvider: tracerProvider, } var err error @@ -137,7 +146,10 @@ func NewStateless( InstallationID: cfg.Options.InstallationID, ServiceName: cfg.Options.Services, SignedJWTKey: sharedKey, - }) + }, googlegrpc.WithStatsHandler(trace.NewClientStatsHandler( + otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(tracerProvider)), + outboundDatabrokerTraceClientOpts..., + ))) if err != nil { return nil, err } @@ -154,7 +166,7 @@ func (s *Stateless) VerifySession(ctx context.Context, r *http.Request, _ *sessi return fmt.Errorf("identity profile load error: %w", err) } - authenticator, err := s.getIdentityProvider(s.options, profile.GetProviderId()) + authenticator, err := s.getIdentityProvider(ctx, s.tracerProvider, s.options, profile.GetProviderId()) if err != nil { return fmt.Errorf("couldn't get identity provider: %w", err) } @@ -355,6 +367,7 @@ func (s *Stateless) AuthenticateSignInURL( for k, v := range queryParams { q[k] = v } + otel.GetTextMapPropagator().Inject(ctx, trace.PomeriumURLQueryCarrier(q)) authenticateURLWithParams.RawQuery = q.Encode() return urlutil.SignInURL( diff --git a/internal/benchmarks/latency_bench_test.go b/internal/benchmarks/latency_bench_test.go index e3de8d94c..17aa19a96 100644 --- a/internal/benchmarks/latency_bench_test.go +++ b/internal/benchmarks/latency_bench_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/pomerium/pomerium/config" "github.com/pomerium/pomerium/internal/testenv" "github.com/pomerium/pomerium/internal/testenv/envutil" "github.com/pomerium/pomerium/internal/testenv/scenarios" @@ -48,7 +49,8 @@ func TestRequestLatency(t *testing.T) { for i := range numRoutes { routes[i] = up.Route(). From(env.SubdomainURL(fmt.Sprintf("from-%d", i))). - PPL(fmt.Sprintf(`{"allow":{"and":["email":{"is":"user%d@example.com"}]}}`, i)) + Policy(func(p *config.Policy) { p.AllowPublicUnauthenticatedAccess = true }) + // PPL(fmt.Sprintf(`{"allow":{"and":["email":{"is":"user%d@example.com"}]}}`, i)) } env.AddUpstream(up) diff --git a/internal/controlplane/events.go b/internal/controlplane/events.go index 90bc086d2..dabbc8cc5 100644 --- a/internal/controlplane/events.go +++ b/internal/controlplane/events.go @@ -7,6 +7,8 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/google/uuid" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + googlegrpc "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" @@ -76,7 +78,7 @@ func (srv *Server) getDataBrokerClient(ctx context.Context) (databrokerpb.DataBr InstallationID: cfg.Options.InstallationID, ServiceName: cfg.Options.Services, SignedJWTKey: sharedKey, - }) + }, googlegrpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(srv.tracerProvider)))) if err != nil { return nil, fmt.Errorf("controlplane: error creating databroker connection: %w", err) } diff --git a/internal/controlplane/grpc_accesslog.go b/internal/controlplane/grpc_accesslog.go index b3f078cfc..ae5f11960 100644 --- a/internal/controlplane/grpc_accesslog.go +++ b/internal/controlplane/grpc_accesslog.go @@ -7,6 +7,8 @@ import ( envoy_data_accesslog_v3 "github.com/envoyproxy/go-control-plane/envoy/data/accesslog/v3" envoy_service_accesslog_v3 "github.com/envoyproxy/go-control-plane/envoy/service/accesslog/v3" "github.com/rs/zerolog" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/pomerium/pomerium/internal/log" ) @@ -21,7 +23,11 @@ func (srv *Server) StreamAccessLogs(stream envoy_service_accesslog_v3.AccessLogS for { msg, err := stream.Recv() if err != nil { - log.Ctx(stream.Context()).Error().Err(err).Msg("access log stream error, disconnecting") + if status.Code(err) == codes.Canceled { + log.Ctx(stream.Context()).Debug().Err(err).Msg("access log stream canceled") + } else { + log.Ctx(stream.Context()).Error().Err(err).Msg("access log stream error, disconnecting") + } return err } diff --git a/internal/controlplane/http.go b/internal/controlplane/http.go index d648fefbf..7378f536f 100644 --- a/internal/controlplane/http.go +++ b/internal/controlplane/http.go @@ -2,6 +2,7 @@ package controlplane import ( + "context" "fmt" "net/http" "time" @@ -9,18 +10,21 @@ import ( "github.com/CAFxX/httpcompression" "github.com/gorilla/mux" "github.com/rs/zerolog" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "github.com/pomerium/pomerium/config" "github.com/pomerium/pomerium/internal/handlers" "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/middleware" "github.com/pomerium/pomerium/internal/telemetry" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/urlutil" hpke_handlers "github.com/pomerium/pomerium/pkg/hpke/handlers" "github.com/pomerium/pomerium/pkg/telemetry/requestid" ) -func (srv *Server) addHTTPMiddleware(root *mux.Router, logger *zerolog.Logger, _ *config.Config) { +func (srv *Server) addHTTPMiddleware(ctx context.Context, root *mux.Router, _ *config.Config) { + logger := log.Ctx(ctx) compressor, err := httpcompression.DefaultAdapter() if err != nil { panic(err) @@ -48,6 +52,7 @@ func (srv *Server) addHTTPMiddleware(root *mux.Router, logger *zerolog.Logger, _ root.Use(telemetry.HTTPStatsHandler(func() string { return srv.currentConfig.Load().Options.InstallationID }, srv.name)) + root.Use(trace.NewHTTPMiddleware(otelhttp.WithTracerProvider(srv.tracerProvider))) } func (srv *Server) mountCommonEndpoints(root *mux.Router, cfg *config.Config) error { diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go index 587d19318..ccb37e514 100644 --- a/internal/controlplane/server.go +++ b/internal/controlplane/server.go @@ -11,6 +11,8 @@ import ( envoy_service_discovery_v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" "github.com/gorilla/mux" "github.com/rs/zerolog" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/health/grpc_health_v1" @@ -25,7 +27,6 @@ import ( "github.com/pomerium/pomerium/internal/events" "github.com/pomerium/pomerium/internal/httputil/reproxy" "github.com/pomerium/pomerium/internal/log" - "github.com/pomerium/pomerium/internal/telemetry" "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/urlutil" "github.com/pomerium/pomerium/internal/version" @@ -34,6 +35,7 @@ import ( "github.com/pomerium/pomerium/pkg/grpcutil" "github.com/pomerium/pomerium/pkg/httputil" "github.com/pomerium/pomerium/pkg/telemetry/requestid" + oteltrace "go.opentelemetry.io/otel/trace" ) // A Service can be mounted on the control plane. @@ -43,6 +45,7 @@ type Service interface { // A Server is the control-plane gRPC and HTTP servers. type Server struct { + coltracepb.UnimplementedTraceServiceServer GRPCListener net.Listener GRPCServer *grpc.Server HTTPListener net.Listener @@ -66,6 +69,9 @@ type Server struct { proxySvc Service haveSetCapacity map[string]bool + + tracerProvider oteltrace.TracerProvider + tracer oteltrace.Tracer } // NewServer creates a new Server. Listener ports are chosen by the OS. @@ -76,7 +82,10 @@ func NewServer( eventsMgr *events.Manager, fileMgr *filemgr.Manager, ) (*Server, error) { + tracerProvider := trace.NewTracerProvider(ctx, "Control Plane") srv := &Server{ + tracerProvider: tracerProvider, + tracer: tracerProvider.Tracer(trace.PomeriumCoreTracer), metricsMgr: metricsMgr, EventsMgr: eventsMgr, filemgr: fileMgr, @@ -105,7 +114,7 @@ func NewServer( ), ) srv.GRPCServer = grpc.NewServer( - grpc.StatsHandler(telemetry.NewGRPCServerStatsHandler(cfg.Options.Services)), + grpc.StatsHandler(trace.NewServerStatsHandler(otelgrpc.NewServerHandler(otelgrpc.WithTracerProvider(tracerProvider)))), grpc.ChainUnaryInterceptor( log.UnaryServerInterceptor(log.Ctx(ctx)), requestid.UnaryServerInterceptor(), @@ -177,7 +186,9 @@ func NewServer( srv.xdsmgr = xdsmgr.NewManager(res) envoy_service_discovery_v3.RegisterAggregatedDiscoveryServiceServer(srv.GRPCServer, srv.xdsmgr) - + if exp := trace.ExporterServerFromContext(ctx); exp != nil { + coltracepb.RegisterTraceServiceServer(srv.GRPCServer, exp) + } return srv, nil } @@ -241,7 +252,7 @@ func (srv *Server) Run(ctx context.Context) error { // OnConfigChange updates the pomerium config options. func (srv *Server) OnConfigChange(ctx context.Context, cfg *config.Config) error { - ctx, span := trace.StartSpan(ctx, "controlplane.Server.OnConfigChange") + ctx, span := srv.tracer.Start(ctx, "controlplane.Server.OnConfigChange") defer span.End() select { @@ -265,7 +276,7 @@ func (srv *Server) EnableProxy(ctx context.Context, svc Service) error { } func (srv *Server) update(ctx context.Context, cfg *config.Config) error { - ctx, span := trace.StartSpan(ctx, "controlplane.Server.update") + ctx, span := srv.tracer.Start(ctx, "controlplane.Server.update") defer span.End() if err := srv.updateRouter(ctx, cfg); err != nil { @@ -283,7 +294,7 @@ func (srv *Server) update(ctx context.Context, cfg *config.Config) error { func (srv *Server) updateRouter(ctx context.Context, cfg *config.Config) error { httpRouter := mux.NewRouter() - srv.addHTTPMiddleware(httpRouter, log.Ctx(ctx), cfg) + srv.addHTTPMiddleware(ctx, httpRouter, cfg) if err := srv.mountCommonEndpoints(httpRouter, cfg); err != nil { return err } diff --git a/internal/controlplane/xds.go b/internal/controlplane/xds.go index 6e8343526..aa47ab6ac 100644 --- a/internal/controlplane/xds.go +++ b/internal/controlplane/xds.go @@ -9,7 +9,6 @@ import ( "golang.org/x/sync/errgroup" "github.com/pomerium/pomerium/internal/log" - "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/pkg/cryptutil" "github.com/pomerium/pomerium/pkg/protoutil" ) @@ -21,7 +20,7 @@ const ( ) func (srv *Server) buildDiscoveryResources(ctx context.Context) (map[string][]*envoy_service_discovery_v3.Resource, error) { - ctx, span := trace.StartSpan(ctx, "controlplane.Server.buildDiscoveryResources") + ctx, span := srv.tracer.Start(ctx, "controlplane.Server.buildDiscoveryResources") defer span.End() cfg := srv.currentConfig.Load() diff --git a/internal/databroker/config_source.go b/internal/databroker/config_source.go index b76986ab3..403bcf481 100644 --- a/internal/databroker/config_source.go +++ b/internal/databroker/config_source.go @@ -22,6 +22,9 @@ import ( "github.com/pomerium/pomerium/pkg/grpc/databroker" "github.com/pomerium/pomerium/pkg/grpcutil" "github.com/pomerium/pomerium/pkg/health" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + oteltrace "go.opentelemetry.io/otel/trace" + googlegrpc "google.golang.org/grpc" ) // ConfigSource provides a new Config source that decorates an underlying config with @@ -35,6 +38,7 @@ type ConfigSource struct { updaterHash uint64 cancel func() enableValidation bool + tracerProvider oteltrace.TracerProvider config.ChangeDispatcher } @@ -50,11 +54,13 @@ type EnableConfigValidation bool // NewConfigSource creates a new ConfigSource. func NewConfigSource( ctx context.Context, + tracerProvider oteltrace.TracerProvider, underlying config.Source, enableValidation EnableConfigValidation, listeners ...config.ChangeListener, ) *ConfigSource { src := &ConfigSource{ + tracerProvider: tracerProvider, enableValidation: bool(enableValidation), dbConfigs: map[string]dbConfig{}, outboundGRPCConnection: new(grpc.CachedOutboundGRPClientConn), @@ -85,7 +91,7 @@ func (src *ConfigSource) GetConfig() *config.Config { type firstTime bool func (src *ConfigSource) rebuild(ctx context.Context, firstTime firstTime) { - _, span := trace.StartSpan(ctx, "databroker.config_source.rebuild") + _, span := trace.Continue(ctx, "databroker.config_source.rebuild") defer span.End() now := time.Now() @@ -259,7 +265,8 @@ func (src *ConfigSource) runUpdater(ctx context.Context, cfg *config.Config) { ctx, src.cancel = context.WithCancel(ctx) - cc, err := src.outboundGRPCConnection.Get(ctx, connectionOptions) + cc, err := src.outboundGRPCConnection.Get(ctx, connectionOptions, + googlegrpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(src.tracerProvider)))) if err != nil { log.Ctx(ctx).Error().Err(err).Msg("databroker: failed to create gRPC connection to data broker") return diff --git a/internal/databroker/config_source_test.go b/internal/databroker/config_source_test.go index 4116b5b1e..bafa9d24a 100644 --- a/internal/databroker/config_source_test.go +++ b/internal/databroker/config_source_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" "google.golang.org/protobuf/proto" @@ -41,7 +42,7 @@ func TestConfigSource(t *testing.T) { defer func() { _ = li.Close() }() _, outboundPort, _ := net.SplitHostPort(li.Addr().String()) - dataBrokerServer := New(ctx) + dataBrokerServer := New(ctx, trace.NewNoopTracerProvider()) srv := grpc.NewServer() databroker.RegisterDataBrokerServiceServer(srv, dataBrokerServer) go func() { _ = srv.Serve(li) }() @@ -65,7 +66,7 @@ func TestConfigSource(t *testing.T) { OutboundPort: outboundPort, Options: base, }) - src := NewConfigSource(ctx, baseSource, EnableConfigValidation(true), func(_ context.Context, cfg *config.Config) { + src := NewConfigSource(ctx, trace.NewNoopTracerProvider(), baseSource, EnableConfigValidation(true), func(_ context.Context, cfg *config.Config) { cfgs <- cfg }) cfgs <- src.GetConfig() diff --git a/internal/databroker/registry.go b/internal/databroker/registry.go index fee359e4a..605b7bf4c 100644 --- a/internal/databroker/registry.go +++ b/internal/databroker/registry.go @@ -9,7 +9,6 @@ import ( "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/registry" "github.com/pomerium/pomerium/internal/registry/inmemory" - "github.com/pomerium/pomerium/internal/telemetry/trace" registrypb "github.com/pomerium/pomerium/pkg/grpc/registry" "github.com/pomerium/pomerium/pkg/storage" ) @@ -25,7 +24,7 @@ func (stream registryWatchServer) Context() context.Context { // Report calls the registry Report method. func (srv *Server) Report(ctx context.Context, req *registrypb.RegisterRequest) (*registrypb.RegisterResponse, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.Report") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Report") defer span.End() r, err := srv.getRegistry(ctx) @@ -38,7 +37,7 @@ func (srv *Server) Report(ctx context.Context, req *registrypb.RegisterRequest) // List calls the registry List method. func (srv *Server) List(ctx context.Context, req *registrypb.ListRequest) (*registrypb.ServiceList, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.List") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.List") defer span.End() r, err := srv.getRegistry(ctx) @@ -52,7 +51,7 @@ func (srv *Server) List(ctx context.Context, req *registrypb.ListRequest) (*regi // Watch calls the registry Watch method. func (srv *Server) Watch(req *registrypb.ListRequest, stream registrypb.Registry_WatchServer) error { ctx := stream.Context() - ctx, span := trace.StartSpan(ctx, "databroker.grpc.Watch") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Watch") defer span.End() r, err := srv.getRegistry(ctx) diff --git a/internal/databroker/server.go b/internal/databroker/server.go index 4a6f207b3..2671bf0cc 100644 --- a/internal/databroker/server.go +++ b/internal/databroker/server.go @@ -22,22 +22,28 @@ import ( "github.com/pomerium/pomerium/pkg/storage" "github.com/pomerium/pomerium/pkg/storage/inmemory" "github.com/pomerium/pomerium/pkg/storage/postgres" + oteltrace "go.opentelemetry.io/otel/trace" ) // Server implements the databroker service using an in memory database. type Server struct { cfg *serverConfig - mu sync.RWMutex - backend storage.Backend - backendCtx context.Context - registry registry.Interface + mu sync.RWMutex + backend storage.Backend + backendCtx context.Context + registry registry.Interface + tracerProvider oteltrace.TracerProvider + tracer oteltrace.Tracer } // New creates a new server. -func New(ctx context.Context, options ...ServerOption) *Server { +func New(ctx context.Context, tracerProvider oteltrace.TracerProvider, options ...ServerOption) *Server { + tracer := tracerProvider.Tracer(trace.PomeriumCoreTracer) srv := &Server{ - backendCtx: ctx, + backendCtx: ctx, + tracerProvider: tracerProvider, + tracer: tracer, } srv.UpdateConfig(ctx, options...) return srv @@ -74,7 +80,7 @@ func (srv *Server) UpdateConfig(ctx context.Context, options ...ServerOption) { // AcquireLease acquires a lease. func (srv *Server) AcquireLease(ctx context.Context, req *databroker.AcquireLeaseRequest) (*databroker.AcquireLeaseResponse, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.AcquireLease") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.AcquireLease") defer span.End() log.Ctx(ctx).Debug(). Str("name", req.GetName()). @@ -101,7 +107,7 @@ func (srv *Server) AcquireLease(ctx context.Context, req *databroker.AcquireLeas // Get gets a record from the in-memory list. func (srv *Server) Get(ctx context.Context, req *databroker.GetRequest) (*databroker.GetResponse, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.Get") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Get") defer span.End() log.Ctx(ctx).Debug(). Str("type", req.GetType()). @@ -128,7 +134,7 @@ func (srv *Server) Get(ctx context.Context, req *databroker.GetRequest) (*databr // ListTypes lists all the record types. func (srv *Server) ListTypes(ctx context.Context, _ *emptypb.Empty) (*databroker.ListTypesResponse, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.ListTypes") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.ListTypes") defer span.End() log.Ctx(ctx).Debug().Msg("list types") @@ -145,7 +151,7 @@ func (srv *Server) ListTypes(ctx context.Context, _ *emptypb.Empty) (*databroker // Query queries for records. func (srv *Server) Query(ctx context.Context, req *databroker.QueryRequest) (*databroker.QueryResponse, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.Query") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Query") defer span.End() log.Ctx(ctx).Debug(). Str("type", req.GetType()). @@ -198,7 +204,7 @@ func (srv *Server) Query(ctx context.Context, req *databroker.QueryRequest) (*da // Put updates an existing record or adds a new one. func (srv *Server) Put(ctx context.Context, req *databroker.PutRequest) (*databroker.PutResponse, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.Put") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Put") defer span.End() records := req.GetRecords() @@ -237,7 +243,7 @@ func (srv *Server) Put(ctx context.Context, req *databroker.PutRequest) (*databr // Patch updates specific fields of an existing record. func (srv *Server) Patch(ctx context.Context, req *databroker.PatchRequest) (*databroker.PatchResponse, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.Patch") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Patch") defer span.End() records := req.GetRecords() @@ -276,9 +282,9 @@ func (srv *Server) Patch(ctx context.Context, req *databroker.PatchRequest) (*da // ReleaseLease releases a lease. func (srv *Server) ReleaseLease(ctx context.Context, req *databroker.ReleaseLeaseRequest) (*emptypb.Empty, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.ReleaseLease") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.ReleaseLease") defer span.End() - log.Ctx(ctx).Debug(). + log.Ctx(ctx).Trace(). Str("name", req.GetName()). Str("id", req.GetId()). Msg("release lease") @@ -298,9 +304,9 @@ func (srv *Server) ReleaseLease(ctx context.Context, req *databroker.ReleaseLeas // RenewLease releases a lease. func (srv *Server) RenewLease(ctx context.Context, req *databroker.RenewLeaseRequest) (*emptypb.Empty, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.RenewLease") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.RenewLease") defer span.End() - log.Ctx(ctx).Debug(). + log.Ctx(ctx).Trace(). Str("name", req.GetName()). Str("id", req.GetId()). Dur("duration", req.GetDuration().AsDuration()). @@ -323,7 +329,7 @@ func (srv *Server) RenewLease(ctx context.Context, req *databroker.RenewLeaseReq // SetOptions sets options for a type in the databroker. func (srv *Server) SetOptions(ctx context.Context, req *databroker.SetOptionsRequest) (*databroker.SetOptionsResponse, error) { - ctx, span := trace.StartSpan(ctx, "databroker.grpc.SetOptions") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.SetOptions") defer span.End() backend, err := srv.getBackend(ctx) @@ -346,7 +352,7 @@ func (srv *Server) SetOptions(ctx context.Context, req *databroker.SetOptionsReq // Sync streams updates for the given record type. func (srv *Server) Sync(req *databroker.SyncRequest, stream databroker.DataBrokerService_SyncServer) error { ctx := stream.Context() - ctx, span := trace.StartSpan(ctx, "databroker.grpc.Sync") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Sync") defer span.End() ctx, cancel := context.WithCancel(ctx) @@ -384,7 +390,7 @@ func (srv *Server) Sync(req *databroker.SyncRequest, stream databroker.DataBroke // SyncLatest returns the latest value of every record in the databroker as a stream of records. func (srv *Server) SyncLatest(req *databroker.SyncLatestRequest, stream databroker.DataBrokerService_SyncLatestServer) error { ctx := stream.Context() - ctx, span := trace.StartSpan(ctx, "databroker.grpc.SyncLatest") + ctx, span := srv.tracer.Start(ctx, "databroker.grpc.SyncLatest") defer span.End() ctx, cancel := context.WithCancel(ctx) diff --git a/internal/databroker/server_test.go b/internal/databroker/server_test.go index aa1c7c5c8..f837e127f 100644 --- a/internal/databroker/server_test.go +++ b/internal/databroker/server_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace/noop" "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -49,10 +50,9 @@ func (h testSyncerHandler) UpdateRecords(ctx context.Context, serverVersion uint } func newServer(cfg *serverConfig) *Server { - return &Server{ - cfg: cfg, - backendCtx: context.Background(), - } + srv := New(context.Background(), noop.NewTracerProvider()) + srv.cfg = cfg + return srv } func TestServer_Get(t *testing.T) { diff --git a/internal/middleware/middleware.go b/internal/middleware/middleware.go index db00f0488..af714465c 100644 --- a/internal/middleware/middleware.go +++ b/internal/middleware/middleware.go @@ -14,7 +14,7 @@ import ( func SetHeaders(headers map[string]string) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ctx, span := trace.StartSpan(r.Context(), "middleware.SetHeaders") + ctx, span := trace.Continue(r.Context(), "middleware.SetHeaders") defer span.End() for key, val := range headers { w.Header().Set(key, val) @@ -29,7 +29,7 @@ func SetHeaders(headers map[string]string) func(next http.Handler) http.Handler func ValidateSignature(sharedKey []byte) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return httputil.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { - ctx, span := trace.StartSpan(r.Context(), "middleware.ValidateSignature") + ctx, span := trace.Continue(r.Context(), "middleware.ValidateSignature") defer span.End() if err := ValidateRequestURL(r, sharedKey); err != nil { return httputil.NewError(http.StatusBadRequest, err) diff --git a/internal/registry/reporter.go b/internal/registry/reporter.go index f34b87e91..964287210 100644 --- a/internal/registry/reporter.go +++ b/internal/registry/reporter.go @@ -15,18 +15,23 @@ import ( "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/pkg/grpc" pb "github.com/pomerium/pomerium/pkg/grpc/registry" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + oteltrace "go.opentelemetry.io/otel/trace" + googlegrpc "google.golang.org/grpc" ) // Reporter periodically submits a list of services available on this instance to the service registry type Reporter struct { cancel func() outboundGRPCConnection *grpc.CachedOutboundGRPClientConn + tracerProvider oteltrace.TracerProvider } // NewReporter creates a new Reporter. -func NewReporter() *Reporter { +func NewReporter(tracerProvider oteltrace.TracerProvider) *Reporter { return &Reporter{ outboundGRPCConnection: new(grpc.CachedOutboundGRPClientConn), + tracerProvider: tracerProvider, } } @@ -52,7 +57,7 @@ func (r *Reporter) OnConfigChange(ctx context.Context, cfg *config.Config) { InstallationID: cfg.Options.InstallationID, ServiceName: cfg.Options.Services, SignedJWTKey: sharedKey, - }) + }, googlegrpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(r.tracerProvider)))) if err != nil { log.Ctx(ctx).Error().Err(err).Msg("connecting to registry") return diff --git a/internal/retry/retry.go b/internal/retry/retry.go index ed3d6f8ac..612623191 100644 --- a/internal/retry/retry.go +++ b/internal/retry/retry.go @@ -9,6 +9,7 @@ import ( "time" "github.com/rs/zerolog" + "go.opentelemetry.io/otel/trace" "github.com/pomerium/pomerium/internal/log" ) @@ -47,7 +48,10 @@ restart: backoff: for { interval := backoff.NextBackOff() - log.Ctx(ctx).Info().Msgf("backing off for %s...", interval.String()) + span := trace.SpanFromContext(ctx) + msg := fmt.Sprintf("backing off for %s...", interval.String()) + span.AddEvent(msg) + log.Ctx(ctx).Info().Msg(msg) timer := time.NewTimer(interval) s := makeSelect(ctx, watches, name, timer.C, fn) next, err := s.Exec(ctx) diff --git a/internal/telemetry/grpc.go b/internal/telemetry/grpc.go deleted file mode 100644 index 54863838d..000000000 --- a/internal/telemetry/grpc.go +++ /dev/null @@ -1,100 +0,0 @@ -package telemetry - -import ( - "context" - "strings" - - "go.opencensus.io/plugin/ocgrpc" - "go.opencensus.io/plugin/ochttp/propagation/b3" - "go.opencensus.io/trace" - "go.opencensus.io/trace/propagation" - "google.golang.org/grpc" - "google.golang.org/grpc/metadata" - grpcstats "google.golang.org/grpc/stats" - - "github.com/pomerium/pomerium/internal/telemetry/metrics" -) - -const ( - grpcTraceBinHeader = "grpc-trace-bin" - b3TraceIDHeader = "x-b3-traceid" - b3SpanIDHeader = "x-b3-spanid" -) - -type tagRPCHandler interface { - TagRPC(context.Context, *grpcstats.RPCTagInfo) context.Context -} - -// GRPCServerStatsHandler provides a grpc stats.Handler for metrics and tracing for a pomerium service -type GRPCServerStatsHandler struct { - service string - metricsHandler tagRPCHandler - grpcstats.Handler -} - -// TagRPC implements grpc.stats.Handler and adds metrics and tracing metadata to the context of a given RPC -func (h *GRPCServerStatsHandler) TagRPC(ctx context.Context, tagInfo *grpcstats.RPCTagInfo) context.Context { - // the opencensus trace handler only supports grpc-trace-bin, so we use that code and support b3 too - - md, _ := metadata.FromIncomingContext(ctx) - name := strings.TrimPrefix(tagInfo.FullMethodName, "/") - name = strings.Replace(name, "/", ".", -1) - - var parent trace.SpanContext - hasParent := false - if traceBin := md[grpcTraceBinHeader]; len(traceBin) > 0 { - parent, hasParent = propagation.FromBinary([]byte(traceBin[0])) - } - - if hdr := md[b3TraceIDHeader]; len(hdr) > 0 { - if tid, ok := b3.ParseTraceID(hdr[0]); ok { - parent.TraceID = tid - hasParent = true - } - } - if hdr := md[b3SpanIDHeader]; len(hdr) > 0 { - if sid, ok := b3.ParseSpanID(hdr[0]); ok { - parent.SpanID = sid - hasParent = true - } - } - - if hasParent { - ctx, _ = trace.StartSpanWithRemoteParent(ctx, name, parent, - trace.WithSpanKind(trace.SpanKindServer)) - } else { - ctx, _ = trace.StartSpan(ctx, name, - trace.WithSpanKind(trace.SpanKindServer)) - } - - // ocgrpc's TagRPC must be called to attach the context rpcDataKey correctly - // https://github.com/census-instrumentation/opencensus-go/blob/bf52d9df8bb2d44cad934587ab946794456cf3c8/plugin/ocgrpc/server_stats_handler.go#L45 - metricCtx := h.metricsHandler.TagRPC(h.Handler.TagRPC(ctx, tagInfo), tagInfo) - return metricCtx -} - -// NewGRPCServerStatsHandler creates a new GRPCServerStatsHandler for a pomerium service -func NewGRPCServerStatsHandler(service string) grpcstats.Handler { - return &GRPCServerStatsHandler{ - service: ServiceName(service), - Handler: &ocgrpc.ServerHandler{}, - metricsHandler: metrics.NewGRPCServerMetricsHandler(ServiceName(service)), - } -} - -// GRPCClientStatsHandler provides DialOptions for grpc clients to instrument network calls with -// both metrics and tracing -type GRPCClientStatsHandler struct { - UnaryInterceptor grpc.UnaryClientInterceptor - // TODO: we should have a streaming interceptor too - grpcstats.Handler -} - -// NewGRPCClientStatsHandler returns a new GRPCClientStatsHandler used to create -// telemetry related client DialOptions -func NewGRPCClientStatsHandler(service string) *GRPCClientStatsHandler { - return &GRPCClientStatsHandler{ - Handler: &ocgrpc.ClientHandler{}, - UnaryInterceptor: metrics.GRPCClientInterceptor(ServiceName(service)), - } -} diff --git a/internal/telemetry/grpc_test.go b/internal/telemetry/grpc_test.go deleted file mode 100644 index 038f58ada..000000000 --- a/internal/telemetry/grpc_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package telemetry - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "go.opencensus.io/plugin/ocgrpc" - "go.opencensus.io/plugin/ochttp/propagation/b3" - "go.opencensus.io/trace" - "google.golang.org/grpc/metadata" - grpcstats "google.golang.org/grpc/stats" -) - -type mockTagHandler struct { - called bool -} - -type mockCtxTag string - -func (m *mockTagHandler) TagRPC(ctx context.Context, _ *grpcstats.RPCTagInfo) context.Context { - m.called = true - return context.WithValue(ctx, mockCtxTag("added"), "true") -} - -func Test_GRPCServerStatsHandler(t *testing.T) { - metricsHandler := &mockTagHandler{} - h := &GRPCServerStatsHandler{ - metricsHandler: metricsHandler, - Handler: &ocgrpc.ServerHandler{}, - } - - ctx := context.WithValue(context.Background(), mockCtxTag("original"), "true") - ctx = metadata.NewIncomingContext(ctx, metadata.MD{ - b3TraceIDHeader: {"9de3f6756f315fef"}, - b3SpanIDHeader: {"b4f83d3096b6bf9c"}, - }) - ctx = h.TagRPC(ctx, &grpcstats.RPCTagInfo{}) - - assert.True(t, metricsHandler.called) - assert.Equal(t, ctx.Value(mockCtxTag("added")), "true") - assert.Equal(t, ctx.Value(mockCtxTag("original")), "true") - - span := trace.FromContext(ctx) - expectedTraceID, _ := b3.ParseTraceID("9de3f6756f315fef") - assert.Equal(t, expectedTraceID, span.SpanContext().TraceID) -} diff --git a/internal/telemetry/trace/buffer.go b/internal/telemetry/trace/buffer.go new file mode 100644 index 000000000..f74ccb8ef --- /dev/null +++ b/internal/telemetry/trace/buffer.go @@ -0,0 +1,266 @@ +package trace + +import ( + "cmp" + "encoding/base64" + "maps" + "slices" + "sync" + "unique" + + "github.com/pomerium/pomerium/internal/hashutil" + oteltrace "go.opentelemetry.io/otel/trace" + commonv1 "go.opentelemetry.io/proto/otlp/common/v1" + resourcev1 "go.opentelemetry.io/proto/otlp/resource/v1" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" + "google.golang.org/protobuf/proto" +) + +type ScopeBuffer struct { + scope *ScopeInfo + spans []*tracev1.Span +} + +func (sb *ScopeBuffer) Insert(spans ...*tracev1.Span) { + sb.spans = append(sb.spans, spans...) +} + +func NewScopeBuffer(scope *ScopeInfo) *ScopeBuffer { + return &ScopeBuffer{ + scope: scope, + } +} + +type ResourceBuffer struct { + resource *ResourceInfo + spansByScope map[string]*ScopeBuffer +} + +func NewResourceBuffer(resource *ResourceInfo) *ResourceBuffer { + return &ResourceBuffer{ + resource: resource, + spansByScope: make(map[string]*ScopeBuffer), + } +} + +func (rb *ResourceBuffer) Insert(scope *ScopeInfo, span *tracev1.Span) { + var spans *ScopeBuffer + if sp, ok := rb.spansByScope[scope.ID()]; ok { + spans = sp + } else { + spans = NewScopeBuffer(scope) + rb.spansByScope[scope.ID()] = spans + } + spans.Insert(span) +} + +func (rb *ResourceBuffer) Flush() []*tracev1.ScopeSpans { + out := make([]*tracev1.ScopeSpans, 0, len(rb.spansByScope)) + for _, key := range slices.Sorted(maps.Keys(rb.spansByScope)) { + spans := rb.spansByScope[key] + slices.SortStableFunc(spans.spans, func(a, b *tracev1.Span) int { + return cmp.Compare(a.StartTimeUnixNano, b.StartTimeUnixNano) + }) + scopeSpans := &tracev1.ScopeSpans{ + Scope: spans.scope.Scope, + SchemaUrl: spans.scope.Schema, + Spans: spans.spans, + } + out = append(out, scopeSpans) + } + clear(rb.spansByScope) + return out +} + +func (rb *ResourceBuffer) FlushAs(rewriteTraceID unique.Handle[oteltrace.TraceID]) []*tracev1.ScopeSpans { + out := make([]*tracev1.ScopeSpans, 0, len(rb.spansByScope)) + for _, key := range slices.Sorted(maps.Keys(rb.spansByScope)) { + spans := rb.spansByScope[key] + { + id := rewriteTraceID.Value() + for _, span := range spans.spans { + copy(span.TraceId, id[:]) + } + } + slices.SortStableFunc(spans.spans, func(a, b *tracev1.Span) int { + return cmp.Compare(a.StartTimeUnixNano, b.StartTimeUnixNano) + }) + scopeSpans := &tracev1.ScopeSpans{ + Scope: spans.scope.Scope, + SchemaUrl: spans.scope.Schema, + Spans: spans.spans, + } + out = append(out, scopeSpans) + } + clear(rb.spansByScope) + return out +} + +func (rb *ResourceBuffer) Merge(other *ResourceBuffer) { + for scope, otherSpans := range other.spansByScope { + if ourSpans, ok := rb.spansByScope[scope]; !ok { + rb.spansByScope[scope] = otherSpans + } else { + ourSpans.Insert(otherSpans.spans...) + } + } + clear(other.spansByScope) +} + +func (rb *ResourceBuffer) MergeAs(other *ResourceBuffer, rewriteTraceID unique.Handle[oteltrace.TraceID]) { + for scope, otherSpans := range other.spansByScope { + { + id := rewriteTraceID.Value() + for _, span := range otherSpans.spans { + copy(span.TraceId, id[:]) + } + } + if ourSpans, ok := rb.spansByScope[scope]; !ok { + rb.spansByScope[scope] = otherSpans + } else { + ourSpans.Insert(otherSpans.spans...) + } + } + clear(other.spansByScope) +} + +type Buffer struct { + scopesByResourceID map[string]*ResourceBuffer +} + +func NewBuffer() *Buffer { + return &Buffer{ + scopesByResourceID: make(map[string]*ResourceBuffer), + } +} + +func (b *Buffer) Insert(resource *ResourceInfo, scope *ScopeInfo, span *tracev1.Span) { + resourceEq := resource.ID() + var scopes *ResourceBuffer + if sc, ok := b.scopesByResourceID[resourceEq]; ok { + scopes = sc + } else { + scopes = NewResourceBuffer(resource) + b.scopesByResourceID[resourceEq] = scopes + } + scopes.Insert(scope, span) +} + +func (b *Buffer) Flush() []*tracev1.ResourceSpans { + out := make([]*tracev1.ResourceSpans, 0, len(b.scopesByResourceID)) + for _, key := range slices.Sorted(maps.Keys(b.scopesByResourceID)) { + scopes := b.scopesByResourceID[key] + resourceSpans := &tracev1.ResourceSpans{ + Resource: scopes.resource.Resource, + ScopeSpans: scopes.Flush(), + SchemaUrl: scopes.resource.Schema, + } + out = append(out, resourceSpans) + } + clear(b.scopesByResourceID) + return out +} + +func (b *Buffer) FlushAs(rewriteTraceID unique.Handle[oteltrace.TraceID]) []*tracev1.ResourceSpans { + out := make([]*tracev1.ResourceSpans, 0, len(b.scopesByResourceID)) + for _, key := range slices.Sorted(maps.Keys(b.scopesByResourceID)) { + scopes := b.scopesByResourceID[key] + resourceSpans := &tracev1.ResourceSpans{ + Resource: scopes.resource.Resource, + ScopeSpans: scopes.FlushAs(rewriteTraceID), + SchemaUrl: scopes.resource.Schema, + } + out = append(out, resourceSpans) + } + clear(b.scopesByResourceID) + return out +} + +func (b *Buffer) Merge(other *Buffer) { + if b != nil { + for k, otherV := range other.scopesByResourceID { + if v, ok := b.scopesByResourceID[k]; !ok { + b.scopesByResourceID[k] = otherV + } else { + v.Merge(otherV) + } + } + } + clear(other.scopesByResourceID) +} + +func (b *Buffer) MergeAs(other *Buffer, rewriteTraceID unique.Handle[oteltrace.TraceID]) { + if b != nil { + for k, otherV := range other.scopesByResourceID { + if v, ok := b.scopesByResourceID[k]; !ok { + newRb := NewResourceBuffer(otherV.resource) + newRb.MergeAs(otherV, rewriteTraceID) + b.scopesByResourceID[k] = newRb + } else { + v.MergeAs(otherV, rewriteTraceID) + } + } + } + clear(other.scopesByResourceID) +} + +func (b *Buffer) IsEmpty() bool { + return len(b.scopesByResourceID) == 0 +} + +type ResourceInfo struct { + Resource *resourcev1.Resource + Schema string + ID func() string +} + +func NewResourceInfo(resource *resourcev1.Resource, resourceSchema string) *ResourceInfo { + ri := &ResourceInfo{ + Resource: resource, + Schema: resourceSchema, + } + ri.ID = sync.OnceValue(ri.computeID) + return ri +} + +func (ri *ResourceInfo) computeID() string { + hash := hashutil.NewDigest() + tmp := resourcev1.Resource{ + Attributes: ri.Resource.Attributes, + } + bytes, _ := proto.Marshal(&tmp) + hash.WriteStringWithLen(ri.Schema) + hash.WriteWithLen(bytes) + return base64.StdEncoding.EncodeToString(hash.Sum(nil)) +} + +type ScopeInfo struct { + Scope *commonv1.InstrumentationScope + Schema string + ID func() string +} + +func NewScopeInfo(scope *commonv1.InstrumentationScope, scopeSchema string) *ScopeInfo { + si := &ScopeInfo{ + Scope: scope, + Schema: scopeSchema, + } + si.ID = sync.OnceValue(si.computeID) + return si +} + +func (si *ScopeInfo) computeID() string { + if si.Scope == nil { + return "(unknown)" + } + hash := hashutil.NewDigest() + tmp := commonv1.InstrumentationScope{ + Name: si.Scope.Name, + Version: si.Scope.Version, + Attributes: si.Scope.Attributes, + } + bytes, _ := proto.Marshal(&tmp) + hash.WriteStringWithLen(si.Schema) + hash.WriteWithLen(bytes) + return base64.StdEncoding.EncodeToString(hash.Sum(nil)) +} diff --git a/internal/telemetry/trace/buffer_test.go b/internal/telemetry/trace/buffer_test.go new file mode 100644 index 000000000..557c9489a --- /dev/null +++ b/internal/telemetry/trace/buffer_test.go @@ -0,0 +1,229 @@ +package trace_test + +import ( + "encoding/binary" + "fmt" + "testing" + "unique" + + "github.com/pomerium/pomerium/internal/telemetry/trace" + "github.com/pomerium/pomerium/internal/testutil" + "github.com/stretchr/testify/assert" + oteltrace "go.opentelemetry.io/otel/trace" + commonv1 "go.opentelemetry.io/proto/otlp/common/v1" + resourcev1 "go.opentelemetry.io/proto/otlp/resource/v1" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" +) + +type ( + Trace uint32 + Span uint32 + Scope uint32 + Schema uint32 + Resource uint32 +) + +func (n Trace) String() string { return fmt.Sprintf("Trace %d", n) } +func (n Span) String() string { return fmt.Sprintf("Span %d", n) } +func (n Scope) String() string { return fmt.Sprintf("Scope %d", n) } +func (n Schema) String() string { return fmt.Sprintf("Schema %d", n) } +func (n Resource) String() string { return fmt.Sprintf("Resource %d", n) } + +func (n Trace) ID() unique.Handle[oteltrace.TraceID] { + id, _ := trace.ToTraceID(n.B()) + return id +} + +func (n Trace) B() []byte { + var id oteltrace.TraceID + binary.BigEndian.PutUint32(id[12:], uint32(n)) + return id[:] +} + +func (n Span) ID() oteltrace.SpanID { + id, _ := trace.ToSpanID(n.B()) + return id +} + +func (n Span) B() []byte { + var id oteltrace.SpanID + binary.BigEndian.PutUint32(id[4:], uint32(n)) + return id[:] +} + +func (n Scope) Make(s ...Schema) *trace.ScopeInfo { + if len(s) == 0 { + s = append(s, Schema(0)) + } + return trace.NewScopeInfo(&commonv1.InstrumentationScope{ + Name: n.String(), + Version: "v1", + Attributes: []*commonv1.KeyValue{ + { + Key: "id", + Value: &commonv1.AnyValue{ + Value: &commonv1.AnyValue_IntValue{ + IntValue: int64(n), + }, + }, + }, + }, + }, s[0].String()) +} + +func (n Resource) Make(s ...Schema) *trace.ResourceInfo { + if len(s) == 0 { + s = append(s, Schema(0)) + } + return trace.NewResourceInfo(&resourcev1.Resource{ + Attributes: []*commonv1.KeyValue{ + { + Key: "name", + Value: &commonv1.AnyValue{ + Value: &commonv1.AnyValue_StringValue{ + StringValue: n.String(), + }, + }, + }, + { + Key: "id", + Value: &commonv1.AnyValue{ + Value: &commonv1.AnyValue_IntValue{ + IntValue: int64(n), + }, + }, + }, + }, + }, s[0].String()) +} + +func Traceparent(trace Trace, span Span, sampled bool) string { + sampledStr := "00" + if sampled { + sampledStr = "01" + } + return fmt.Sprintf("00-%s-%s-%s", trace.ID().Value(), span.ID(), sampledStr) +} + +func TestBuffer(t *testing.T) { + t.Parallel() + + // start time determines sort order of spans within a resource+scope group + s := []*tracev1.Span{ + {TraceId: Trace(1).B(), SpanId: Span(1).B(), StartTimeUnixNano: 1}, + {TraceId: Trace(1).B(), SpanId: Span(2).B(), StartTimeUnixNano: 2}, + {TraceId: Trace(2).B(), SpanId: Span(3).B(), StartTimeUnixNano: 3}, + {TraceId: Trace(2).B(), SpanId: Span(4).B(), StartTimeUnixNano: 4}, + {TraceId: Trace(1).B(), SpanId: Span(5).B(), StartTimeUnixNano: 5}, + {TraceId: Trace(1).B(), SpanId: Span(6).B(), StartTimeUnixNano: 6}, + {TraceId: Trace(2).B(), SpanId: Span(7).B(), StartTimeUnixNano: 7}, + {TraceId: Trace(2).B(), SpanId: Span(8).B(), StartTimeUnixNano: 8}, + {TraceId: Trace(1).B(), SpanId: Span(9).B(), StartTimeUnixNano: 9}, + {TraceId: Trace(1).B(), SpanId: Span(10).B(), StartTimeUnixNano: 10}, + {TraceId: Trace(2).B(), SpanId: Span(11).B(), StartTimeUnixNano: 11}, + {TraceId: Trace(2).B(), SpanId: Span(12).B(), StartTimeUnixNano: 12}, + {TraceId: Trace(1).B(), SpanId: Span(13).B(), StartTimeUnixNano: 13}, + {TraceId: Trace(1).B(), SpanId: Span(14).B(), StartTimeUnixNano: 14}, + {TraceId: Trace(2).B(), SpanId: Span(15).B(), StartTimeUnixNano: 15}, + {TraceId: Trace(2).B(), SpanId: Span(16).B(), StartTimeUnixNano: 16}, + } + + newTestBuffer := func() *trace.Buffer { + b := trace.NewBuffer() + b.Insert(Resource(1).Make(), Scope(1).Make(), s[0]) + b.Insert(Resource(1).Make(), Scope(1).Make(), s[1]) + b.Insert(Resource(1).Make(), Scope(1).Make(), s[2]) + b.Insert(Resource(1).Make(), Scope(1).Make(), s[3]) + b.Insert(Resource(1).Make(), Scope(2).Make(), s[4]) + b.Insert(Resource(1).Make(), Scope(2).Make(), s[5]) + b.Insert(Resource(1).Make(), Scope(2).Make(), s[6]) + b.Insert(Resource(1).Make(), Scope(2).Make(), s[7]) + b.Insert(Resource(2).Make(), Scope(1).Make(), s[8]) + b.Insert(Resource(2).Make(), Scope(1).Make(), s[9]) + b.Insert(Resource(2).Make(), Scope(1).Make(), s[10]) + b.Insert(Resource(2).Make(), Scope(1).Make(), s[11]) + b.Insert(Resource(2).Make(), Scope(2).Make(), s[12]) + b.Insert(Resource(2).Make(), Scope(2).Make(), s[13]) + b.Insert(Resource(2).Make(), Scope(2).Make(), s[14]) + b.Insert(Resource(2).Make(), Scope(2).Make(), s[15]) + return b + } + + newExpectedSpans := func() []*tracev1.ResourceSpans { + return []*tracev1.ResourceSpans{ + { + Resource: Resource(1).Make().Resource, + ScopeSpans: []*tracev1.ScopeSpans{ + { + Scope: Scope(1).Make().Scope, + Spans: []*tracev1.Span{s[0], s[1], s[2], s[3]}, + SchemaUrl: Schema(0).String(), + }, + { + Scope: Scope(2).Make().Scope, + Spans: []*tracev1.Span{s[4], s[5], s[6], s[7]}, + SchemaUrl: Schema(0).String(), + }, + }, + SchemaUrl: Schema(0).String(), + }, + { + Resource: Resource(2).Make().Resource, + ScopeSpans: []*tracev1.ScopeSpans{ + { + Scope: Scope(1).Make().Scope, + Spans: []*tracev1.Span{s[8], s[9], s[10], s[11]}, + SchemaUrl: Schema(0).String(), + }, + { + Scope: Scope(2).Make().Scope, + Spans: []*tracev1.Span{s[12], s[13], s[14], s[15]}, + SchemaUrl: Schema(0).String(), + }, + }, + SchemaUrl: Schema(0).String(), + }, + } + } + t.Run("Flush", func(t *testing.T) { + b := newTestBuffer() + actual := b.Flush() + assert.True(t, b.IsEmpty()) + testutil.AssertProtoEqual(t, newExpectedSpans(), actual) + }) + t.Run("FlushAs", func(t *testing.T) { + b := newTestBuffer() + actual := b.FlushAs(Trace(100).ID()) + assert.True(t, b.IsEmpty()) + expected := newExpectedSpans() + for _, resourceSpans := range expected { + for _, scopeSpans := range resourceSpans.ScopeSpans { + for _, span := range scopeSpans.Spans { + span.TraceId = Trace(100).B() + } + } + } + testutil.AssertProtoEqual(t, expected, actual) + }) + + t.Run("Default scope", func(t *testing.T) { + b := trace.NewBuffer() + b.Insert(Resource(1).Make(Schema(2)), trace.NewScopeInfo(nil, ""), s[0]) + b.Insert(Resource(1).Make(Schema(2)), trace.NewScopeInfo(nil, ""), s[1]) + b.Insert(Resource(1).Make(Schema(2)), trace.NewScopeInfo(nil, ""), s[2]) + actual := b.Flush() + testutil.AssertProtoEqual(t, []*tracev1.ResourceSpans{ + { + Resource: Resource(1).Make(Schema(2)).Resource, + ScopeSpans: []*tracev1.ScopeSpans{ + { + Scope: nil, + Spans: []*tracev1.Span{s[0], s[1], s[2]}, + SchemaUrl: "", + }, + }, + SchemaUrl: Schema(2).String(), + }, + }, actual) + }) +} diff --git a/internal/telemetry/trace/carriers.go b/internal/telemetry/trace/carriers.go new file mode 100644 index 000000000..ac1a1bf05 --- /dev/null +++ b/internal/telemetry/trace/carriers.go @@ -0,0 +1,28 @@ +package trace + +import ( + "net/url" + + "go.opentelemetry.io/otel/propagation" +) + +type PomeriumURLQueryCarrier url.Values + +// Get implements propagation.TextMapCarrier. +func (q PomeriumURLQueryCarrier) Get(key string) string { + return url.Values(q).Get("pomerium_" + key) +} + +// Set implements propagation.TextMapCarrier. +func (q PomeriumURLQueryCarrier) Set(key string, value string) { + url.Values(q).Set("pomerium_"+key, value) +} + +// Keys implements propagation.TextMapCarrier. +func (q PomeriumURLQueryCarrier) Keys() []string { + // this function is never called in otel, and the way it would be + // implemented in this instance is unclear. + panic("unimplemented") +} + +var _ propagation.TextMapCarrier = PomeriumURLQueryCarrier{} diff --git a/internal/telemetry/trace/carriers_test.go b/internal/telemetry/trace/carriers_test.go new file mode 100644 index 000000000..1e63df4b6 --- /dev/null +++ b/internal/telemetry/trace/carriers_test.go @@ -0,0 +1,30 @@ +package trace_test + +import ( + "net/url" + "testing" + + "github.com/pomerium/pomerium/internal/telemetry/trace" + "github.com/stretchr/testify/assert" +) + +func TestPomeriumURLQueryCarrier(t *testing.T) { + t.Parallel() + values := url.Values{} + carrier := trace.PomeriumURLQueryCarrier(values) + assert.Empty(t, carrier.Get("foo")) + carrier.Set("foo", "bar") + assert.Equal(t, url.Values{ + "pomerium_foo": []string{"bar"}, + }, values) + assert.Equal(t, "bar", carrier.Get("foo")) + carrier.Set("foo", "bar2") + assert.Equal(t, url.Values{ + "pomerium_foo": []string{"bar2"}, + }, values) + assert.Equal(t, "bar2", carrier.Get("foo")) + + assert.Panics(t, func() { + carrier.Keys() + }) +} diff --git a/internal/telemetry/trace/client.go b/internal/telemetry/trace/client.go new file mode 100644 index 000000000..5ab4510d5 --- /dev/null +++ b/internal/telemetry/trace/client.go @@ -0,0 +1,227 @@ +package trace + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "strings" + "sync" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + v1 "go.opentelemetry.io/proto/otlp/trace/v1" +) + +var ( + ErrNoClient = errors.New("no client") + ErrClientStopped = errors.New("client is stopped") +) + +type SyncClient interface { + otlptrace.Client + + Update(ctx context.Context, newClient otlptrace.Client) error +} + +func NewSyncClient(client otlptrace.Client) SyncClient { + return &syncClient{ + client: client, + } +} + +type syncClient struct { + mu sync.Mutex + client otlptrace.Client + waitForNewClient chan struct{} +} + +var _ SyncClient = (*syncClient)(nil) + +// Start implements otlptrace.Client. +func (ac *syncClient) Start(ctx context.Context) error { + ac.mu.Lock() + defer ac.mu.Unlock() + if ac.waitForNewClient != nil { + panic("bug: Start called during Stop or Update") + } + if ac.client == nil { + return ErrNoClient + } + return ac.client.Start(ctx) +} + +// Stop implements otlptrace.Client. +func (ac *syncClient) Stop(ctx context.Context) error { + ac.mu.Lock() + defer ac.mu.Unlock() + if ac.waitForNewClient != nil { + panic("bug: Stop called concurrently") + } + return ac.resetLocked(ctx, nil) +} + +func (ac *syncClient) resetLocked(ctx context.Context, newClient otlptrace.Client) error { + if ac.client == nil { + return ErrNoClient + } + ac.waitForNewClient = make(chan struct{}) + ac.mu.Unlock() + + err := ac.client.Stop(ctx) + + ac.mu.Lock() + close(ac.waitForNewClient) + ac.waitForNewClient = nil + ac.client = newClient + return err +} + +// UploadTraces implements otlptrace.Client. +func (ac *syncClient) UploadTraces(ctx context.Context, protoSpans []*v1.ResourceSpans) error { + ac.mu.Lock() + if ac.waitForNewClient != nil { + wait := ac.waitForNewClient + ac.mu.Unlock() + select { + case <-wait: + ac.mu.Lock() + case <-ctx.Done(): + return context.Cause(ctx) + } + } else if ac.client == nil { + ac.mu.Unlock() + return ErrNoClient + } + client := ac.client + ac.mu.Unlock() + if client == nil { + return ErrClientStopped + } + return client.UploadTraces(ctx, protoSpans) +} + +func (ac *syncClient) Update(ctx context.Context, newClient otlptrace.Client) error { + if newClient != nil { + if err := newClient.Start(ctx); err != nil { + return fmt.Errorf("error starting new client: %w", err) + } + } + ac.mu.Lock() + defer ac.mu.Unlock() + if ac.waitForNewClient != nil { + panic("bug: Update called during Stop") + } + if newClient == ac.client { + return nil + } + return ac.resetLocked(ctx, newClient) +} + +// NewRemoteClientFromEnv creates an otlp trace client using the well-known +// environment variables defined in the [OpenTelemetry documentation]. +// +// [OpenTelemetry documentation]: https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/ +func NewRemoteClientFromEnv() otlptrace.Client { + if os.Getenv("OTEL_SDK_DISABLED") == "true" { + return NoopClient{} + } + + exporter, ok := os.LookupEnv("OTEL_TRACES_EXPORTER") + if !ok { + exporter = "none" + } + + switch strings.ToLower(strings.TrimSpace(exporter)) { + case "none", "noop", "": + return NoopClient{} + case "otlp": + var protocol string + if v, ok := os.LookupEnv("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"); ok { + protocol = v + } else if v, ok := os.LookupEnv("OTEL_EXPORTER_OTLP_PROTOCOL"); ok { + protocol = v + } else { + // try to guess the expected protocol from the port number + var endpoint string + var specific bool + if v, ok := os.LookupEnv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"); ok { + endpoint = v + specific = true + } else if v, ok := os.LookupEnv("OTEL_EXPORTER_OTLP_ENDPOINT"); ok { + endpoint = v + } + protocol = BestEffortProtocolFromOTLPEndpoint(endpoint, specific) + } + switch strings.ToLower(strings.TrimSpace(protocol)) { + case "grpc": + return otlptracegrpc.NewClient() + case "http/protobuf", "": + return otlptracehttp.NewClient() + default: + otel.Handle(fmt.Errorf(`unknown otlp trace exporter protocol %q, expected "grpc" or "http/protobuf"`, protocol)) + return NoopClient{} + } + default: + otel.Handle(fmt.Errorf(`unknown otlp trace exporter %q, expected "otlp" or "none"`, exporter)) + return NoopClient{} + } +} + +func BestEffortProtocolFromOTLPEndpoint(endpoint string, specificEnv bool) string { + if endpoint == "" { + return "" + } + u, err := url.Parse(endpoint) + if err != nil { + return "" + } + switch u.Port() { + case "4318": + return "http/protobuf" + case "4317": + return "grpc" + default: + // For http, if the signal-specific form of the endpoint env variable + // (e.g. $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT) is used, the /v1/ + // ^^^^^^ + // path must be present. Otherwise, the path must _not_ be present, + // because the sdk will add it. + // This doesn't apply to grpc endpoints, so assume grpc if there is a + // conflict here. + hasPath := len(strings.Trim(u.Path, "/")) > 0 + switch { + case hasPath && specificEnv: + return "http/protobuf" + case !hasPath && specificEnv: + return "grpc" + case hasPath && !specificEnv: + // would be invalid for http, so assume it's grpc on a subpath + return "grpc" + case !hasPath && !specificEnv: + // could be either, but default to http + return "http/protobuf" + } + panic("unreachable") + } +} + +type NoopClient struct{} + +// Start implements otlptrace.Client. +func (n NoopClient) Start(context.Context) error { + return nil +} + +// Stop implements otlptrace.Client. +func (n NoopClient) Stop(context.Context) error { + return nil +} + +// UploadTraces implements otlptrace.Client. +func (n NoopClient) UploadTraces(context.Context, []*v1.ResourceSpans) error { + return nil +} diff --git a/internal/telemetry/trace/client_test.go b/internal/telemetry/trace/client_test.go new file mode 100644 index 000000000..17a3f15ca --- /dev/null +++ b/internal/telemetry/trace/client_test.go @@ -0,0 +1,468 @@ +package trace_test + +import ( + "context" + "fmt" + "regexp" + "runtime" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/pomerium/pomerium/internal/log" + "github.com/pomerium/pomerium/internal/telemetry/trace" + "github.com/pomerium/pomerium/internal/telemetry/trace/mock_otlptrace" + "github.com/pomerium/pomerium/internal/testenv" + "github.com/pomerium/pomerium/internal/testenv/scenarios" + "github.com/pomerium/pomerium/internal/testenv/snippets" + "github.com/pomerium/pomerium/internal/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" + "go.uber.org/mock/gomock" +) + +func TestSyncClient(t *testing.T) { + t.Run("No client", func(t *testing.T) { + sc := trace.NewSyncClient(nil) + assert.ErrorIs(t, sc.Start(context.Background()), trace.ErrNoClient) + assert.ErrorIs(t, sc.UploadTraces(context.Background(), nil), trace.ErrNoClient) + assert.ErrorIs(t, sc.Stop(context.Background()), trace.ErrNoClient) + }) + + t.Run("Valid client", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockClient := mock_otlptrace.NewMockClient(ctrl) + start := mockClient.EXPECT(). + Start(gomock.Any()). + Return(nil) + upload := mockClient.EXPECT(). + UploadTraces(gomock.Any(), gomock.Any()). + Return(nil). + After(start) + mockClient.EXPECT(). + Stop(gomock.Any()). + Return(nil). + After(upload) + sc := trace.NewSyncClient(mockClient) + assert.NoError(t, sc.Start(context.Background())) + assert.NoError(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{})) + assert.NoError(t, sc.Stop(context.Background())) + }) + t.Run("Update", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockClient1 := mock_otlptrace.NewMockClient(ctrl) + mockClient2 := mock_otlptrace.NewMockClient(ctrl) + + start1 := mockClient1.EXPECT(). + Start(gomock.Any()). + Return(nil) + upload1 := mockClient1.EXPECT(). + UploadTraces(gomock.Any(), gomock.Any()). + Return(nil). + After(start1) + start2 := mockClient2.EXPECT(). + Start(gomock.Any()). + Return(nil). + After(upload1) + stop1 := mockClient1.EXPECT(). + Stop(gomock.Any()). + Return(nil). + After(start2) + upload2 := mockClient2.EXPECT(). + UploadTraces(gomock.Any(), gomock.Any()). + Return(nil). + After(stop1) + mockClient2.EXPECT(). + Stop(gomock.Any()). + Return(nil). + After(upload2) + sc := trace.NewSyncClient(mockClient1) + assert.NoError(t, sc.Start(context.Background())) + assert.NoError(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{})) + assert.NoError(t, sc.Update(context.Background(), mockClient2)) + assert.NoError(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{})) + assert.NoError(t, sc.Stop(context.Background())) + }) + + spinWait := func(counter *atomic.Int32, until int32) error { + startTime := time.Now() + for counter.Load() != until { + if time.Since(startTime) > 1*time.Second { + return fmt.Errorf("timed out waiting for counter to equal %d", until) + } + } + return nil + } + + t.Run("Concurrent UploadTraces", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockClient1 := mock_otlptrace.NewMockClient(ctrl) + count := atomic.Int32{} + unlock := make(chan struct{}) + concurrency := min(runtime.NumCPU(), 4) + mockClient1.EXPECT(). + UploadTraces(gomock.Any(), gomock.Any()). + DoAndReturn(func(context.Context, []*tracev1.ResourceSpans) error { + count.Add(1) + defer count.Add(-1) + <-unlock + return nil + }). + Times(concurrency) + sc := trace.NewSyncClient(mockClient1) + start := make(chan struct{}) + for range concurrency { + go func() { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + <-start + require.NoError(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{})) + }() + } + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + close(start) + assert.NoError(t, spinWait(&count, int32(concurrency))) + }) + + t.Run("Concurrent Update/UploadTraces", func(t *testing.T) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + ctrl := gomock.NewController(t) + mockClient1 := mock_otlptrace.NewMockClient(ctrl) + mockClient2 := mock_otlptrace.NewMockClient(ctrl) + uploadTracesCount1 := atomic.Int32{} + uploadTracesCount2 := atomic.Int32{} + unlock1 := make(chan struct{}) + unlock2 := make(chan struct{}) + waitForStop := make(chan struct{}) + concurrency := min(runtime.NumCPU(), 4) + + // start 1 -> upload 1 -> start 2 -> stop 1 -> upload 2 -> stop 2 + fStart1 := mockClient1.EXPECT(). + Start(gomock.Any()). + Return(nil) + fUpload1 := mockClient1.EXPECT(). + UploadTraces(gomock.Any(), gomock.Any()). + DoAndReturn(func(context.Context, []*tracev1.ResourceSpans) error { + // called from non-test threads + uploadTracesCount1.Add(1) + defer uploadTracesCount1.Add(-1) + <-unlock1 + return nil + }). + Times(concurrency). + After(fStart1) + fStart2 := mockClient2.EXPECT(). + Start(gomock.Any()). + Return(nil). + After(fUpload1) + fStop1 := mockClient1.EXPECT(). + Stop(gomock.Any()). + DoAndReturn(func(context.Context) error { + // called from test thread + close(unlock1) + assert.NoError(t, spinWait(&uploadTracesCount1, 0)) + return nil + }). + After(fStart2) + fUpload2 := mockClient2.EXPECT(). + UploadTraces(gomock.Any(), gomock.Any()). + DoAndReturn(func(context.Context, []*tracev1.ResourceSpans) error { + // called from non-test threads + uploadTracesCount2.Add(1) + defer uploadTracesCount2.Add(-1) + <-unlock2 + return nil + }). + Times(concurrency). + After(fStop1) + mockClient2.EXPECT(). + Stop(gomock.Any()). + DoAndReturn(func(context.Context) error { + // called from test thread + close(unlock2) + assert.NoError(t, spinWait(&uploadTracesCount2, 0)) + close(waitForStop) + // no way around sleeping here - we have to give the other threads time + // to call UploadTraces and block waiting on waitForNewClient to be + // closed, which happens after this function returns + time.Sleep(10 * time.Millisecond) + return nil + }). + After(fUpload2) + sc := trace.NewSyncClient(mockClient1) + require.NoError(t, sc.Start(context.Background())) + + for range concurrency { + go func() { + require.NoError(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{})) + }() + } + require.NoError(t, spinWait(&uploadTracesCount1, int32(concurrency))) + // at this point, all calls to UploadTraces for client1 are blocked + + for range concurrency { + go func() { + <-unlock1 // wait for client1.Stop + // after this, calls to UploadTraces will block waiting for the + // new client, instead of using the old one we're about to close + require.NoError(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{})) + }() + } + require.NoError(t, sc.Update(context.Background(), mockClient2)) + require.NoError(t, spinWait(&uploadTracesCount2, int32(concurrency))) + // at this point, all calls to UploadTraces for client2 are blocked. + + // while SyncClient is waiting for the underlying client to stop during + // sc.Stop(), *new* calls to sc.UploadTraces will wait for it to stop, then + // error with trace.ErrClientStopped, but the previous calls blocked in + // client2 will complete without error. + for range concurrency { + go func() { + <-waitForStop + assert.ErrorIs(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{}), trace.ErrClientStopped) + }() + } + assert.NoError(t, sc.Stop(context.Background())) + + // sanity checks + assert.ErrorIs(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{}), trace.ErrNoClient) + assert.ErrorIs(t, sc.Start(context.Background()), trace.ErrNoClient) + assert.ErrorIs(t, sc.Stop(context.Background()), trace.ErrNoClient) + assert.NoError(t, sc.Update(context.Background(), nil)) + }) +} + +type errHandler struct { + err error +} + +var _ otel.ErrorHandler = (*errHandler)(nil) + +func (h *errHandler) Handle(err error) { + h.err = err +} + +func TestNewRemoteClientFromEnv(t *testing.T) { + env := testenv.New(t, testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags)) + + receiver := scenarios.NewOTLPTraceReceiver() + env.Add(receiver) + + grpcEndpoint := receiver.GRPCEndpointURL() + httpEndpoint := receiver.HTTPEndpointURL() + + env.Start() + snippets.WaitStartupComplete(env) + + for _, tc := range []struct { + name string + env map[string]string + newClientErr string + uploadErr string + expectNoSpans bool + }{ + { + name: "GRPC endpoint, auto protocol", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": grpcEndpoint.Value(), + }, + }, + { + name: "GRPC endpoint, alternate env, auto protocol", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_ENDPOINT": grpcEndpoint.Value(), + }, + uploadErr: "net/http: HTTP/1.x transport connection broken", + }, + { + name: "HTTP endpoint, auto protocol", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": httpEndpoint.Value(), + }, + }, + { + name: "HTTP endpoint, alternate env, auto protocol", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_ENDPOINT": strings.TrimSuffix(httpEndpoint.Value(), "/v1/traces"), // path is added automatically by the sdk here + }, + }, + { + name: "GRPC endpoint, explicit protocol", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": grpcEndpoint.Value(), + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "grpc", + }, + }, + { + name: "HTTP endpoint, explicit protocol", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": httpEndpoint.Value(), + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "http/protobuf", + }, + }, + { + name: "no exporter", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": httpEndpoint.Value(), + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "http/protobuf", + }, + expectNoSpans: true, + }, + { + name: "no exporter", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "noop", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": httpEndpoint.Value(), + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "http/protobuf", + }, + expectNoSpans: true, + }, + { + name: "no exporter", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "none", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": httpEndpoint.Value(), + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "http/protobuf", + }, + expectNoSpans: true, + }, + { + name: "invalid exporter", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "invalid", + }, + newClientErr: `unknown otlp trace exporter "invalid", expected "otlp" or "none"`, + }, + { + name: "invalid protocol", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": grpcEndpoint.Value(), + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "invalid", + }, + newClientErr: `unknown otlp trace exporter protocol "invalid", expected "grpc" or "http/protobuf"`, + }, + { + name: "valid configuration, but sdk disabled", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": grpcEndpoint.Value(), + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "grpc", + "OTEL_SDK_DISABLED": "true", + }, + expectNoSpans: true, + }, + { + name: "valid configuration, wrong value for sdk disabled env", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": grpcEndpoint.Value(), + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "grpc", + "OTEL_SDK_DISABLED": "1", // only "true" works according to the spec + }, + }, + { + name: "endpoint variable precedence", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_ENDPOINT": "invalid", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": grpcEndpoint.Value(), // should take precedence + "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc", + }, + }, + { + name: "protocol variable precedence", + env: map[string]string{ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_PROTOCOL": "invalid", + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "grpc", // should take precedence + "OTEL_EXPORTER_OTLP_ENDPOINT": grpcEndpoint.Value(), + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.env { + t.Setenv(k, v) + } + handler := &errHandler{} + oldErrHandler := otel.GetErrorHandler() + otel.SetErrorHandler(handler) + t.Cleanup(func() { otel.SetErrorHandler(oldErrHandler) }) + + if tc.uploadErr != "" { + recorder := env.NewLogRecorder(testenv.WithSkipCloseDelay()) + defer func() { + recorder.Match([]map[string]any{ + { + "level": "error", + "error": regexp.MustCompile(`^Post "[^"]+": net/http: HTTP/1.x transport connection broken: malformed HTTP response.*$`), + "message": "error uploading traces", + }, + }) + }() + } + + remoteClient := trace.NewRemoteClientFromEnv() + ctx := trace.Options{ + RemoteClient: remoteClient, + }.NewContext(log.Ctx(env.Context()).WithContext(context.Background())) + + if tc.newClientErr != "" { + assert.ErrorContains(t, handler.err, tc.newClientErr) + return + } + + tp := trace.NewTracerProvider(ctx, t.Name()) + + _, span := tp.Tracer(trace.PomeriumCoreTracer).Start(ctx, "test span") + span.End() + + assert.NoError(t, trace.ForceFlush(ctx)) + assert.NoError(t, trace.ShutdownContext(ctx)) + + if tc.uploadErr != "" { + return + } + + results := testutil.NewTraceResults(receiver.FlushResourceSpans()) + if tc.expectNoSpans { + results.MatchTraces(t, testutil.MatchOptions{Exact: true}) + } else { + results.MatchTraces(t, testutil.MatchOptions{ + Exact: true, + }, testutil.Match{Name: t.Name() + ": test span", TraceCount: 1, Services: []string{t.Name()}}) + } + }) + } +} + +func TestBestEffortProtocolFromOTLPEndpoint(t *testing.T) { + t.Run("Well-known port numbers", func(t *testing.T) { + assert.Equal(t, "grpc", trace.BestEffortProtocolFromOTLPEndpoint("http://127.0.0.1:4317", true)) + assert.Equal(t, "http/protobuf", trace.BestEffortProtocolFromOTLPEndpoint("http://127.0.0.1:4318", true)) + }) + t.Run("path presence", func(t *testing.T) { + assert.Equal(t, "http/protobuf", trace.BestEffortProtocolFromOTLPEndpoint("http://127.0.0.1:12345", false)) + assert.Equal(t, "grpc", trace.BestEffortProtocolFromOTLPEndpoint("http://127.0.0.1:12345", true)) + assert.Equal(t, "grpc", trace.BestEffortProtocolFromOTLPEndpoint("http://127.0.0.1:12345/v1/traces", false)) + assert.Equal(t, "http/protobuf", trace.BestEffortProtocolFromOTLPEndpoint("http://127.0.0.1:12345/v1/traces", true)) + }) + t.Run("invalid inputs", func(t *testing.T) { + assert.Equal(t, "", trace.BestEffortProtocolFromOTLPEndpoint("", false)) + assert.Equal(t, "", trace.BestEffortProtocolFromOTLPEndpoint("http://\x7f", false)) + }) +} diff --git a/internal/telemetry/trace/datadog.go b/internal/telemetry/trace/datadog.go deleted file mode 100644 index 10b346ca1..000000000 --- a/internal/telemetry/trace/datadog.go +++ /dev/null @@ -1,34 +0,0 @@ -package trace - -import ( - datadog "github.com/DataDog/opencensus-go-exporter-datadog" - octrace "go.opencensus.io/trace" -) - -type datadogProvider struct { - exporter *datadog.Exporter -} - -func (provider *datadogProvider) Register(opts *TracingOptions) error { - dOpts := datadog.Options{ - Service: opts.Service, - TraceAddr: opts.DatadogAddress, - } - dex, err := datadog.NewExporter(dOpts) - if err != nil { - return err - } - octrace.RegisterExporter(dex) - provider.exporter = dex - return nil -} - -func (provider *datadogProvider) Unregister() error { - if provider.exporter == nil { - return nil - } - octrace.UnregisterExporter(provider.exporter) - provider.exporter.Stop() - provider.exporter = nil - return nil -} diff --git a/internal/telemetry/trace/debug.go b/internal/telemetry/trace/debug.go new file mode 100644 index 000000000..1809ebe40 --- /dev/null +++ b/internal/telemetry/trace/debug.go @@ -0,0 +1,119 @@ +package trace + +import ( + "context" + "fmt" + "io" + "os" + "runtime" + "strings" + + "go.opentelemetry.io/otel/attribute" + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +type DebugFlags uint32 + +const ( + // If set, adds the "caller" attribute to each trace with the source location + // where the trace was started. + TrackSpanCallers = (1 << iota) + + // If set, keeps track of all span references and will attempt to wait for + // all traces to complete when shutting down a trace context. + // Use with caution, this will cause increasing memory usage over time. + TrackSpanReferences = (1 << iota) + + // If set, keeps track of all observed spans, including span context and + // all attributes. + // Use with caution, this will cause significantly increasing memory usage + // over time. + TrackAllSpans = (1 << iota) | TrackSpanCallers + + // If set, will log all trace ID mappings on close. + LogTraceIDMappings = (1 << iota) + + // If set, will log all spans observed by the exporter on close. These spans + // may belong to incomplete traces. + // + // Enables [TrackAllSpans] + LogAllSpans = (1 << iota) | TrackAllSpans + + // If set, will log the raw json payloads and timestamps of export requests + // on close. + // Use with caution, this will cause significantly increasing memory usage + // over time. + LogAllEvents = (1 << iota) + + // If set, will log all exported spans when a warning is issued on close + // (requires warning flags to also be set) + // + // Enables [TrackAllSpans] + LogAllSpansOnWarn = (1 << iota) | TrackAllSpans + + // If set, will log all trace ID mappings when a warning is issued on close. + // (requires warning flags to also be set) + LogTraceIDMappingsOnWarn = (1 << iota) + + // If set, will print a warning to stderr on close if there are any incomplete + // traces (traces with no observed root spans) + WarnOnIncompleteTraces = (1 << iota) + + // If set, will print a warning to stderr on close if there are any incomplete + // spans (spans started, but not ended) + WarnOnIncompleteSpans = (1 << iota) + + // If set, will print a warning to stderr on close if there are any spans + // which reference unknown parent spans. + // + // Enables [TrackSpanReferences] + WarnOnUnresolvedReferences = (1 << iota) | TrackSpanReferences + + // If set, configures Envoy to flush every span individually, disabling its + // internal buffer. + EnvoyFlushEverySpan = (1 << iota) +) + +func (df DebugFlags) Check(flags DebugFlags) bool { + return (df & flags) == flags +} + +type stackTraceProcessor struct{} + +// ForceFlush implements trace.SpanProcessor. +func (s *stackTraceProcessor) ForceFlush(context.Context) error { + return nil +} + +// OnEnd implements trace.SpanProcessor. +func (*stackTraceProcessor) OnEnd(sdktrace.ReadOnlySpan) { +} + +// OnStart implements trace.SpanProcessor. +func (*stackTraceProcessor) OnStart(_ context.Context, s sdktrace.ReadWriteSpan) { + _, file, line, _ := runtime.Caller(2) + s.SetAttributes(attribute.String("caller", fmt.Sprintf("%s:%d", file, line))) +} + +// Shutdown implements trace.SpanProcessor. +func (s *stackTraceProcessor) Shutdown(context.Context) error { + return nil +} + +var debugMessageWriter io.Writer + +func startMsg(title string) *strings.Builder { + msg := &strings.Builder{} + msg.WriteString("\n==================================================\n") + msg.WriteString(title) + return msg +} + +func endMsg(msg *strings.Builder) { + msg.WriteString("==================================================\n") + w := debugMessageWriter + if w == nil { + w = os.Stderr + } + fmt.Fprint(w, msg.String()) +} diff --git a/internal/telemetry/trace/doc.go b/internal/telemetry/trace/doc.go deleted file mode 100644 index c1da792c4..000000000 --- a/internal/telemetry/trace/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package trace contains support for OpenCensus distributed tracing. -package trace diff --git a/internal/telemetry/trace/global.go b/internal/telemetry/trace/global.go new file mode 100644 index 000000000..16b7a5a4f --- /dev/null +++ b/internal/telemetry/trace/global.go @@ -0,0 +1,46 @@ +package trace + +import ( + "context" + + "go.opentelemetry.io/contrib/propagators/autoprop" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/embedded" +) + +// PomeriumCoreTracer should be used for all tracers created in pomerium core. +const PomeriumCoreTracer = "pomerium.io/core" + +func init() { + otel.SetTextMapPropagator(autoprop.NewTextMapPropagator()) +} + +// UseGlobalPanicTracer sets the global tracer provider to one whose tracers +// panic when starting spans. This can be used to locate errant usages of the +// global tracer, and is enabled automatically in some tests. It is otherwise +// not used by default, since pomerium is used as a library in some places that +// might use the global tracer provider. +func UseGlobalPanicTracer() { + otel.SetTracerProvider(panicTracerProvider{}) +} + +type panicTracerProvider struct { + embedded.TracerProvider +} + +// Tracer implements trace.TracerProvider. +func (w panicTracerProvider) Tracer(string, ...trace.TracerOption) trace.Tracer { + return panicTracer{} +} + +type panicTracer struct { + embedded.Tracer +} + +var _ trace.Tracer = panicTracer{} + +// Start implements trace.Tracer. +func (p panicTracer) Start(context.Context, string, ...trace.SpanStartOption) (context.Context, trace.Span) { + panic("global tracer used") +} diff --git a/internal/telemetry/trace/global_test.go b/internal/telemetry/trace/global_test.go new file mode 100644 index 000000000..852d2c82a --- /dev/null +++ b/internal/telemetry/trace/global_test.go @@ -0,0 +1,22 @@ +package trace_test + +import ( + "context" + "testing" + + "github.com/pomerium/pomerium/internal/telemetry/trace" + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace/noop" +) + +func TestUseGlobalPanicTracer(t *testing.T) { + t.Cleanup(func() { + otel.SetTracerProvider(noop.NewTracerProvider()) + }) + trace.UseGlobalPanicTracer() + tracer := otel.GetTracerProvider().Tracer("test") + assert.Panics(t, func() { + tracer.Start(context.Background(), "span") + }) +} diff --git a/internal/telemetry/trace/jaeger.go b/internal/telemetry/trace/jaeger.go deleted file mode 100644 index 23502ef24..000000000 --- a/internal/telemetry/trace/jaeger.go +++ /dev/null @@ -1,37 +0,0 @@ -package trace - -import ( - "contrib.go.opencensus.io/exporter/jaeger" - octrace "go.opencensus.io/trace" -) - -type jaegerProvider struct { - exporter *jaeger.Exporter -} - -func (provider *jaegerProvider) Register(opts *TracingOptions) error { - jOpts := jaeger.Options{ - ServiceName: opts.Service, - AgentEndpoint: opts.JaegerAgentEndpoint, - } - if opts.JaegerCollectorEndpoint != nil { - jOpts.CollectorEndpoint = opts.JaegerCollectorEndpoint.String() - } - jex, err := jaeger.NewExporter(jOpts) - if err != nil { - return err - } - octrace.RegisterExporter(jex) - provider.exporter = jex - return nil -} - -func (provider *jaegerProvider) Unregister() error { - if provider.exporter == nil { - return nil - } - octrace.UnregisterExporter(provider.exporter) - provider.exporter.Flush() - provider.exporter = nil - return nil -} diff --git a/internal/telemetry/trace/main_test.go b/internal/telemetry/trace/main_test.go new file mode 100644 index 000000000..b546c8e45 --- /dev/null +++ b/internal/telemetry/trace/main_test.go @@ -0,0 +1,13 @@ +package trace_test + +import ( + "os" + "testing" + + "github.com/pomerium/pomerium/internal/telemetry/trace" +) + +func TestMain(m *testing.M) { + trace.UseGlobalPanicTracer() + os.Exit(m.Run()) +} diff --git a/internal/telemetry/trace/middleware.go b/internal/telemetry/trace/middleware.go new file mode 100644 index 000000000..f80d04d1b --- /dev/null +++ b/internal/telemetry/trace/middleware.go @@ -0,0 +1,162 @@ +package trace + +import ( + "context" + "fmt" + "net/http" + "reflect" + + "github.com/gorilla/mux" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/stats" +) + +func NewHTTPMiddleware(opts ...otelhttp.Option) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + routeStr := "" + route := mux.CurrentRoute(r) + if route != nil { + var err error + routeStr, err = route.GetPathTemplate() + if err != nil { + routeStr, err = route.GetPathRegexp() + if err != nil { + routeStr = "" + } + } + } + traceparent := r.Header.Get("Traceparent") + if traceparent != "" { + xPomeriumTraceparent := r.Header.Get("X-Pomerium-Traceparent") + if xPomeriumTraceparent != "" { + sc, err := ParseTraceparent(xPomeriumTraceparent) + if err == nil { + r.Header.Set("Traceparent", WithTraceFromSpanContext(traceparent, sc)) + ctx := otel.GetTextMapPropagator().Extract(r.Context(), propagation.HeaderCarrier(r.Header)) + r = r.WithContext(ctx) + } + } + } + otelhttp.NewHandler(next, fmt.Sprintf("Server: %s %s", r.Method, routeStr), opts...).ServeHTTP(w, r) + }) + } +} + +func NewServerStatsHandler(base stats.Handler) stats.Handler { + return &serverStatsHandlerWrapper{ + base: base, + } +} + +type serverStatsHandlerWrapper struct { + base stats.Handler +} + +func (w *serverStatsHandlerWrapper) wrapContext(ctx context.Context) context.Context { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return ctx + } + traceparent := md.Get("traceparent") + xPomeriumTraceparent := md.Get("x-pomerium-traceparent") + if len(traceparent) > 0 && traceparent[0] != "" && len(xPomeriumTraceparent) > 0 && xPomeriumTraceparent[0] != "" { + newTracectx, err := ParseTraceparent(xPomeriumTraceparent[0]) + if err != nil { + return ctx + } + + md.Set("traceparent", WithTraceFromSpanContext(traceparent[0], newTracectx)) + return metadata.NewIncomingContext(ctx, md) + } + return ctx +} + +// HandleConn implements stats.Handler. +func (w *serverStatsHandlerWrapper) HandleConn(ctx context.Context, stats stats.ConnStats) { + w.base.HandleConn(w.wrapContext(ctx), stats) +} + +// HandleRPC implements stats.Handler. +func (w *serverStatsHandlerWrapper) HandleRPC(ctx context.Context, stats stats.RPCStats) { + w.base.HandleRPC(w.wrapContext(ctx), stats) +} + +// TagConn implements stats.Handler. +func (w *serverStatsHandlerWrapper) TagConn(ctx context.Context, info *stats.ConnTagInfo) context.Context { + return w.base.TagConn(w.wrapContext(ctx), info) +} + +// TagRPC implements stats.Handler. +func (w *serverStatsHandlerWrapper) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context { + return w.base.TagRPC(w.wrapContext(ctx), info) +} + +type clientStatsHandlerWrapper struct { + ClientStatsHandlerOptions + base stats.Handler +} + +type ClientStatsHandlerOptions struct { + statsInterceptor func(ctx context.Context, rs stats.RPCStats) stats.RPCStats +} + +type ClientStatsHandlerOption func(*ClientStatsHandlerOptions) + +func (o *ClientStatsHandlerOptions) apply(opts ...ClientStatsHandlerOption) { + for _, op := range opts { + op(o) + } +} + +// WithStatsInterceptor calls the given function to modify the rpc stats before +// passing it to the stats handler during HandleRPC events. +// +// The interceptor MUST NOT modify the RPCStats it is given. It should instead +// return a copy of the underlying object with the same type, with any +// modifications made to the copy. +func WithStatsInterceptor(statsInterceptor func(ctx context.Context, rs stats.RPCStats) stats.RPCStats) ClientStatsHandlerOption { + return func(o *ClientStatsHandlerOptions) { + o.statsInterceptor = statsInterceptor + } +} + +func NewClientStatsHandler(base stats.Handler, opts ...ClientStatsHandlerOption) stats.Handler { + options := ClientStatsHandlerOptions{} + options.apply(opts...) + return &clientStatsHandlerWrapper{ + ClientStatsHandlerOptions: options, + base: base, + } +} + +// HandleConn implements stats.Handler. +func (w *clientStatsHandlerWrapper) HandleConn(ctx context.Context, stats stats.ConnStats) { + w.base.HandleConn(ctx, stats) +} + +// HandleRPC implements stats.Handler. +func (w *clientStatsHandlerWrapper) HandleRPC(ctx context.Context, stats stats.RPCStats) { + if w.statsInterceptor != nil { + modified := w.statsInterceptor(ctx, stats) + if reflect.TypeOf(stats) != reflect.TypeOf(modified) { + panic("bug: stats interceptor returned a message of a different type") + } + w.base.HandleRPC(ctx, modified) + } else { + w.base.HandleRPC(ctx, stats) + } +} + +// TagConn implements stats.Handler. +func (w *clientStatsHandlerWrapper) TagConn(ctx context.Context, info *stats.ConnTagInfo) context.Context { + return w.base.TagConn(ctx, info) +} + +// TagRPC implements stats.Handler. +func (w *clientStatsHandlerWrapper) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context { + return w.base.TagRPC(ctx, info) +} diff --git a/internal/telemetry/trace/middleware_test.go b/internal/telemetry/trace/middleware_test.go new file mode 100644 index 000000000..47fc6b369 --- /dev/null +++ b/internal/telemetry/trace/middleware_test.go @@ -0,0 +1,336 @@ +package trace_test + +import ( + "context" + "errors" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/pomerium/pomerium/internal/telemetry/trace" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + oteltrace "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/test/bufconn" +) + +var cases = []struct { + name string + setTraceparent string + setPomeriumTraceparent string + check func(t testing.TB, ctx context.Context) +}{ + { + name: "x-pomerium-traceparent not present", + setTraceparent: Traceparent(Trace(1), Span(1), true), + check: func(t testing.TB, ctx context.Context) { + span := oteltrace.SpanFromContext(ctx) + assert.Equal(t, Trace(1).ID().Value(), span.SpanContext().TraceID()) + assert.Equal(t, Span(1).ID(), span.SpanContext().SpanID()) + assert.True(t, span.SpanContext().IsSampled()) + }, + }, + { + name: "x-pomerium-traceparent present", + setTraceparent: Traceparent(Trace(2), Span(2), true), + setPomeriumTraceparent: Traceparent(Trace(1), Span(1), true), + check: func(t testing.TB, ctx context.Context) { + span := oteltrace.SpanFromContext(ctx) + assert.Equal(t, Trace(1).ID().Value(), span.SpanContext().TraceID()) + assert.Equal(t, Span(2).ID(), span.SpanContext().SpanID()) + assert.True(t, span.SpanContext().IsSampled()) + }, + }, + { + name: "x-pomerium-traceparent present, force sampling off", + setTraceparent: Traceparent(Trace(2), Span(2), true), + setPomeriumTraceparent: Traceparent(Trace(1), Span(1), false), + check: func(t testing.TB, ctx context.Context) { + span := oteltrace.SpanFromContext(ctx) + assert.Equal(t, Trace(1).ID().Value(), span.SpanContext().TraceID()) + assert.Equal(t, Span(2).ID(), span.SpanContext().SpanID()) + assert.Equal(t, false, span.SpanContext().IsSampled()) + }, + }, + { + name: "x-pomerium-traceparent present, force sampling on", + setTraceparent: Traceparent(Trace(2), Span(2), false), + setPomeriumTraceparent: Traceparent(Trace(1), Span(1), true), + check: func(t testing.TB, ctx context.Context) { + span := oteltrace.SpanFromContext(ctx) + assert.Equal(t, Trace(1).ID().Value(), span.SpanContext().TraceID()) + assert.Equal(t, Span(2).ID(), span.SpanContext().SpanID()) + assert.Equal(t, true, span.SpanContext().IsSampled()) + }, + }, + { + name: "malformed x-pomerium-traceparent", + setTraceparent: Traceparent(Trace(2), Span(2), false), + setPomeriumTraceparent: "00-xxxxxx-yyyyyy-03", + check: func(t testing.TB, ctx context.Context) { + span := oteltrace.SpanFromContext(ctx) + assert.Equal(t, Trace(2).ID().Value(), span.SpanContext().TraceID()) + assert.Equal(t, Span(2).ID(), span.SpanContext().SpanID()) + assert.Equal(t, false, span.SpanContext().IsSampled()) + }, + }, +} + +func TestHTTPMiddleware(t *testing.T) { + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := httptest.NewRequest(http.MethodGet, "/foo", nil) + if tc.setTraceparent != "" { + r.Header.Add("Traceparent", tc.setTraceparent) + } + if tc.setPomeriumTraceparent != "" { + r.Header.Add("X-Pomerium-Traceparent", tc.setPomeriumTraceparent) + } + w := httptest.NewRecorder() + trace.NewHTTPMiddleware( + otelhttp.WithTracerProvider(noop.NewTracerProvider()), + )(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + tc.check(t, r.Context()) + })).ServeHTTP(w, r) + }) + } +} + +func TestGRPCMiddleware(t *testing.T) { + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + srv := grpc.NewServer( + grpc.StatsHandler(trace.NewServerStatsHandler(otelgrpc.NewServerHandler( + otelgrpc.WithTracerProvider(noop.NewTracerProvider())))), + grpc.Creds(insecure.NewCredentials()), + ) + lis := bufconn.Listen(4096) + grpc_testing.RegisterTestServiceServer(srv, &testServer{ + fn: func(ctx context.Context) { + tc.check(t, ctx) + }, + }) + go srv.Serve(lis) + t.Cleanup(srv.Stop) + + client, err := grpc.NewClient("passthrough://ignore", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithStatsHandler(otelgrpc.NewClientHandler( + otelgrpc.WithTracerProvider(noop.NewTracerProvider()))), + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { + return lis.DialContext(ctx) + }), + ) + require.NoError(t, err) + + ctx := context.Background() + if tc.setTraceparent != "" { + ctx = metadata.AppendToOutgoingContext(ctx, + "traceparent", tc.setTraceparent, + ) + } + if tc.setPomeriumTraceparent != "" { + ctx = metadata.AppendToOutgoingContext(ctx, + "x-pomerium-traceparent", tc.setPomeriumTraceparent, + ) + } + _, err = grpc_testing.NewTestServiceClient(client).EmptyCall(ctx, &grpc_testing.Empty{}) + assert.NoError(t, err) + }) + } +} + +type testServer struct { + grpc_testing.UnimplementedTestServiceServer + fn func(ctx context.Context) +} + +func (ts *testServer) EmptyCall(ctx context.Context, _ *grpc_testing.Empty) (*grpc_testing.Empty, error) { + ts.fn(ctx) + return &grpc_testing.Empty{}, nil +} + +type mockHandler struct { + handleConn func(ctx context.Context, stats stats.ConnStats) + handleRPC func(ctx context.Context, stats stats.RPCStats) + tagConn func(ctx context.Context, info *stats.ConnTagInfo) context.Context + tagRPC func(ctx context.Context, info *stats.RPCTagInfo) context.Context +} + +// HandleConn implements stats.Handler. +func (m *mockHandler) HandleConn(ctx context.Context, stats stats.ConnStats) { + m.handleConn(ctx, stats) +} + +// HandleRPC implements stats.Handler. +func (m *mockHandler) HandleRPC(ctx context.Context, stats stats.RPCStats) { + m.handleRPC(ctx, stats) +} + +// TagConn implements stats.Handler. +func (m *mockHandler) TagConn(ctx context.Context, info *stats.ConnTagInfo) context.Context { + return m.tagConn(ctx, info) +} + +// TagRPC implements stats.Handler. +func (m *mockHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context { + return m.tagRPC(ctx, info) +} + +var _ stats.Handler = (*mockHandler)(nil) + +func TestStatsInterceptor(t *testing.T) { + var outBegin *stats.Begin + var outEnd *stats.End + base := &mockHandler{ + handleRPC: func(_ context.Context, rs stats.RPCStats) { + switch rs := rs.(type) { + case *stats.Begin: + outBegin = rs + case *stats.End: + outEnd = rs + } + }, + } + interceptor := func(_ context.Context, rs stats.RPCStats) stats.RPCStats { + switch rs := rs.(type) { + case *stats.Begin: + return &stats.Begin{ + Client: rs.Client, + BeginTime: rs.BeginTime.Add(-1 * time.Minute), + FailFast: rs.FailFast, + IsClientStream: rs.IsClientStream, + IsServerStream: rs.IsServerStream, + IsTransparentRetryAttempt: rs.IsTransparentRetryAttempt, + } + case *stats.End: + return &stats.End{ + Client: rs.Client, + BeginTime: rs.BeginTime, + EndTime: rs.EndTime, + Trailer: rs.Trailer, + Error: errors.New("modified"), + } + } + return rs + } + handler := trace.NewClientStatsHandler( + base, + trace.WithStatsInterceptor(interceptor), + ) + inBegin := &stats.Begin{ + Client: true, + BeginTime: time.Now(), + FailFast: true, + IsClientStream: true, + IsServerStream: false, + IsTransparentRetryAttempt: false, + } + handler.HandleRPC(context.Background(), inBegin) + assert.NotNil(t, outBegin) + assert.NotSame(t, inBegin, outBegin) + assert.Equal(t, inBegin.BeginTime.Add(-1*time.Minute), outBegin.BeginTime) + assert.Equal(t, inBegin.Client, outBegin.Client) + assert.Equal(t, inBegin.FailFast, outBegin.FailFast) + assert.Equal(t, inBegin.IsClientStream, outBegin.IsClientStream) + assert.Equal(t, inBegin.IsServerStream, outBegin.IsServerStream) + assert.Equal(t, inBegin.IsTransparentRetryAttempt, outBegin.IsTransparentRetryAttempt) + + inEnd := &stats.End{ + Client: true, + BeginTime: time.Now(), + EndTime: time.Now().Add(1 * time.Minute), + Trailer: metadata.Pairs("a", "b", "c", "d"), + Error: errors.New("input"), + } + handler.HandleRPC(context.Background(), inEnd) + assert.NotNil(t, outEnd) + assert.NotSame(t, inEnd, outEnd) + assert.Equal(t, inEnd.Client, outEnd.Client) + assert.Equal(t, inEnd.BeginTime, outEnd.BeginTime) + assert.Equal(t, inEnd.EndTime, outEnd.EndTime) + assert.Equal(t, inEnd.Trailer, outEnd.Trailer) + assert.Equal(t, "input", inEnd.Error.Error()) + assert.Equal(t, "modified", outEnd.Error.Error()) +} + +func TestStatsInterceptor_Nil(t *testing.T) { + var outCtx context.Context + var outConnStats stats.ConnStats + var outRPCStats stats.RPCStats + var outConnTagInfo *stats.ConnTagInfo + var outRPCTagInfo *stats.RPCTagInfo + base := &mockHandler{ + handleConn: func(ctx context.Context, stats stats.ConnStats) { + outCtx = ctx + outConnStats = stats + }, + handleRPC: func(ctx context.Context, stats stats.RPCStats) { + outCtx = ctx + outRPCStats = stats + }, + tagConn: func(ctx context.Context, info *stats.ConnTagInfo) context.Context { + outCtx = ctx + outConnTagInfo = info + return ctx + }, + tagRPC: func(ctx context.Context, info *stats.RPCTagInfo) context.Context { + outCtx = ctx + outRPCTagInfo = info + return ctx + }, + } + handler := trace.NewClientStatsHandler( + base, + trace.WithStatsInterceptor(nil), + ) + + inCtx := context.Background() + inConnStats := &stats.ConnBegin{} + inRPCStats := &stats.Begin{} + inConnTagInfo := &stats.ConnTagInfo{} + inRPCTagInfo := &stats.RPCTagInfo{} + + handler.HandleConn(inCtx, inConnStats) + assert.Equal(t, inCtx, outCtx) + assert.Same(t, inConnStats, outConnStats) + + handler.HandleRPC(inCtx, inRPCStats) + assert.Equal(t, inCtx, outCtx) + assert.Same(t, inRPCStats, outRPCStats) + + handler.TagConn(inCtx, inConnTagInfo) + assert.Equal(t, inCtx, outCtx) + assert.Same(t, inConnTagInfo, outConnTagInfo) + + handler.TagRPC(inCtx, inRPCTagInfo) + assert.Equal(t, inCtx, outCtx) + assert.Same(t, inRPCTagInfo, outRPCTagInfo) +} + +func TestStatsInterceptor_Bug(t *testing.T) { + handler := trace.NewClientStatsHandler( + &mockHandler{ + handleRPC: func(_ context.Context, _ stats.RPCStats) { + t.Error("should not be reached") + }, + }, + trace.WithStatsInterceptor(func(_ context.Context, rs stats.RPCStats) stats.RPCStats { + _ = rs.(*stats.Begin) + return &stats.End{} + }), + ) + assert.PanicsWithValue(t, "bug: stats interceptor returned a message of a different type", func() { + handler.HandleRPC(context.Background(), &stats.Begin{}) + }) +} diff --git a/internal/telemetry/trace/mock_otlptrace/mock_client.go b/internal/telemetry/trace/mock_otlptrace/mock_client.go new file mode 100644 index 000000000..d0f4c37be --- /dev/null +++ b/internal/telemetry/trace/mock_otlptrace/mock_client.go @@ -0,0 +1,84 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: go.opentelemetry.io/otel/exporters/otlp/otlptrace (interfaces: Client) +// +// Generated by this command: +// +// mockgen go.opentelemetry.io/otel/exporters/otlp/otlptrace Client +// + +// Package mock_otlptrace is a generated GoMock package. +package mock_otlptrace + +import ( + context "context" + reflect "reflect" + + v1 "go.opentelemetry.io/proto/otlp/trace/v1" + gomock "go.uber.org/mock/gomock" +) + +// MockClient is a mock of Client interface. +type MockClient struct { + ctrl *gomock.Controller + recorder *MockClientMockRecorder + isgomock struct{} +} + +// MockClientMockRecorder is the mock recorder for MockClient. +type MockClientMockRecorder struct { + mock *MockClient +} + +// NewMockClient creates a new mock instance. +func NewMockClient(ctrl *gomock.Controller) *MockClient { + mock := &MockClient{ctrl: ctrl} + mock.recorder = &MockClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockClient) EXPECT() *MockClientMockRecorder { + return m.recorder +} + +// Start mocks base method. +func (m *MockClient) Start(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Start", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Start indicates an expected call of Start. +func (mr *MockClientMockRecorder) Start(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockClient)(nil).Start), ctx) +} + +// Stop mocks base method. +func (m *MockClient) Stop(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Stop", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Stop indicates an expected call of Stop. +func (mr *MockClientMockRecorder) Stop(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*MockClient)(nil).Stop), ctx) +} + +// UploadTraces mocks base method. +func (m *MockClient) UploadTraces(ctx context.Context, protoSpans []*v1.ResourceSpans) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UploadTraces", ctx, protoSpans) + ret0, _ := ret[0].(error) + return ret0 +} + +// UploadTraces indicates an expected call of UploadTraces. +func (mr *MockClientMockRecorder) UploadTraces(ctx, protoSpans any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UploadTraces", reflect.TypeOf((*MockClient)(nil).UploadTraces), ctx, protoSpans) +} diff --git a/internal/telemetry/trace/queue.go b/internal/telemetry/trace/queue.go new file mode 100644 index 000000000..9b3b47c93 --- /dev/null +++ b/internal/telemetry/trace/queue.go @@ -0,0 +1,817 @@ +package trace + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "slices" + "strconv" + "sync" + "sync/atomic" + "time" + "unique" + + lru "github.com/hashicorp/golang-lru/v2" + "github.com/pomerium/pomerium/internal/log" + "github.com/rs/zerolog" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + oteltrace "go.opentelemetry.io/otel/trace" + coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" +) + +var ( + maxPendingTraces atomic.Int32 + maxCachedTraceIDs atomic.Int32 +) + +func init() { + envOrDefault := func(envName string, def int32) int32 { + if val, ok := os.LookupEnv(envName); ok { + if num, err := strconv.ParseInt(val, 10, 32); err == nil { + return int32(num) + } + } + return def + } + maxPendingTraces.Store(envOrDefault("POMERIUM_OTEL_MAX_PENDING_TRACES", 8192)) + maxCachedTraceIDs.Store(envOrDefault("POMERIUM_OTEL_MAX_CACHED_TRACE_IDS", 16384)) +} + +func SetMaxPendingTraces(num int32) { + maxPendingTraces.Store(max(num, 0)) +} + +func SetMaxCachedTraceIDs(num int32) { + maxCachedTraceIDs.Store(max(num, 0)) +} + +type SpanExportQueue struct { + mu sync.Mutex + logger *zerolog.Logger + client otlptrace.Client + pendingResourcesByTraceID *lru.Cache[unique.Handle[oteltrace.TraceID], *Buffer] + knownTraceIDMappings *lru.Cache[unique.Handle[oteltrace.TraceID], unique.Handle[oteltrace.TraceID]] + uploadC chan []*tracev1.ResourceSpans + closing bool + closed chan struct{} + debugFlags DebugFlags + debugAllEnqueuedSpans map[oteltrace.SpanID]*tracev1.Span + tracker *spanTracker + observer *spanObserver + debugEvents []DebugEvent +} + +func NewSpanExportQueue(ctx context.Context, client otlptrace.Client) *SpanExportQueue { + debug := DebugFlagsFromContext(ctx) + var observer *spanObserver + if debug.Check(TrackSpanReferences) { + observer = newSpanObserver() + } + q := &SpanExportQueue{ + logger: log.Ctx(ctx), + client: client, + uploadC: make(chan []*tracev1.ResourceSpans, 64), + closed: make(chan struct{}), + debugFlags: debug, + debugAllEnqueuedSpans: make(map[oteltrace.SpanID]*tracev1.Span), + tracker: newSpanTracker(observer, debug), + observer: observer, + } + var err error + q.pendingResourcesByTraceID, err = lru.NewWithEvict(int(maxPendingTraces.Load()), q.onEvict) + if err != nil { + panic(err) + } + q.knownTraceIDMappings, err = lru.New[unique.Handle[oteltrace.TraceID], unique.Handle[oteltrace.TraceID]](int(maxCachedTraceIDs.Load())) + if err != nil { + panic(err) + } + go q.runUploader() + return q +} + +func (q *SpanExportQueue) runUploader() { + defer close(q.closed) + for resourceSpans := range q.uploadC { + ctx, ca := context.WithTimeout(context.Background(), 10*time.Second) + if err := q.client.UploadTraces(ctx, resourceSpans); err != nil { + q.logger.Err(err).Msg("error uploading traces") + } + ca() + } +} + +func (q *SpanExportQueue) onEvict(traceID unique.Handle[oteltrace.TraceID], buf *Buffer) { + if buf.IsEmpty() { + // if the buffer is not empty, it was evicted automatically + return + } else if mapping, ok := q.knownTraceIDMappings.Get(traceID); ok && mapping == zeroTraceID { + q.logger.Debug(). + Str("traceID", traceID.Value().String()). + Msg("dropping unsampled trace") + return + } + + select { + case q.uploadC <- buf.Flush(): + q.logger.Warn(). + Str("traceID", traceID.Value().String()). + Msg("trace export buffer is full, uploading oldest incomplete trace") + default: + q.logger.Warn(). + Str("traceID", traceID.Value().String()). + Msg("trace export buffer and upload queues are full, dropping trace") + } +} + +func (q *SpanExportQueue) insertPendingSpanLocked( + resource *ResourceInfo, + scope *ScopeInfo, + traceID unique.Handle[oteltrace.TraceID], + span *tracev1.Span, +) { + var pendingTraceResources *Buffer + + if ptr, ok := q.pendingResourcesByTraceID.Get(traceID); ok { + pendingTraceResources = ptr + } else { + pendingTraceResources = NewBuffer() + q.pendingResourcesByTraceID.Add(traceID, pendingTraceResources) + } + pendingTraceResources.Insert(resource, scope, span) +} + +func (q *SpanExportQueue) resolveTraceIDMappingLocked(out *Buffer, original, target unique.Handle[oteltrace.TraceID]) { + q.knownTraceIDMappings.Add(original, target) + + if target == zeroTraceID && original != zeroTraceID { + // mapping a trace id to zero indicates we should drop the trace + q.pendingResourcesByTraceID.Remove(original) + return + } + + if originalPending, ok := q.pendingResourcesByTraceID.Peek(original); ok { + if original == target { + out.Merge(originalPending) + } else { + // check if the target id is also pending + if targetPending, ok := q.pendingResourcesByTraceID.Peek(target); ok { + targetPending.MergeAs(originalPending, target) + } else { + out.MergeAs(originalPending, target) + } + } + q.pendingResourcesByTraceID.Remove(original) + } +} + +func (q *SpanExportQueue) getTraceIDMappingLocked(id unique.Handle[oteltrace.TraceID]) (unique.Handle[oteltrace.TraceID], bool) { + v, ok := q.knownTraceIDMappings.Get(id) + return v, ok +} + +func (q *SpanExportQueue) isKnownTracePendingLocked(id unique.Handle[oteltrace.TraceID]) bool { + _, ok := q.pendingResourcesByTraceID.Get(id) // will update the key's recent-ness in the lru + return ok +} + +var ErrShuttingDown = errors.New("exporter is shutting down") + +func (q *SpanExportQueue) Enqueue(ctx context.Context, req *coltracepb.ExportTraceServiceRequest) error { + q.mu.Lock() + defer q.mu.Unlock() + if q.closing { + return ErrShuttingDown + } + + if q.debugFlags.Check(LogAllEvents) { + q.debugEvents = append(q.debugEvents, DebugEvent{ + Timestamp: time.Now(), + Request: proto.Clone(req).(*coltracepb.ExportTraceServiceRequest), + }) + } + + // Spans are processed in two passes: + // 1. Look through each span to check if we have not yet seen its trace ID. + // If we haven't, and the span is a root span (no parent, or marked as such + // by us), mark the trace as observed, and (if indicated) keep track of the + // trace ID we need to rewrite it as, so that other spans we see later in + // this trace can also be rewritten the same way. + // If we find a new trace ID for which there are pending non-root spans, + // collect them and rewrite their trace IDs (if necessary), and prepare + // them to be uploaded. + // + // At this point, all trace IDs for the spans in the request are known. + // + // 2. Look through each span again, this time to filter out any spans in + // the request which belong to "pending" traces (known trace IDs for which + // we have not yet seen a root span), adding them to the list of pending + // spans for their corresponding trace IDs. They will be uploaded in the + // future once we have observed a root span for those traces, or if they + // are evicted by the queue. + + // Pass 1 + toUpload := NewBuffer() + for _, resource := range req.ResourceSpans { + for _, scope := range resource.ScopeSpans { + SPANS: + for _, span := range scope.Spans { + FormatSpanName(span) + spanID, ok := ToSpanID(span.SpanId) + if !ok { + continue + } + if q.debugFlags.Check(TrackAllSpans) { + q.debugAllEnqueuedSpans[spanID] = span + } + trackSpanReferences := q.debugFlags.Check(TrackSpanReferences) + parentSpanID, ok := ToSpanID(span.ParentSpanId) + if !ok { + continue + } + traceID, ok := ToTraceID(span.TraceId) + if !ok { + continue + } + if trackSpanReferences { + q.observer.Observe(spanID) + } + if mapping, ok := q.getTraceIDMappingLocked(traceID); ok { + if trackSpanReferences && mapping != zeroTraceID && parentSpanID.IsValid() { + q.observer.ObserveReference(parentSpanID, spanID) + } + } else { + // Observed a new trace ID. Check if the span is a root span + isRootSpan := !parentSpanID.IsValid() // no parent == root span + + // Assume the trace is sampled, because it was exported. span.Flags + // is an unreliable way to detect whether the span was sampled, + // because neither envoy nor opentelemetry-go encode the sampling + // decision there, assuming unsampled spans would not be exported + // (this was not taking into account tail-based sampling strategies) + // https://github.com/open-telemetry/opentelemetry-proto/issues/166 + isSampled := true + + mappedTraceID := traceID + for _, attr := range span.Attributes { + switch attr.Key { + case "pomerium.traceparent": + tp, err := ParseTraceparent(attr.GetValue().GetStringValue()) + if err != nil { + data, _ := protojson.Marshal(span) + log.Ctx(ctx). + Err(err). + Str("span", string(data)). + Msg("error processing span") + continue SPANS + } + mappedTraceID = unique.Make(tp.TraceID()) + // use the sampling decision from pomerium.traceparent instead + isSampled = tp.IsSampled() + case "pomerium.external-parent-span": + // This is a non-root span whose parent we do not expect to see + // here. For example, if a request originated externally from a + // system that is uploading its own spans out-of-band from us, + // we will never observe a root span for this trace and it would + // otherwise get stuck in the queue. + if !isRootSpan && q.debugFlags.Check(TrackSpanReferences) { + value, err := oteltrace.SpanIDFromHex(attr.GetValue().GetStringValue()) + if err != nil { + data, _ := protojson.Marshal(span) + log.Ctx(ctx). + Err(err). + Str("span", string(data)). + Msg("error processing span: invalid value for pomerium.external-parent-span") + } else { + q.observer.Observe(value) // mark this id as observed + } + } + isRootSpan = true + } + } + + if q.debugFlags.Check(TrackSpanReferences) { + if isSampled && parentSpanID.IsValid() { + q.observer.ObserveReference(parentSpanID, spanID) + } + } + + if !isSampled { + // We have observed a new trace that is not sampled (regardless of + // whether or not it is a root span). Resolve it using the zero + // trace ID to indicate that all spans for this trace should be + // dropped. + q.resolveTraceIDMappingLocked(toUpload, traceID, zeroTraceID) + } else if isRootSpan { + // We have observed a new trace that is sampled and is a root span. + // Resolve it using the mapped trace ID (if present), or its own + // trace ID (indicating it does not need to be rewritten). + // If the mapped trace is pending, this does not flush pending + // spans to the output buffer (toUpload), but instead merges them + // into the mapped trace's pending buffer. + q.resolveTraceIDMappingLocked(toUpload, traceID, mappedTraceID) + } + } + } + } + } + + // Pass 2 + for _, resource := range req.ResourceSpans { + resourceInfo := NewResourceInfo(resource.Resource, resource.SchemaUrl) + for _, scope := range resource.ScopeSpans { + scopeInfo := NewScopeInfo(scope.Scope, scope.SchemaUrl) + for _, span := range scope.Spans { + traceID, ok := ToTraceID(span.TraceId) + if !ok { + continue + } + + if mapping, hasMapping := q.getTraceIDMappingLocked(traceID); hasMapping { + if mapping == zeroTraceID { + continue // the trace has been dropped + } + id := mapping.Value() + copy(span.TraceId, id[:]) + // traceID = mapping + if q.isKnownTracePendingLocked(mapping) { + q.insertPendingSpanLocked(resourceInfo, scopeInfo, mapping, span) + } else { + toUpload.Insert(resourceInfo, scopeInfo, span) + } + } else { + q.insertPendingSpanLocked(resourceInfo, scopeInfo, traceID, span) + } + } + } + } + if resourceSpans := toUpload.Flush(); len(resourceSpans) > 0 { + q.uploadC <- resourceSpans + } + return nil +} + +var ( + ErrIncompleteTraces = errors.New("exporter shut down with incomplete traces") + ErrIncompleteSpans = errors.New("exporter shut down with incomplete spans") + ErrIncompleteUploads = errors.New("exporter shut down with pending trace uploads") + ErrMissingParentSpans = errors.New("exporter shut down with missing parent spans") +) + +func (q *SpanExportQueue) WaitForSpans(maxDuration time.Duration) error { + if !q.debugFlags.Check(TrackSpanReferences) { + return nil + } + done := make(chan struct{}) + go func() { + defer close(done) + q.observer.wait(q.debugAllEnqueuedSpans, 10*time.Second) + }() + select { + case <-done: + return nil + case <-time.After(maxDuration): + return ErrMissingParentSpans + } +} + +func (q *SpanExportQueue) Close(ctx context.Context) error { + q.mu.Lock() + q.closing = true + close(q.uploadC) + q.mu.Unlock() + select { + case <-ctx.Done(): + log.Ctx(ctx).Error().Msg("exporter stopped before all traces could be exported") + // drain uploadC + for range q.uploadC { + } + return context.Cause(ctx) + case <-q.closed: + q.mu.Lock() + defer q.mu.Unlock() + err := q.runOnCloseChecksLocked() + log.Ctx(ctx).Debug().Err(err).Msg("exporter stopped") + return err + } +} + +func (q *SpanExportQueue) runOnCloseChecksLocked() error { + didWarn := false + if q.debugFlags.Check(TrackSpanReferences) { + var unknownParentIDs []string + for id, via := range q.observer.referencedIDs { + if via.IsValid() { + if q.debugFlags.Check(TrackAllSpans) { + if viaSpan, ok := q.debugAllEnqueuedSpans[via]; ok { + unknownParentIDs = append(unknownParentIDs, fmt.Sprintf("%s via %s (%s)", id, via, viaSpan.Name)) + } else { + unknownParentIDs = append(unknownParentIDs, fmt.Sprintf("%s via %s", id, via)) + } + } + } + } + if len(unknownParentIDs) > 0 { + didWarn = true + msg := startMsg("WARNING: parent spans referenced but never seen:\n") + for _, str := range unknownParentIDs { + msg.WriteString(str) + msg.WriteString("\n") + } + endMsg(msg) + } + } + incomplete := q.pendingResourcesByTraceID.Len() > 0 + if incomplete && q.debugFlags.Check(WarnOnIncompleteTraces) { + didWarn = true + msg := startMsg("WARNING: exporter shut down with incomplete traces\n") + keys := q.pendingResourcesByTraceID.Keys() + values := q.pendingResourcesByTraceID.Values() + for i, k := range keys { + v := values[i] + fmt.Fprintf(msg, "- Trace: %s\n", k.Value()) + for _, pendingScope := range v.scopesByResourceID { + msg.WriteString(" - Resource:\n") + for _, v := range pendingScope.resource.Resource.Attributes { + fmt.Fprintf(msg, " %s=%s\n", v.Key, v.Value.String()) + } + for _, spanBuffer := range pendingScope.spansByScope { + if spanBuffer.scope != nil { + fmt.Fprintf(msg, " Scope: %s\n", spanBuffer.scope.ID()) + } else { + msg.WriteString(" Scope: (unknown)\n") + } + msg.WriteString(" Spans:\n") + longestName := 0 + for _, span := range spanBuffer.spans { + longestName = max(longestName, len(span.Name)+2) + } + for _, span := range spanBuffer.spans { + spanID, ok := ToSpanID(span.SpanId) + if !ok { + continue + } + traceID, ok := ToTraceID(span.TraceId) + if !ok { + continue + } + parentSpanID, ok := ToSpanID(span.ParentSpanId) + if !ok { + continue + } + _, seenParent := q.debugAllEnqueuedSpans[parentSpanID] + var missing string + if !seenParent { + missing = " [missing]" + } + fmt.Fprintf(msg, " - %-*s (trace: %s | span: %s | parent:%s %s)\n", longestName, + "'"+span.Name+"'", traceID.Value(), spanID, missing, parentSpanID) + for _, attr := range span.Attributes { + if attr.Key == "caller" { + fmt.Fprintf(msg, " => caller: '%s'\n", attr.Value.GetStringValue()) + break + } + } + } + } + } + } + endMsg(msg) + } + + if q.debugFlags.Check(LogTraceIDMappings) || (didWarn && q.debugFlags.Check(LogTraceIDMappingsOnWarn)) { + msg := startMsg("Known trace ids:\n") + keys := q.knownTraceIDMappings.Keys() + values := q.knownTraceIDMappings.Values() + for i, k := range keys { + v := values[i] + if k != v { + if v == zeroTraceID { + fmt.Fprintf(msg, "%s (dropped)\n", k.Value()) + } else { + fmt.Fprintf(msg, "%s => %s\n", k.Value(), v.Value()) + } + } else { + fmt.Fprintf(msg, "%s (no change)\n", k.Value()) + } + } + endMsg(msg) + } + if q.debugFlags.Check(LogAllSpans) || (didWarn && q.debugFlags.Check(LogAllSpansOnWarn)) { + msg := startMsg("All exported spans:\n") + longestName := 0 + for _, span := range q.debugAllEnqueuedSpans { + longestName = max(longestName, len(span.Name)+2) + } + for _, span := range q.debugAllEnqueuedSpans { + spanID, ok := ToSpanID(span.SpanId) + if !ok { + continue + } + traceID, ok := ToTraceID(span.TraceId) + if !ok { + continue + } + parentSpanID, ok := ToSpanID(span.ParentSpanId) + if !ok { + continue + } + fmt.Fprintf(msg, "%-*s (trace: %s | span: %s | parent: %s)", longestName, + "'"+span.Name+"'", traceID.Value(), spanID, parentSpanID) + var foundCaller bool + for _, attr := range span.Attributes { + if attr.Key == "caller" { + fmt.Fprintf(msg, " => %s\n", attr.Value.GetStringValue()) + foundCaller = true + break + } + } + if !foundCaller { + msg.WriteString("\n") + } + } + endMsg(msg) + } + if q.debugFlags.Check(LogAllEvents) { + msg := startMsg("All Events:\n") + msg.WriteByte('[') + for i, event := range q.debugEvents { + msg.WriteString("\n ") + eventData, _ := json.Marshal(event) + msg.Write(eventData) + if i < len(q.debugEvents)-1 { + msg.WriteByte(',') + } else { + msg.WriteString("\n]") + } + } + msg.WriteByte('\n') + endMsg(msg) + } + if incomplete { + return ErrIncompleteTraces + } + return nil +} + +type DebugEvent struct { + Timestamp time.Time `json:"timestamp"` + Request *coltracepb.ExportTraceServiceRequest `json:"request"` +} + +func (e DebugEvent) MarshalJSON() ([]byte, error) { + type debugEvent struct { + Timestamp time.Time `json:"timestamp"` + Request json.RawMessage `json:"request"` + } + reqData, _ := protojson.Marshal(e.Request) + return json.Marshal(debugEvent{ + Timestamp: e.Timestamp, + Request: reqData, + }) +} + +func (e *DebugEvent) UnmarshalJSON(b []byte) error { + type debugEvent struct { + Timestamp time.Time `json:"timestamp"` + Request json.RawMessage `json:"request"` + } + var ev debugEvent + if err := json.Unmarshal(b, &ev); err != nil { + return err + } + e.Timestamp = ev.Timestamp + var msg coltracepb.ExportTraceServiceRequest + if err := protojson.Unmarshal(ev.Request, &msg); err != nil { + return err + } + e.Request = &msg + return nil +} + +type spanTracker struct { + inflightSpans sync.Map + allSpans sync.Map + debugFlags DebugFlags + observer *spanObserver + shutdownOnce sync.Once +} + +func newSpanTracker(observer *spanObserver, debugFlags DebugFlags) *spanTracker { + return &spanTracker{ + observer: observer, + debugFlags: debugFlags, + } +} + +type spanInfo struct { + Name string + SpanContext oteltrace.SpanContext + Parent oteltrace.SpanContext + caller string + startTime time.Time +} + +// ForceFlush implements trace.SpanProcessor. +func (t *spanTracker) ForceFlush(context.Context) error { + return nil +} + +// OnEnd implements trace.SpanProcessor. +func (t *spanTracker) OnEnd(s sdktrace.ReadOnlySpan) { + id := s.SpanContext().SpanID() + t.inflightSpans.Delete(id) +} + +// OnStart implements trace.SpanProcessor. +func (t *spanTracker) OnStart(_ context.Context, s sdktrace.ReadWriteSpan) { + id := s.SpanContext().SpanID() + t.inflightSpans.Store(id, struct{}{}) + if t.debugFlags.Check(TrackSpanReferences) { + t.observer.Observe(id) + } + if t.debugFlags.Check(TrackAllSpans) { + var caller string + for _, attr := range s.Attributes() { + if attr.Key == "caller" { + caller = attr.Value.AsString() + break + } + } + t.allSpans.Store(id, &spanInfo{ + Name: s.Name(), + SpanContext: s.SpanContext(), + Parent: s.Parent(), + caller: caller, + startTime: s.StartTime(), + }) + } +} + +// Shutdown implements trace.SpanProcessor. +func (t *spanTracker) Shutdown(_ context.Context) error { + if t.debugFlags == 0 { + return nil + } + didWarn := false + t.shutdownOnce.Do(func() { + if t.debugFlags.Check(WarnOnIncompleteSpans) { + if t.debugFlags.Check(TrackAllSpans) { + incompleteSpans := []*spanInfo{} + t.inflightSpans.Range(func(key, _ any) bool { + if info, ok := t.allSpans.Load(key); ok { + incompleteSpans = append(incompleteSpans, info.(*spanInfo)) + } + return true + }) + if len(incompleteSpans) > 0 { + didWarn = true + msg := startMsg("WARNING: spans not ended:\n") + longestName := 0 + for _, span := range incompleteSpans { + longestName = max(longestName, len(span.Name)+2) + } + for _, span := range incompleteSpans { + var startedAt string + if span.caller != "" { + startedAt = " | started at: " + span.caller + } + fmt.Fprintf(msg, "%-*s (trace: %s | span: %s | parent: %s%s)\n", longestName, "'"+span.Name+"'", + span.SpanContext.TraceID(), span.SpanContext.SpanID(), span.Parent.SpanID(), startedAt) + } + endMsg(msg) + } + } else { + incompleteSpans := []oteltrace.SpanID{} + t.inflightSpans.Range(func(key, _ any) bool { + incompleteSpans = append(incompleteSpans, key.(oteltrace.SpanID)) + return true + }) + if len(incompleteSpans) > 0 { + didWarn = true + msg := startMsg("WARNING: spans not ended:\n") + for _, span := range incompleteSpans { + fmt.Fprintf(msg, "%s\n", span) + } + msg.WriteString("Note: set TrackAllSpans flag for more info\n") + endMsg(msg) + } + } + } + + if t.debugFlags.Check(LogAllSpans) || (t.debugFlags.Check(LogAllSpansOnWarn) && didWarn) { + allSpans := []*spanInfo{} + t.allSpans.Range(func(_, value any) bool { + allSpans = append(allSpans, value.(*spanInfo)) + return true + }) + slices.SortFunc(allSpans, func(a, b *spanInfo) int { + return a.startTime.Compare(b.startTime) + }) + msg := startMsg("All observed spans:\n") + longestName := 0 + for _, span := range allSpans { + longestName = max(longestName, len(span.Name)+2) + } + for _, span := range allSpans { + var startedAt string + if span.caller != "" { + startedAt = " | started at: " + span.caller + } + fmt.Fprintf(msg, "%-*s (trace: %s | span: %s | parent: %s%s)\n", longestName, "'"+span.Name+"'", + span.SpanContext.TraceID(), span.SpanContext.SpanID(), span.Parent.SpanID(), startedAt) + } + endMsg(msg) + } + }) + if didWarn { + return ErrIncompleteSpans + } + return nil +} + +func newSpanObserver() *spanObserver { + return &spanObserver{ + referencedIDs: map[oteltrace.SpanID]oteltrace.SpanID{}, + cond: sync.NewCond(&sync.Mutex{}), + } +} + +type spanObserver struct { + cond *sync.Cond + referencedIDs map[oteltrace.SpanID]oteltrace.SpanID + unobservedIDs int +} + +func (obs *spanObserver) ObserveReference(id oteltrace.SpanID, via oteltrace.SpanID) { + obs.cond.L.Lock() + defer obs.cond.L.Unlock() + if _, referenced := obs.referencedIDs[id]; !referenced { + obs.referencedIDs[id] = via // referenced, but not observed + // It is possible for new unobserved references to come in while waiting, + // but incrementing the counter wouldn't satisfy the condition so we don't + // need to signal the waiters + obs.unobservedIDs++ + } +} + +func (obs *spanObserver) Observe(id oteltrace.SpanID) { + obs.cond.L.Lock() + defer obs.cond.L.Unlock() + if observed, referenced := obs.referencedIDs[id]; !referenced || observed.IsValid() { // NB: subtle condition + obs.referencedIDs[id] = zeroSpanID + if referenced { + obs.unobservedIDs-- + obs.cond.Broadcast() + } + } +} + +func (obs *spanObserver) wait(debugAllEnqueuedSpans map[oteltrace.SpanID]*tracev1.Span, warnAfter time.Duration) { + done := make(chan struct{}) + defer close(done) + go func() { + select { + case <-done: + return + case <-time.After(warnAfter): + obs.debugWarnWaiting(debugAllEnqueuedSpans) + } + }() + + obs.cond.L.Lock() + for obs.unobservedIDs > 0 { + obs.cond.Wait() + } + obs.cond.L.Unlock() +} + +func (obs *spanObserver) debugWarnWaiting(debugAllEnqueuedSpans map[oteltrace.SpanID]*tracev1.Span) { + obs.cond.L.Lock() + msg := startMsg(fmt.Sprintf("Waiting on %d unobserved spans:\n", obs.unobservedIDs)) + for id, via := range obs.referencedIDs { + if via.IsValid() { + fmt.Fprintf(msg, "%s via %s", id, via) + if span := debugAllEnqueuedSpans[id]; span != nil { + createdAt := "(unknown)" + for _, attr := range span.Attributes { + if attr.Key == "caller" { + createdAt = attr.Value.GetStringValue() + break + } + } + fmt.Fprintf(msg, "'%s' (trace: %s | created: %s)\n", span.GetName(), span.TraceId, createdAt) + } else { + msg.WriteString("\n") + } + } + } + endMsg(msg) + obs.cond.L.Unlock() +} diff --git a/internal/telemetry/trace/queue_test.go b/internal/telemetry/trace/queue_test.go new file mode 100644 index 000000000..d5d98004b --- /dev/null +++ b/internal/telemetry/trace/queue_test.go @@ -0,0 +1,798 @@ +package trace_test + +import ( + "bytes" + "context" + "embed" + "fmt" + "io/fs" + "regexp" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/pomerium/pomerium/internal/telemetry/trace" + "github.com/pomerium/pomerium/internal/telemetry/trace/mock_otlptrace" + "github.com/pomerium/pomerium/internal/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + oteltrace "go.opentelemetry.io/otel/trace" + coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" + commonv1 "go.opentelemetry.io/proto/otlp/common/v1" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" + "go.uber.org/mock/gomock" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" +) + +//go:embed testdata +var testdata embed.FS + +func TestSpanExportQueue_Replay(t *testing.T) { + for _, tc := range []struct { + name string + file string + check func(t testing.TB, inputs, outputs *testutil.TraceResults) + }{ + { + name: "single trace", + file: "testdata/recording_01_single_trace.json", + check: func(t testing.TB, inputs, outputs *testutil.TraceResults) { + inputs.AssertEqual(t, outputs) + }, + }, + { + name: "rewriting multiple traces", + file: "testdata/recording_02_multi_trace.json", + check: func(t testing.TB, inputs, outputs *testutil.TraceResults) { + inputTraces := inputs.GetTraces().WithoutErrors() + outputTraces := outputs.GetTraces().WithoutErrors() + + // find upstream trace + var inputUpstreamTrace, outputUpstreamTrace *testutil.TraceDetails + isUpstreamTrace := func(v *testutil.TraceDetails) bool { + if strings.HasPrefix(v.Name, "Envoy: ingress:") { + for _, attr := range v.Spans[0].Raw.Attributes { + if attr.Key == "http.url" { + if regexp.MustCompile(`https://127\.0\.0\.1:\d+/foo`).MatchString(attr.Value.GetStringValue()) { + return true + } + } + } + } + return false + } + for _, v := range inputTraces.ByID { + if isUpstreamTrace(v) { + inputUpstreamTrace = v + break + } + } + for _, v := range outputTraces.ByID { + if isUpstreamTrace(v) { + outputUpstreamTrace = v + break + } + } + equal, diff := inputUpstreamTrace.Equal(outputUpstreamTrace) + if !equal { + assert.Failf(t, "upstream traces not equal:\n%s", diff) + return + } + + // find downstream traces + // should be composed of: + // - 'ingress: GET foo.localhost.pomerium.io:/foo' + // - 'internal: GET authenticate.localhost.pomerium.io:/.pomerium/sign_in' (unauthorized) + // - 'internal: GET authenticate.localhost.pomerium.io:/oauth2/callback' + // - 'internal: GET authenticate.localhost.pomerium.io:/.pomerium/sign_in' (authorized) + // - 'internal: GET foo.localhost.pomerium.io:/.pomerium/callback/' + envoyOutputTraces := outputTraces.ByParticipant["Envoy"] + // there should be two + require.Len(t, envoyOutputTraces, 2) + // find which one is not the upstream trace + var downstreamTrace *testutil.TraceDetails + if envoyOutputTraces[0].ID == outputUpstreamTrace.ID { + downstreamTrace = envoyOutputTraces[1] + } else { + downstreamTrace = envoyOutputTraces[0] + } + tree := downstreamTrace.SpanTree() + require.Empty(t, tree.DetachedParents) + parts := tree.Root.Children + require.Len(t, parts, 5) + assert.True(t, regexp.MustCompile(`ingress: GET foo\.localhost\.pomerium\.io:\d+/foo`).MatchString(parts[0].Span.Raw.Name)) + assert.True(t, regexp.MustCompile(`internal: GET authenticate\.localhost\.pomerium\.io:\d+/.pomerium/sign_in`).MatchString(parts[1].Span.Raw.Name)) + assert.True(t, regexp.MustCompile(`internal: GET authenticate\.localhost\.pomerium\.io:\d+/oauth2/callback`).MatchString(parts[2].Span.Raw.Name)) + assert.True(t, regexp.MustCompile(`internal: GET authenticate\.localhost\.pomerium\.io:\d+/.pomerium/sign_in`).MatchString(parts[3].Span.Raw.Name)) + assert.True(t, regexp.MustCompile(`internal: GET foo\.localhost\.pomerium\.io:\d+/.pomerium/callback`).MatchString(parts[4].Span.Raw.Name)) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mockClient := mock_otlptrace.NewMockClient(ctrl) + var resultsMu sync.Mutex + outputSpans := [][]*tracev1.ResourceSpans{} + mockClient.EXPECT(). + UploadTraces(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, protoSpans []*tracev1.ResourceSpans) error { + resultsMu.Lock() + defer resultsMu.Unlock() + outputSpans = append(outputSpans, protoSpans) + return nil + }). + AnyTimes() + recording1, err := fs.ReadFile(testdata, tc.file) + require.NoError(t, err) + + rec, err := testutil.LoadEventRecording(recording1) + require.NoError(t, err) + + ctx := trace.Options{ + DebugFlags: trace.TrackAllSpans | trace.WarnOnIncompleteSpans | trace.WarnOnIncompleteTraces | trace.WarnOnUnresolvedReferences, + }.NewContext(context.Background()) + queue := trace.NewSpanExportQueue(ctx, mockClient) + + recCloned := rec.Clone() + + err = rec.Replay(func(ctx context.Context, req *coltracepb.ExportTraceServiceRequest) (*coltracepb.ExportTraceServiceResponse, error) { + return &coltracepb.ExportTraceServiceResponse{}, queue.Enqueue(ctx, req) + }) + assert.NoError(t, err) + + // wait for all calls to UploadTraces to complete + ctx, ca := context.WithTimeout(context.Background(), 1*time.Second) + defer ca() + assert.NoError(t, queue.Close(ctx)) + + recCloned.Normalize(rec.NormalizedTo()) + + inputRequests := []*coltracepb.ExportTraceServiceRequest{} + for _, ev := range recCloned.Events() { + inputRequests = append(inputRequests, ev.Request) + } + inputs := testutil.NewTraceResults(testutil.FlattenExportRequests(inputRequests)) + outputs := testutil.NewTraceResults(testutil.FlattenResourceSpans(outputSpans)) + tc.check(t, inputs, outputs) + }) + } +} + +func TestSpanExportQueue_Enqueue(t *testing.T) { + type ( + mapped struct { + s Span + t Trace + } + action struct { + exports []Span + uploads []any // int|mapped|*tracev1.Span + } + testCase struct { + name string + spans []*tracev1.Span // note: span ids are set automatically by index + actions []action + // if actionSets is present, repeats the same test case for each entry + actionSets [][]action + } + ) + + traceparent := func(trace Trace, span Span, sampled ...bool) *commonv1.KeyValue { + if len(sampled) == 0 { + sampled = append(sampled, true) + } + return &commonv1.KeyValue{ + Key: "pomerium.traceparent", + Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_StringValue{ + StringValue: Traceparent(trace, span, sampled[0]), + }}, + } + } + externalParent := func(span Span) *commonv1.KeyValue { + return &commonv1.KeyValue{ + Key: "pomerium.external-parent-span", + Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_StringValue{ + StringValue: span.ID().String(), + }}, + } + } + attrs := func(kvs ...*commonv1.KeyValue) []*commonv1.KeyValue { return kvs } + + cases := []testCase{ + { + name: "single trace", + spans: []*tracev1.Span{ + // |<========>| Span 1 + // | <======> | Span 2 + // | <====> | Span 3 + // T123456789A- + Span(1): { + TraceId: Trace(1).B(), + ParentSpanId: nil, + StartTimeUnixNano: 1, + EndTimeUnixNano: 0xA, + }, + Span(2): { + TraceId: Trace(1).B(), + ParentSpanId: Span(1).B(), + StartTimeUnixNano: 2, + EndTimeUnixNano: 9, + }, + Span(3): { + TraceId: Trace(1).B(), + ParentSpanId: Span(2).B(), + StartTimeUnixNano: 3, + EndTimeUnixNano: 8, + }, + }, + actionSets: [][]action{ + // root span first + { + {exports: []Span{1}, uploads: []any{1}}, + {exports: []Span{2, 3}, uploads: []any{2, 3}}, + }, + { + {exports: []Span{1, 2}, uploads: []any{1, 2}}, + {exports: []Span{3}, uploads: []any{3}}, + }, + { + {exports: []Span{1, 2, 3}, uploads: []any{1, 2, 3}}, + }, + { + {exports: []Span{1, 3, 2}, uploads: []any{1, 2, 3}}, + }, + { + {exports: []Span{1}, uploads: []any{1}}, + {exports: []Span{2}, uploads: []any{2}}, + {exports: []Span{3}, uploads: []any{3}}, + }, + { + {exports: []Span{1}, uploads: []any{1}}, + {exports: []Span{3}, uploads: []any{3}}, + {exports: []Span{2}, uploads: []any{2}}, + }, + // root span last + { + {exports: []Span{2}, uploads: []any{}}, + {exports: []Span{3}, uploads: []any{}}, + {exports: []Span{1}, uploads: []any{1, 2, 3}}, + }, + { + {exports: []Span{3}, uploads: []any{}}, + {exports: []Span{2}, uploads: []any{}}, + {exports: []Span{1}, uploads: []any{1, 2, 3}}, + }, + { + {exports: []Span{2, 3}, uploads: []any{}}, + {exports: []Span{1}, uploads: []any{1, 2, 3}}, + }, + { + {exports: []Span{3, 2}, uploads: []any{}}, + {exports: []Span{1}, uploads: []any{1, 2, 3}}, + }, + { + {exports: []Span{3}, uploads: []any{}}, + {exports: []Span{2, 1}, uploads: []any{1, 2, 3}}, + }, + { + {exports: []Span{2, 3, 1}, uploads: []any{1, 2, 3}}, + }, + // root span in the middle + { + {exports: []Span{2}, uploads: []any{}}, + {exports: []Span{1}, uploads: []any{1, 2}}, + {exports: []Span{3}, uploads: []any{3}}, + }, + { + {exports: []Span{3}, uploads: []any{}}, + {exports: []Span{1}, uploads: []any{1, 3}}, + {exports: []Span{2}, uploads: []any{2}}, + }, + { + {exports: []Span{3}, uploads: []any{}}, + {exports: []Span{1, 2}, uploads: []any{1, 2, 3}}, + }, + { + {exports: []Span{2}, uploads: []any{}}, + {exports: []Span{1, 3}, uploads: []any{1, 2, 3}}, + }, + }, + }, + { + name: "two correlated traces", + spans: []*tracev1.Span{ + // |<=====> | Span 1 (Trace 1) + // | <===> | Span 2 (Trace 1) + // | <=> | Span 3 (Trace 1) + // | <======>| Span 4 (Trace 2) + // | <====> | Span 5 (Trace 2) + // T123456789ABCDEF- + Span(1): { + TraceId: Trace(1).B(), + ParentSpanId: nil, + StartTimeUnixNano: 1, + EndTimeUnixNano: 7, + }, + Span(2): { + TraceId: Trace(1).B(), + ParentSpanId: Span(1).B(), + StartTimeUnixNano: 2, + EndTimeUnixNano: 6, + }, + Span(3): { + TraceId: Trace(1).B(), + ParentSpanId: Span(2).B(), + StartTimeUnixNano: 3, + EndTimeUnixNano: 5, + }, + Span(4): { + TraceId: Trace(2).B(), + ParentSpanId: nil, + Attributes: attrs(traceparent(Trace(1), Span(1))), + StartTimeUnixNano: 8, + EndTimeUnixNano: 0xF, + }, + Span(5): { + TraceId: Trace(2).B(), + ParentSpanId: Span(4).B(), + Attributes: attrs(traceparent(Trace(1), Span(1))), + StartTimeUnixNano: 9, + EndTimeUnixNano: 0xE, + }, + }, + actionSets: [][]action{ + 0: { + { + exports: []Span{1, 2, 3, 4, 5}, + uploads: []any{1, 2, 3, mapped{4, Trace(1)}, mapped{5, Trace(1)}}, + }, + }, + 1: { + {exports: []Span{2, 3, 5}, uploads: []any{}}, + { + exports: []Span{1, 4}, + uploads: []any{1, 2, 3, mapped{4, Trace(1)}, mapped{5, Trace(1)}}, + }, + }, + 2: { + {exports: []Span{2, 3, 5}, uploads: []any{}}, + { + exports: []Span{1}, + uploads: []any{1, 2, 3}, + }, + { + exports: []Span{4}, + uploads: []any{mapped{4, Trace(1)}, mapped{5, Trace(1)}}, + }, + }, + 3: { + {exports: []Span{2, 3, 5}, uploads: []any{}}, + { + exports: []Span{4, 1}, + uploads: []any{1, 2, 3, mapped{4, Trace(1)}, mapped{5, Trace(1)}}, + }, + }, + 4: { + {exports: []Span{2, 3, 5}, uploads: []any{}}, + {exports: []Span{4}, uploads: []any{}}, // root span, but mapped to a pending trace + { + exports: []Span{1}, + uploads: []any{1, 2, 3, mapped{4, Trace(1)}, mapped{5, Trace(1)}}, + }, + }, + }, + }, + { + name: "external parent", + spans: []*tracev1.Span{ + // |??????????| Span 1 (external) + // | <======> | Span 2 (internal) + // | <====> | Span 3 + // T123456789A- + Span(2): { + TraceId: Trace(1).B(), + ParentSpanId: Span(1).B(), + StartTimeUnixNano: 2, + EndTimeUnixNano: 9, + Attributes: attrs(externalParent(Span(1))), + }, + Span(3): { + TraceId: Trace(1).B(), + ParentSpanId: Span(2).B(), + StartTimeUnixNano: 3, + EndTimeUnixNano: 8, + }, + }, + actionSets: [][]action{ + { + {exports: []Span{3}, uploads: []any{}}, + {exports: []Span{2}, uploads: []any{2, 3}}, + }, + { + {exports: []Span{2, 3}, uploads: []any{2, 3}}, + }, + { + {exports: []Span{3, 2}, uploads: []any{3, 2}}, + }, + }, + }, + } + + generatedCases := []testCase{} + for _, tc := range cases { + for i, s := range tc.spans { + if s == nil { + continue + } + s.SpanId = Span(i).B() + } + if len(tc.actionSets) > 0 { + generated := []testCase{} + for i, actions := range tc.actionSets { + generated = append(generated, testCase{ + name: fmt.Sprintf("%s (action set %d of %d)", tc.name, i+1, len(tc.actionSets)), + spans: tc.spans, + actions: actions, + }) + } + generatedCases = append(generatedCases, generated...) + } else { + generatedCases = append(generatedCases, tc) + } + } + for _, tc := range generatedCases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mockClient := mock_otlptrace.NewMockClient(ctrl) + var resultsMu sync.Mutex + outputSpans := make(chan []*tracev1.ResourceSpans, 64) + mockClient.EXPECT(). + UploadTraces(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, protoSpans []*tracev1.ResourceSpans) error { + resultsMu.Lock() + defer resultsMu.Unlock() + outputSpans <- protoSpans + return nil + }). + AnyTimes() + + ctx := trace.Options{ + DebugFlags: trace.TrackAllSpans | trace.WarnOnIncompleteSpans | trace.WarnOnIncompleteTraces | trace.WarnOnUnresolvedReferences, + }.NewContext(context.Background()) + queue := trace.NewSpanExportQueue(ctx, mockClient) + + for actionIdx, action := range tc.actions { + spans := []*tracev1.Span{} + for _, idx := range action.exports { + spans = append(spans, proto.Clone(tc.spans[idx]).(*tracev1.Span)) + } + assert.NoError(t, queue.Enqueue(ctx, &coltracepb.ExportTraceServiceRequest{ + ResourceSpans: []*tracev1.ResourceSpans{ + { + Resource: Resource(1).Make().Resource, + ScopeSpans: []*tracev1.ScopeSpans{{Scope: Scope(1).Make().Scope, Spans: spans}}, + }, + }, + })) + if len(action.uploads) == 0 { + for range 5 { + runtime.Gosched() + require.Empty(t, outputSpans) + } + continue + } + expectedSpans := &tracev1.ResourceSpans{ + Resource: Resource(1).Make().Resource, + ScopeSpans: []*tracev1.ScopeSpans{{Scope: Scope(1).Make().Scope}}, + } + for _, expectedUpload := range action.uploads { + switch up := expectedUpload.(type) { + case int: + expectedSpans.ScopeSpans[0].Spans = append(expectedSpans.ScopeSpans[0].Spans, tc.spans[up]) + case mapped: + clone := proto.Clone(tc.spans[up.s]).(*tracev1.Span) + clone.TraceId = up.t.B() + expectedSpans.ScopeSpans[0].Spans = append(expectedSpans.ScopeSpans[0].Spans, clone) + case *tracev1.Span: + expectedSpans.ScopeSpans[0].Spans = append(expectedSpans.ScopeSpans[0].Spans, up) + default: + panic(fmt.Sprintf("test bug: unexpected type: %T", up)) + } + } + select { + case resourceSpans := <-outputSpans: + expected := testutil.NewTraceResults([]*tracev1.ResourceSpans{expectedSpans}) + actual := testutil.NewTraceResults(resourceSpans) + actual.AssertEqual(t, expected, "action %d/%d", actionIdx+1, len(tc.actions)) + case <-time.After(1 * time.Second): + t.Fatalf("timed out waiting for upload (action %d/%d)", actionIdx+1, len(tc.actions)) + } + } + if !t.Failed() { + close(outputSpans) + // ensure the queue is read fully + if !assert.Empty(t, outputSpans) { + for _, out := range <-outputSpans { + t.Log(protojson.Format(out)) + } + } + } + }) + } +} + +func TestSpanObserver(t *testing.T) { + t.Run("observe single reference", func(t *testing.T) { + obs := trace.NewSpanObserver() + assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) + + obs.ObserveReference(Span(1).ID(), Span(2).ID()) + assert.Equal(t, []oteltrace.SpanID{Span(1).ID()}, obs.XUnobservedIDs()) + obs.Observe(Span(1).ID()) + assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) + }) + t.Run("observe multiple references", func(t *testing.T) { + obs := trace.NewSpanObserver() + + obs.ObserveReference(Span(1).ID(), Span(2).ID()) + obs.ObserveReference(Span(1).ID(), Span(3).ID()) + obs.ObserveReference(Span(1).ID(), Span(4).ID()) + assert.Equal(t, []oteltrace.SpanID{Span(1).ID()}, obs.XUnobservedIDs()) + obs.Observe(Span(1).ID()) + assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) + }) + t.Run("observe before reference", func(t *testing.T) { + obs := trace.NewSpanObserver() + + obs.Observe(Span(1).ID()) + assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) + obs.ObserveReference(Span(1).ID(), Span(2).ID()) + assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) + }) + + t.Run("wait", func(t *testing.T) { + obs := trace.NewSpanObserver() + obs.ObserveReference(Span(1).ID(), Span(2).ID()) + obs.Observe(Span(2).ID()) + obs.ObserveReference(Span(3).ID(), Span(4).ID()) + obs.Observe(Span(4).ID()) + obs.ObserveReference(Span(5).ID(), Span(6).ID()) + obs.Observe(Span(6).ID()) + waitOkToExit := atomic.Bool{} + waitExited := atomic.Bool{} + go func() { + defer waitExited.Store(true) + obs.XWait() + assert.True(t, waitOkToExit.Load(), "wait exited early") + }() + + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + obs.Observe(Span(1).ID()) + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + obs.Observe(Span(3).ID()) + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + waitOkToExit.Store(true) + obs.Observe(Span(5).ID()) + assert.Eventually(t, waitExited.Load, 10*time.Millisecond, 1*time.Millisecond) + }) + + t.Run("new references observed during wait", func(t *testing.T) { + obs := trace.NewSpanObserver() + obs.ObserveReference(Span(1).ID(), Span(2).ID()) + obs.Observe(Span(2).ID()) + obs.ObserveReference(Span(3).ID(), Span(4).ID()) + obs.Observe(Span(4).ID()) + obs.ObserveReference(Span(5).ID(), Span(6).ID()) + obs.Observe(Span(6).ID()) + waitOkToExit := atomic.Bool{} + waitExited := atomic.Bool{} + go func() { + defer waitExited.Store(true) + obs.XWait() + assert.True(t, waitOkToExit.Load(), "wait exited early") + }() + + assert.Equal(t, []oteltrace.SpanID{Span(1).ID(), Span(3).ID(), Span(5).ID()}, obs.XUnobservedIDs()) + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + obs.Observe(Span(1).ID()) + assert.Equal(t, []oteltrace.SpanID{Span(3).ID(), Span(5).ID()}, obs.XUnobservedIDs()) + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + obs.Observe(Span(3).ID()) + assert.Equal(t, []oteltrace.SpanID{Span(5).ID()}, obs.XUnobservedIDs()) + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + // observe a new reference + obs.ObserveReference(Span(7).ID(), Span(8).ID()) + obs.Observe(Span(8).ID()) + assert.Equal(t, []oteltrace.SpanID{Span(5).ID(), Span(7).ID()}, obs.XUnobservedIDs()) + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + obs.Observe(Span(5).ID()) + assert.Equal(t, []oteltrace.SpanID{Span(7).ID()}, obs.XUnobservedIDs()) + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + waitOkToExit.Store(true) + obs.Observe(Span(7).ID()) + assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) + assert.Eventually(t, waitExited.Load, 10*time.Millisecond, 1*time.Millisecond) + }) + + t.Run("multiple waiters", func(t *testing.T) { + t.Parallel() + obs := trace.NewSpanObserver() + obs.ObserveReference(Span(1).ID(), Span(2).ID()) + obs.Observe(Span(2).ID()) + + waitersExited := atomic.Int32{} + for range 10 { + go func() { + defer waitersExited.Add(1) + obs.XWait() + }() + } + + assert.Equal(t, []oteltrace.SpanID{Span(1).ID()}, obs.XUnobservedIDs()) + time.Sleep(10 * time.Millisecond) + assert.Equal(t, int32(0), waitersExited.Load()) + + obs.Observe(Span(1).ID()) + + startTime := time.Now() + for waitersExited.Load() != 10 { + if time.Since(startTime) > 1*time.Millisecond { + t.Fatal("timed out") + } + runtime.Gosched() + } + }) +} + +func TestSpanTracker(t *testing.T) { + t.Run("no debug flags", func(t *testing.T) { + t.Parallel() + obs := trace.NewSpanObserver() + tracker := trace.NewSpanTracker(obs, 0) + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracker)) + tracer := tp.Tracer("test") + assert.Equal(t, []oteltrace.SpanID{}, tracker.XInflightSpans()) + _, span1 := tracer.Start(context.Background(), "span 1") + assert.Equal(t, []oteltrace.SpanID{span1.SpanContext().SpanID()}, tracker.XInflightSpans()) + assert.Equal(t, []oteltrace.SpanID{}, obs.XObservedIDs()) + span1.End() + assert.Equal(t, []oteltrace.SpanID{}, tracker.XInflightSpans()) + assert.Equal(t, []oteltrace.SpanID{}, obs.XObservedIDs()) + }) + t.Run("with TrackSpanReferences debug flag", func(t *testing.T) { + t.Parallel() + obs := trace.NewSpanObserver() + tracker := trace.NewSpanTracker(obs, trace.TrackSpanReferences) + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracker)) + tracer := tp.Tracer("test") + assert.Equal(t, []oteltrace.SpanID{}, tracker.XInflightSpans()) + _, span1 := tracer.Start(context.Background(), "span 1") + assert.Equal(t, []oteltrace.SpanID{span1.SpanContext().SpanID()}, tracker.XInflightSpans()) + assert.Equal(t, []oteltrace.SpanID{span1.SpanContext().SpanID()}, obs.XObservedIDs()) + span1.End() + assert.Equal(t, []oteltrace.SpanID{}, tracker.XInflightSpans()) + assert.Equal(t, []oteltrace.SpanID{span1.SpanContext().SpanID()}, obs.XObservedIDs()) + }) +} + +func TestSpanTrackerWarnings(t *testing.T) { + t.Run("WarnOnIncompleteSpans", func(t *testing.T) { + var buf bytes.Buffer + trace.SetDebugMessageWriterForTest(t, &buf) + + obs := trace.NewSpanObserver() + tracker := trace.NewSpanTracker(obs, trace.WarnOnIncompleteSpans) + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracker)) + tracer := tp.Tracer("test") + _, span1 := tracer.Start(context.Background(), "span 1") + + assert.ErrorIs(t, tp.Shutdown(context.Background()), trace.ErrIncompleteSpans) + + assert.Equal(t, fmt.Sprintf(` +================================================== +WARNING: spans not ended: +%s +Note: set TrackAllSpans flag for more info +================================================== +`, span1.SpanContext().SpanID()), buf.String()) + }) + + t.Run("WarnOnIncompleteSpans with TrackAllSpans", func(t *testing.T) { + var buf bytes.Buffer + trace.SetDebugMessageWriterForTest(t, &buf) + + obs := trace.NewSpanObserver() + tracker := trace.NewSpanTracker(obs, trace.WarnOnIncompleteSpans|trace.TrackAllSpans) + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracker)) + tracer := tp.Tracer("test") + _, span1 := tracer.Start(context.Background(), "span 1") + + assert.ErrorIs(t, tp.Shutdown(context.Background()), trace.ErrIncompleteSpans) + + assert.Equal(t, fmt.Sprintf(` +================================================== +WARNING: spans not ended: +'span 1' (trace: %s | span: %s | parent: 0000000000000000) +================================================== +`, span1.SpanContext().TraceID(), span1.SpanContext().SpanID()), buf.String()) + }) + + t.Run("WarnOnIncompleteSpans with TrackAllSpans and stackTraceProcessor", func(t *testing.T) { + var buf bytes.Buffer + trace.SetDebugMessageWriterForTest(t, &buf) + + obs := trace.NewSpanObserver() + tracker := trace.NewSpanTracker(obs, trace.WarnOnIncompleteSpans|trace.TrackAllSpans) + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(&trace.XStackTraceProcessor{}), sdktrace.WithSpanProcessor(tracker)) + tracer := tp.Tracer("test") + _, span1 := tracer.Start(context.Background(), "span 1") + _, file, line, _ := runtime.Caller(0) + line-- + + assert.ErrorIs(t, tp.Shutdown(context.Background()), trace.ErrIncompleteSpans) + + assert.Equal(t, fmt.Sprintf(` +================================================== +WARNING: spans not ended: +'span 1' (trace: %s | span: %s | parent: 0000000000000000 | started at: %s:%d) +================================================== +`, span1.SpanContext().TraceID(), span1.SpanContext().SpanID(), file, line), buf.String()) + }) + + t.Run("LogAllSpansOnWarn", func(t *testing.T) { + var buf bytes.Buffer + trace.SetDebugMessageWriterForTest(t, &buf) + + obs := trace.NewSpanObserver() + tracker := trace.NewSpanTracker(obs, trace.WarnOnIncompleteSpans|trace.TrackAllSpans|trace.LogAllSpansOnWarn) + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(&trace.XStackTraceProcessor{}), sdktrace.WithSpanProcessor(tracker)) + tracer := tp.Tracer("test") + _, span1 := tracer.Start(context.Background(), "span 1") + time.Sleep(10 * time.Millisecond) + span1.End() + time.Sleep(10 * time.Millisecond) + _, span2 := tracer.Start(context.Background(), "span 2") + _, file, line, _ := runtime.Caller(0) + line-- + + tp.Shutdown(context.Background()) + + assert.Equal(t, + fmt.Sprintf(` +================================================== +WARNING: spans not ended: +'span 2' (trace: %[1]s | span: %[2]s | parent: 0000000000000000 | started at: %[3]s:%[4]d) +================================================== + +================================================== +All observed spans: +'span 1' (trace: %[5]s | span: %[6]s | parent: 0000000000000000 | started at: %[3]s:%[7]d) +'span 2' (trace: %[1]s | span: %[2]s | parent: 0000000000000000 | started at: %[3]s:%[4]d) +================================================== +`, + span2.SpanContext().TraceID(), span2.SpanContext().SpanID(), file, line, + span1.SpanContext().TraceID(), span1.SpanContext().SpanID(), line-4, + ), buf.String()) + }) +} diff --git a/internal/telemetry/trace/server.go b/internal/telemetry/trace/server.go new file mode 100644 index 000000000..204b8b41d --- /dev/null +++ b/internal/telemetry/trace/server.go @@ -0,0 +1,96 @@ +package trace + +import ( + "context" + "errors" + "net" + "time" + + coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/test/bufconn" + + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +// Export implements ptraceotlp.GRPCServer. +func (srv *ExporterServer) Export(ctx context.Context, req *coltracepb.ExportTraceServiceRequest) (*coltracepb.ExportTraceServiceResponse, error) { + if err := srv.spanExportQueue.Enqueue(ctx, req); err != nil { + return nil, err + } + return &coltracepb.ExportTraceServiceResponse{}, nil +} + +type ExporterServer struct { + coltracepb.UnimplementedTraceServiceServer + spanExportQueue *SpanExportQueue + server *grpc.Server + remoteClient otlptrace.Client + cc *grpc.ClientConn +} + +func NewServer(ctx context.Context, remoteClient otlptrace.Client) *ExporterServer { + ex := &ExporterServer{ + spanExportQueue: NewSpanExportQueue(ctx, remoteClient), + remoteClient: remoteClient, + server: grpc.NewServer(grpc.Creds(insecure.NewCredentials())), + } + coltracepb.RegisterTraceServiceServer(ex.server, ex) + return ex +} + +func (srv *ExporterServer) Start(ctx context.Context) { + lis := bufconn.Listen(4096) + go func() { + if err := srv.remoteClient.Start(ctx); err != nil { + panic(err) + } + _ = srv.server.Serve(lis) + }() + cc, err := grpc.NewClient("passthrough://ignore", + grpc.WithContextDialer(func(context.Context, string) (net.Conn, error) { + return lis.Dial() + }), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + panic(err) + } + srv.cc = cc +} + +func (srv *ExporterServer) NewClient() otlptrace.Client { + return otlptracegrpc.NewClient( + otlptracegrpc.WithGRPCConn(srv.cc), + otlptracegrpc.WithTimeout(1*time.Minute), + ) +} + +func (srv *ExporterServer) SpanProcessors() []sdktrace.SpanProcessor { + return []sdktrace.SpanProcessor{srv.spanExportQueue.tracker} +} + +func (srv *ExporterServer) Shutdown(ctx context.Context) error { + stopped := make(chan struct{}) + go func() { + srv.server.GracefulStop() + close(stopped) + }() + select { + case <-stopped: + case <-ctx.Done(): + return context.Cause(ctx) + } + var errs []error + if err := srv.spanExportQueue.WaitForSpans(30 * time.Second); err != nil { + errs = append(errs, err) + } + if err := srv.spanExportQueue.Close(ctx); err != nil { + errs = append(errs, err) + } + if err := srv.remoteClient.Stop(ctx); err != nil { + errs = append(errs, err) + } + return errors.Join(errs...) +} diff --git a/internal/telemetry/trace/testdata/README.md b/internal/telemetry/trace/testdata/README.md new file mode 100644 index 000000000..46837ef8a --- /dev/null +++ b/internal/telemetry/trace/testdata/README.md @@ -0,0 +1,9 @@ +These trace recordings are generated as follows: + +- recording_01_single_trace.json: + +`go test -v -run "^TestOTLPTracing$" -env.trace-debug-flags=+32 github.com/pomerium/pomerium/internal/testenv/selftests | grep -ozP "(?s)(?<=All Events:\n).*?(?=\n=====)"` + +- recording_02_multi_trace.json: + +`go test -v -run "^TestOTLPTracing_TraceCorrelation$" -env.trace-debug-flags=+32 github.com/pomerium/pomerium/internal/testenv/selftests | grep -ozP "(?s)(?<=All Events:\n).*?(?=\n=====)"` diff --git a/internal/telemetry/trace/testdata/recording_01_single_trace.json b/internal/telemetry/trace/testdata/recording_01_single_trace.json new file mode 100644 index 000000000..baa0dbd9d --- /dev/null +++ b/internal/telemetry/trace/testdata/recording_01_single_trace.json @@ -0,0 +1,38 @@ +[ + {"timestamp":"2024-12-27T21:30:25.813350216Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"RaoMfsRKjao=","parentSpanId":"5f1kYvXF3VA=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025808414000","endTimeUnixNano":"1735335025809828000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"dq09RB/npOA=","parentSpanId":"5f1kYvXF3VA=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025809683000","endTimeUnixNano":"1735335025812488000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"5f1kYvXF3VA=","parentSpanId":"kNePRE2AarU=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025808050000","endTimeUnixNano":"1735335025812520000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"b78a9ab1-d815-4a6d-b6a5-e473f7eeeb6b"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=Z1Fi-CygBcWOsGaemJ9Kjck8J5CFO2-Q0NaLJ_090KA%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-09f757396858b657-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-ea7902624e0bf002-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"891"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-ea7902624e0bf002-01"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-09f757396858b657-01"}}]}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.819585151Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"tjexoDV7/Hw=","parentSpanId":"CHEWQLz/Esg=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025815586000","endTimeUnixNano":"1735335025816391000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"dMDfBRWWNfg=","parentSpanId":"CHEWQLz/Esg=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025816362000","endTimeUnixNano":"1735335025819200000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"CHEWQLz/Esg=","parentSpanId":"M+elTBSUJY8=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025815369000","endTimeUnixNano":"1735335025819217000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"58a6db9c-74a6-9afc-b6a1-1ad0761df706"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=bVhDeVhqYXBxRzNLS0F5dm1LVXRzUTZrUHVtSDgrcjdhQSswdmZsT3o2eE81OGJsaEgvOEY4UUVZbFV6em95elFySUNmeFJYelJ4Y0RkRWErUVdtWnc9PXwxNzM1MzM1MDI1fFr-RTRGNqnLn2IMEbk7FiyJdpSLrfG2toEU04qhst7IcWtWqDDKpQf-rvb4AOVFEyb_DCJImrMZ_JKA0yiP0S_U54-vwXFi6BfICA4m3NAFaFycuWP-DbKAQAfk9Uba9ichigqepuu4l8w3EGZwlpOR_H98gkVp8i0wUsRSi0QvJExUdEYP2R6hvWCaEF24kJZGzEgc3K8tFLuKKXF51oZzS8uTXWxZKboeoN_QAE7OJ5s2oD_oUJeQW1mU7W5l1lslB214Zv24ft7fLAbVQYJBd14DDoHfg4WZpQE8v3qIuCCMNcgCod69B1Fa-h4Glz138FEfNbnOgFjiCkW63brqiutwbtIn_Voy034oAAK95qCQvwc0B5RCf2peV7-rb3IrZk6h_ABEeNeRZEkKZtsixVns0ew7rEzfUlcPFre-3D7RK0HfV66arIKuNr4qnMLBuIiDO9PuucWyY4Rft3VAp8Onv-ENIWwLVkrOGvnDFtqubfCnutmW1cqXzF1oVuckiEOwmG0k0vUCk7xR8LLEkNBkh3s3yF3F8LvZLlfesQIl-dNXAWWKIonfCyb_fLiciYxlROetfJXFkMr6LKvgCUHB4AAkx_X--KpHabCpBVnFrEIPv2E59D-eLo7J9Sf4Yhd6nDzdAgj5B4BgDPKSLWvAvjRe5-kD2AT2AGcG-pIBVuKZoI7avNO0GJM23k0-b6_djF13Cg8%3D"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"659"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"33e7a54c1494258f"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.823365016Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"8Z+UA49FUMc=","parentSpanId":"6nkCYk4L8AI=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025800352000","endTimeUnixNano":"1735335025804479000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"1ec195fd-430d-9427-9716-caddcb178407"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"1450"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-ea7902624e0bf002-01"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-ea7902624e0bf002-01"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"ea7902624e0bf002"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"P2zcRg+jsFU=","parentSpanId":"8Z+UA49FUMc=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025800818000","endTimeUnixNano":"1735335025804528000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_unauthorized"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"cnJtNlqDufs=","parentSpanId":"uV3raBlyRYI=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025820836000","endTimeUnixNano":"1735335025822775000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"adb206d8-f691-983a-a797-4e35bd587cea"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"1450"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-b95deb6819724582-01"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-b95deb6819724582-01"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"b95deb6819724582"}}]}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.826015448Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"d+7biQunPZg=","parentSpanId":"lJrLEsDUNLM=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025823671000","endTimeUnixNano":"1735335025824511000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"K5Za89EMcFE=","parentSpanId":"lJrLEsDUNLM=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025824482000","endTimeUnixNano":"1735335025825573000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"lJrLEsDUNLM=","parentSpanId":"I9tnFqm43jY=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025823485000","endTimeUnixNano":"1735335025825591000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"88f0b537-f3c1-4574-bfb1-4e9d3f07ec4f"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=PUoMiJkZMinjfOP-kixkfrqo8r54-oHR-9Z-N1_VU5I%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-985db05505a0159d-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-b95deb6819724582-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"894"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-985db05505a0159d-01"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-b95deb6819724582-01"}}]}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.830946117Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"12NNbneNbUM=","parentSpanId":"RmDNaV3g6Fw=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025826885000","endTimeUnixNano":"1735335025827735000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ETof3b34RNg=","parentSpanId":"RmDNaV3g6Fw=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025827707000","endTimeUnixNano":"1735335025830487000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"RmDNaV3g6Fw=","parentSpanId":"wsNTIF8DnsE=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025826714000","endTimeUnixNano":"1735335025830508000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"15101583-1ffb-9f6e-b511-efcd4ca14681"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=TUQrZ3dFSUpycGJ5TFp4c09pdGkvMDB3RWhsUkdLVEZ4Ni9OYmZOMWJuTG5xTlI3OE4vNjdQd0I4cGFSUU1QOUFTWXVqOEs4Z3lMenJhaks4ejRIdVE9PXwxNzM1MzM1MDI1fNlsbQS7GploSTVB0WfdtdH7gi0aLdGkm-Zw-kVVwqJF9rJZSPHGDEYpgyyhScfl32qa-0GvpPpCEPxXfroMBfIx0VSczgLJty1vJ_7-Wvuc_6EdlHR9x544l5p1a7zUqdftP975Xmdcj7tkJwsje3ttJ5uVESkFOy6sN0PAyQihwcYPd0aCvVnLPgVqroEbrbkPcVavCRI8sVE0Awx0X2YzkgBPWzKHmH43iAie-kM0uMZYFPFRtEFxFvD43PscwW-prc3qShMzp4C5cNJt2TRQCRJlXcUr_ccnecAjs3jdys3da_nFHMsbhpDgDfHaYsnL86zq_QVdjWJZFMpIEHOXQUVi2Q7PTm_n9iymHah7FPcWDaAo5BRsKr7fM4_JaHT_BsnrvLMRSV8c61gtzpBoDnIKrrvr7hzua4i1OFkR5XNRSOIy0gh1Re2ZpBFUo0H8miVsqD8Hqx7N1rxFopl3R-spO3a9Arin74L-5gZu1DsgkxVRa9oF5ntJ9ZigAKCnFFRgrmHynjxD4QMxGKR0tFBrlLBYiZFlWpxMYD-bmq6jccxwA8491DQAePeeyWbLIL4-9OIsBoZqIRmJac1gou6EqxJKsECNxxLCCreNKPYcNJKWdY6ZERWT2rVYlzaYowFnfV9KDvQAoaGV_D7rg3Dw7XGnQvo5NHAYGXv5sW7LB-vlPU_5CuVjV2bwmwcy6Ybez22aD1g%3D"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"660"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"c2c353205f039ec1"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.835894006Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"moqlVoGI3u8=","parentSpanId":"cnJtNlqDufs=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025821062000","endTimeUnixNano":"1735335025823076000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_unauthorized"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ANOpBD0iW7E=","parentSpanId":"wQTpo0tPYe8=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025833706000","endTimeUnixNano":"1735335025835562000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"d521e804-e422-9d58-9bd7-6b0ad4803942"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"1450"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-c104e9a34b4f61ef-01"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-c104e9a34b4f61ef-01"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"c104e9a34b4f61ef"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Cw7qX+Mq85E=","parentSpanId":"ANOpBD0iW7E=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025833842000","endTimeUnixNano":"1735335025835586000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_unauthorized"}}],"status":{}}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.838343385Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"aQWRcZQDEL0=","parentSpanId":"cc3o4WXmDUw=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025836159000","endTimeUnixNano":"1735335025836894000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"JMTzABiGm6s=","parentSpanId":"cc3o4WXmDUw=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025836862000","endTimeUnixNano":"1735335025837975000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"cc3o4WXmDUw=","parentSpanId":"dQPCN0PsSTM=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025835958000","endTimeUnixNano":"1735335025837991000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"2fad3a29-7864-427a-b449-6ee502094ba3"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=AwwHIua79vR9AvSBdLu7fXeMAAtUCvoAW35-81UtSuE%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-0f614b85a586f0da-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-c104e9a34b4f61ef-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"890"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-c104e9a34b4f61ef-01"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-0f614b85a586f0da-01"}}]}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.842701287Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"zZCpocublXw=","parentSpanId":"FWcsZeIymTo=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025839230000","endTimeUnixNano":"1735335025839938000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"rLKtQLC36X0=","parentSpanId":"FWcsZeIymTo=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025839887000","endTimeUnixNano":"1735335025842321000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"FWcsZeIymTo=","parentSpanId":"Z7eWk7+izdA=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025839066000","endTimeUnixNano":"1735335025842341000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"03113e71-88c3-94e9-8879-11a03f39105c"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=djBzMm5teGFzNDQxSlorL2ZYVUN2Nk1XVHAraVFnSjhtU01IRGRlU0czSm8zRUlsM296bjlEc0o4VVhXSHFPOTd3QnlDVEhtSlp1dElXS3ExOWx5dVE9PXwxNzM1MzM1MDI1fPYv0au-VRFGwhaPnlvMz_uEJ22N0pzR8VwFreItQnBEo4C5PRNtMUFyKPuWzxHL67dfMAFNXNaYKrpHwE-RshBnJbE4oxQd6W705zHKAwm2lJsMMMHJkHZmBUk_Tghkuj7AsIlCKQHAG4Dd3NuGBjqd3Kqs9kwin5JnZat8aa2-5KX5Lk4QUFo_sRMebvIARsqK2QubYU114RUM62nzm_HmkKMcvh1QsGQ1s7pr_QepoDs7hSj_oXUs_Tc7c9n97sBGwWXfGc4yKLdodqyubc31qdSmHxcU5I8-FEFbbhMWZ3Pk51N7sMOjoYoP2oupL6d5DSf9bCermdQNAm4UElB1aVY0l_rmg282JhxP_5qBCLAml8xhGHT6JxMHAKA8H6MpNBusSbcfPmfuO_px6ipSnKE9ZsmY4y8_nbMZUaIotKED5NJn9YUlKDgrI58R050HTSbv-zy-rBVpLRwFhk2aC5dhYSyxVc6QxEYIrKvnVyouumm4bvB9xankZB_A1ESpBrAPszKOoznYh1y72LR8P18o7tOvUWxonfsKbnNeZ92RZOOIeUcaJ6-ZYUVO4Os5DrL4QSTByrikq4hdsABQHu89omt_6iS2t_salQ6eVrGQLI9E9yHzoDTVzWsYf8DZ9NbpI9OHMBFB7nJ8DDVm7XXvpgKtRezh2JVu9C3BZHLswq_0dJrE2hH30U1_pXy4mAOjddNCVt4%3D"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"663"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"67b79693bfa2cdd0"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.851209249Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ix5QFxi6p84=","parentSpanId":"CXI9ud47kkA=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025849044000","endTimeUnixNano":"1735335025849730000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"UBrFUNOcDBc=","parentSpanId":"CXI9ud47kkA=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025849700000","endTimeUnixNano":"1735335025850755000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"CXI9ud47kkA=","parentSpanId":"yYgi/sxRPQ8=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025848863000","endTimeUnixNano":"1735335025850780000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"2c099712-3542-40aa-b3f2-f4f5cf85e4fc"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=KuYlGzyljFFgiYixx3bBF6bdlA-O1S065hn1lQuRHgM%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-6e31980f6e94316e-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5a6b77e41fb1dd4a-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"893"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-6e31980f6e94316e-01"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5a6b77e41fb1dd4a-01"}}]}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.856404271Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"RjOOCHrl7Pg=","parentSpanId":"+VzL5gg3U4o=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025851987000","endTimeUnixNano":"1735335025852708000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"B7oHC/7Ojus=","parentSpanId":"+VzL5gg3U4o=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025852679000","endTimeUnixNano":"1735335025855477000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"+VzL5gg3U4o=","parentSpanId":"xa5A0dmcNOo=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025851819000","endTimeUnixNano":"1735335025855507000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"e6ac2da1-292e-9a26-957c-d315a30ceae4"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=K05oR213S1ZvdUlZMlh0TDFaRlU3SUlZdmc4OGIxOUFoanBDaXkvNlpZc3ZUeklnc0VQMm1CYjFGYkYrK3ZYdXpnNkNtYS9MZUtleU9DY3NMN0VNUUE9PXwxNzM1MzM1MDI1fF-scOL_CXRp0kDbE4ZjdGwJnPC2MQY02ucYYTtu54IwwWu6QgQgYZ-L1nC0_8TPDlspFKyI_65zC2CAiEPwji6gGkC65hOD0mDcjWFyaMY8NgZJYvackeVgm0tLWY0KPOyJmJEk3oGmXGYlPoPJ_sz8ywot5peE8tV3_7FUgL2k61nEBRmqffg_ICtN21fxrk63NHmQZBw5E6_fj_DSwMT0XeLiu3I2sJSMP8gAXgZ-9KNAU34h9NmBwAqkyCJ5HP7N0cYwc0wCWoZVO8kgI_MrasJvIyvf6UKnp9oBDp8Ro4pTlrukgQgcj1-Wu2x6pCl9xpBT32Ftwl4Vuvr_zI7V6MoqZMKywDBNfsApHUkZTUedX0ZLK9A29RXEVkNqr9OevFVyVsMyOvTZ_tfS7_5hfwzHjyOnHq7q2COA1ocRzWCMaXtIeSk8WwL_K6hCf9vNrpSi0_agf2JWLbefcjy8AUAWjpp6jt4My9JhdRZ-qBmnL973LPUmyiIbYkC0Km1nIe71rItMLqx2SZCwuc8-9jq7DshSiSNzNqMD35PGdiiEawIhMFJwhWRWKCDMPBCYeAkYIODnfJWGBbOrD7kFReqvfO9Jm2zbuaw9tPw6HCHWY8XwIyAougw7FP15F9Vqtzc4imuTmsvkOYJNTvI8lz8Sk2ALwlPo63QbI9DFjlE_uDuGb-HqiUqe0eoIx4EqP9Ni_Qj_AkY%3D"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"660"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"c5ae40d1d99c34ea"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.870897034Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ojpEo1xw0Xg=","parentSpanId":"Wmt35B+x3Uo=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025846526000","endTimeUnixNano":"1735335025848559000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"c51ade07-1ab1-99e7-954d-d001ce8000e2"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"1450"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-5a6b77e41fb1dd4a-01"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"5a6b77e41fb1dd4a"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-5a6b77e41fb1dd4a-01"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"pMCRYOIpDEU=","parentSpanId":"ojpEo1xw0Xg=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025846682000","endTimeUnixNano":"1735335025848588000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_unauthorized"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"tKKFRvA/fOQ=","parentSpanId":"iwk4YAWR/kU=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025868257000","endTimeUnixNano":"1735335025870559000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"98273d47-4ad1-99b2-894b-1969b54c82a8"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"1450"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-8b0938600591fe45-01"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"8b0938600591fe45"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-8b0938600591fe45-01"}}]}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.873291353Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"zprk0rw2m8Q=","parentSpanId":"YtH+uCZXuRo=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025871096000","endTimeUnixNano":"1735335025871827000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"fOxEGb9Booo=","parentSpanId":"YtH+uCZXuRo=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025871799000","endTimeUnixNano":"1735335025872933000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"YtH+uCZXuRo=","parentSpanId":"ChvrQ2kjOv4=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025870948000","endTimeUnixNano":"1735335025872951000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"9f0a905b-af50-4ab6-bd35-31149b175f03"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=NfuC-nJofCctq_72Lvl602hVWbnwI3s-MmMAYICg8RM%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-9690e8cd7247b772-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-8b0938600591fe45-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"894"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-9690e8cd7247b772-01"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-8b0938600591fe45-01"}}]}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.877652095Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"zsr1/Xcbq2A=","parentSpanId":"99HdbStkmnE=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025874182000","endTimeUnixNano":"1735335025874878000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"rid7A7oKsBM=","parentSpanId":"99HdbStkmnE=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025874850000","endTimeUnixNano":"1735335025877277000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"99HdbStkmnE=","parentSpanId":"DtfBvkQHXZs=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025874035000","endTimeUnixNano":"1735335025877299000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"d609fa5c-f8fd-9547-bba0-89464298b9a1"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=NlpEbWhsdDRlbm1nNEVtSWR0U1BsY3RWU3kyRTNvR09DK3o1NnRLUEtGNCtCNUk5NmE0dUE2N01KM0xkdnk2WGgwTjN1eGQ2cG1rLzdweE4wc1JCbFE9PXwxNzM1MzM1MDI1fNPhIc3ToLHpvS_Gvc8aQRaU88cZYSsnQK8fP27HBXGJKzuRaASTXkCUMwIy2bfmT5GrsBxx8-MmupZH8PE_9xtv96G9GEbRgrrFdukuhrQei_80gU6OgUoTwTtCC9jC8T37myP2Oy72fpFGDoXxzEzRGhSyMAHQJy_HBv3PLX-UxHuResc5NH_Yj7aDx16xDkDAWmc-S1VeeUW0zqT8HgnzfXFYdZqf2_tnV12NUp-CLGXLRsU-3XJIsSwy5CRy1mfIGspxD-SxBHxPaRjKMY9FMxc4CsDmR4eiF28glstJrheqExh9wDhG0TFmgwLtEJ1Vq_hLTmV3RZO8jx8ZtdV1id1slzj8fwzRXH-IVKgTJ1uLcA28LLI1ZzJzD2PrTkfISjg6-5nUJv-OYQcQlQI21pxL1s4ICAa0l3KzE5i5_yeymQaGeAfSLIJ3-5RTTwBMrx5zgMlco96WRrJZ604XqQSOgiGoxSLA9aHu__QOK9OU7GFdga92Y67T3vkKwvWXeX3w2EAF__tzDF5dy--oGborqKGcv0fkKeMgRyfxkxJ-E1w0Z7ppcwL6LL1q78zKI8JcLzgV8MyVDcVnHRrkIToVZ5YYavQnbXK_7lfo_XbKyifs4ZqPmwNY0WyGZ0j_ZZTOLRo8lQR2y5NOlTZRWCz3sqkmOJ3SeAoGrHiQ0tZDfFgvO99ucbtQj1_PCyO6u2-JaIpH6fg%3D"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"660"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"0ed7c1be44075d9b"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.890024754Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"AxcQF3bUAPA=","parentSpanId":"tKKFRvA/fOQ=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025868412000","endTimeUnixNano":"1735335025870665000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_unauthorized"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"1e87sjvJiZc=","parentSpanId":"YEKY6t1fm8A=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025887639000","endTimeUnixNano":"1735335025889676000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"44d9cddd-502b-9709-803f-895824cdd2e5"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"1450"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-604298eadd5f9bc0-01"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"604298eadd5f9bc0"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-604298eadd5f9bc0-01"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"VlLCvKW20c8=","parentSpanId":"1e87sjvJiZc=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025887761000","endTimeUnixNano":"1735335025889701000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_unauthorized"}}],"status":{}}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.892699935Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"TnTIhQSeD8s=","parentSpanId":"L7c60gf+bKk=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025890215000","endTimeUnixNano":"1735335025890903000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"jh13uEWyq54=","parentSpanId":"L7c60gf+bKk=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025890877000","endTimeUnixNano":"1735335025892386000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"L7c60gf+bKk=","parentSpanId":"X170o5Ll3/I=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025890067000","endTimeUnixNano":"1735335025892402000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"7aa3c6c4-084b-4830-9982-8d15416ab378"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=dqScDf0iAzuxtV8kOVFuK6LkRdrngMgT7ZF5WZSWxrM%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-75518e62ed861ee4-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-604298eadd5f9bc0-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"890"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-604298eadd5f9bc0-01"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-75518e62ed861ee4-01"}}]}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.897046547Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"fpe5wrNxals=","parentSpanId":"wZndbBgTSW8=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025893566000","endTimeUnixNano":"1735335025894320000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"zaz8bXwr3x0=","parentSpanId":"wZndbBgTSW8=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025894292000","endTimeUnixNano":"1735335025896622000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"wZndbBgTSW8=","parentSpanId":"BDPsEOd6+nI=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025893422000","endTimeUnixNano":"1735335025896638000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"efd64acb-d818-977c-8592-7c99c36e1622"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=eE9BRU5YQUZpTzFaV0pPK28rTmF0NU1xempEV1NzU3hhM1lDL1I2Zm5KNFRkM0NPd3RQY2wxZDAvVVFJaVB1MTN6enlwa1h1NDFaZmRHZGFIdFQxVlE9PXwxNzM1MzM1MDI1fEw2htjJsKXWR0ubmlecrlxxyWaM5lyrfQKVov8UW14H8_5YI7twjd0qGPWd8uLPpHaUpE5zR5PVk37RnPpkNC2J5NteJv5IN51QbBerrsuGuDZ650Sbqr9-p9FXOKmBBcoKRPW_0N-B7PWMWkQIsY5rVbjYeXgfOjuadl0PNkCA0awKZQhOwfJwr8waX88ypxsRszpU26pXB75zl5tO-AOtrROKkjsv_Y0i-Vvvhj7VGlRPsJ2xvNqK9DuJLbdjmyKrSutz1IsGwhs6NWyd82yaid7x7qCob3VkStVsNeLhjcniP9xjD22Hbi50itMD1HyXCTu_hAVyEYw2SHC842nad-4tP1EEvQkB05PBwTT5G324_lbicp6lFCSzGT9rS2MPEuz12m8Uq9hdJEj18am1pZM7vBXgevH6K0mwlpb4eIgqceOwxbgSHeGw07op-0jdeGnbz1REdKGGui4pWW5PmxcCjDiAG6g6kP6DrXUhPvAVXPK3GhpK4lxuLfU84zuQr81Xo0EmYOxQGjvdcYepBDn9anS1tEghjCk7iptxM6zO11Q7vMFI9wcVHrNl3sSCAM2k21BFmPEyWcnWvPFsQMsegbSznjD2X52WjB7k19H406-bHN-d5Py0dcMo185Y1aAIZ3m2CyXNMgPlrBPT-5XDSvf8113VVpTk8Yazr1rx0DzKlmLhWaUo0Nxm9mPwxK17XRvpdSg%3D"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"660"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"0433ec10e77afa72"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.937586142Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"x4/fCcrN2l0=","parentSpanId":"nnEajOq1wt8=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025935495000","endTimeUnixNano":"1735335025936254000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"8j7gcdiyUN4=","parentSpanId":"nnEajOq1wt8=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025936225000","endTimeUnixNano":"1735335025937234000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"nnEajOq1wt8=","parentSpanId":"NEEa6gf13GY=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025935358000","endTimeUnixNano":"1735335025937250000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"bc0d83c8-6396-4767-ba4b-22aab151f5dd"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=1DclRi_1pUHTYKNXCgmLO7xIgxsFHQ6PnZo6vg2QNpc%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-e310a150afa55100-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-61e9711f41e1b82b-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"893"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-e310a150afa55100-01"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-61e9711f41e1b82b-01"}}]}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.945677139Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Ktn4PSE/Eio=","parentSpanId":"QacsP82TJOA=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025942057000","endTimeUnixNano":"1735335025942751000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"PbLOU7xbMgE=","parentSpanId":"QacsP82TJOA=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025942723000","endTimeUnixNano":"1735335025945322000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"QacsP82TJOA=","parentSpanId":"Sn6DR8I82JI=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025941876000","endTimeUnixNano":"1735335025945338000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"96383362-b0e1-9643-a55b-d3552e7f956d"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=Vld4RWw1Q2pKNkZ2NXNST3pzVTdFMi9qSTNMY0E3bm91bDNYUDNyNWVJbUMrekFzSW5WejIySEtxclJscnBvUkkvVWY1RStubmcrT1g3S1llcklSUWc9PXwxNzM1MzM1MDI1fCZPSA2y7mrm6xOOpuWELdtqzK8AmnUFMhpMijy4u4IE5SwEboZafPLVOuG6XLO-hnn5Xf2lTXraFGd6UpqWGelZT7VfHlpWPhDdf2uCYIAj1u6TYRgPS3_hMilmsO6AvSWmX37-XsPL4I78VhCqbL2FoEKz_CgyCAVhowdooxX6INcvZ7-G8BCJX16jnZKu7o3gX8xFl0mrHYKV1EAPY1V2YsxRTGRS2ptkcqEuxXw9YVOQIIsC6OsphlbN91V6qHGhzBRcLX8CZ-8E_2D9Osgiy4-C-koOCtzRLLFJy9S6oq-8Z1c3PK_wGhTu-syftdKFKjKjdRo97DWgjVz_lnKWeYC2usX32gaExGKkVkMERCaQ6JKoi-TXo4kY1g-AywEfn_gehBqAqzlXQ5VHEDdCth5C2BHjOhgUKcDv7W4Oqmp_93_U28elEz4Q72JUpSaDDe1fKA_eArNUrQBJTJ6N6bABEwL52xkh0ifRwWcERMCwnc1bOlAIDs2T1yWW27hdYrpSOZbAwdKuSji2x1ut0-DqgnGoDMMqzgS98MUPsC9I3PNltfu_DgC959VjH7H6_HmE2q1nPELeZUXRCGzAZTOFE23bD0ks6uyipS3jaVb1qxVClG3VMuSLf9SdV2ESH7MWJaXGyvhN8c1KBsiZp5L-7gqCnsZOMYQrJNQIsbdic-DkeRlmQYbs8iZykp-cADl2uTKek-0%3D"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"659"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"4a7e8347c23cd892"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.983824805Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"pVa2P4/shJY=","parentSpanId":"YelxH0HhuCs=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025933096000","endTimeUnixNano":"1735335025935055000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"92ff535f-4f0a-98fc-b12c-76a42b2de861"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"1450"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-61e9711f41e1b82b-01"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"61e9711f41e1b82b"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-61e9711f41e1b82b-01"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"HG6gDV9Mh1I=","parentSpanId":"pVa2P4/shJY=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025933254000","endTimeUnixNano":"1735335025935080000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_unauthorized"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"aOAQNAsmhVU=","parentSpanId":"VDHmxoPFG3g=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025979317000","endTimeUnixNano":"1735335025983431000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"846d72af-c15f-9579-83e0-abc9ad6219ab"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"1450"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-5431e6c683c51b78-01"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"5431e6c683c51b78"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-5431e6c683c51b78-01"}}]}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.986602409Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"U+zO7kgs2kw=","parentSpanId":"UEUDZ2+1HPY=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025984046000","endTimeUnixNano":"1735335025984989000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"C27ROXjc3Mg=","parentSpanId":"UEUDZ2+1HPY=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025984957000","endTimeUnixNano":"1735335025986182000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"UEUDZ2+1HPY=","parentSpanId":"V4KDwU2b7No=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025983829000","endTimeUnixNano":"1735335025986199000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"1b82a31f-db14-4a9a-8d57-7d7c791a52f5"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=ZnX70Wyz8dCAymop0VMwrObf9dt9EeGcdvdSJMWerio%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-a0e19ab5c251cf8a-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5431e6c683c51b78-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"894"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-a0e19ab5c251cf8a-01"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5431e6c683c51b78-01"}}]}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.993743113Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"pqDfjSA5zY8=","parentSpanId":"BHj177uY9Us=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025987402000","endTimeUnixNano":"1735335025988217000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"rn0P4iWyrGg=","parentSpanId":"BHj177uY9Us=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025988185000","endTimeUnixNano":"1735335025993144000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"BHj177uY9Us=","parentSpanId":"X0cwWL6jdNw=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025987256000","endTimeUnixNano":"1735335025993166000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"515da6a0-e13d-9b0c-b59f-618472b894db"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=TlVucmJGSVM1TTkxQjdVeFpFTndpblBPMUpqdjBoOTBCTkN1NVBPa1RWWGkzcC9YNE1Td3RYc3IyOHZQS05HSVA5am9EbngyT0pNdzBzdEQ4Kzhrbmc9PXwxNzM1MzM1MDI1fGFr6SjNR6oY3LbAckTb1zOQwi4DF7qp73tBPnxNCM9uAN-E7benf9dM4rTw1MXWLEUpA1XTw-NdsI5DXa5mG-guOz-4MjnubljhndMmKy6H-mdPjrJwqpirLHSjGNIOMh6prOpDlt1i-RnnOB_wtIY1wxUCPntd-aCiBGfViIgB0aCMQq6YcqcJYohYyPYYILz6IczkvpPa5cevuvnCJ20J7X69_6AJ0tYzwo9PVmYrGFyTJULaNfw-2G-nUAFuP4-FHv4PexN-NB_Qn0kM4KsX6482RPpNetBM8hHfHbdlr6J_018NOc7Jpv092cxDmMpwzeTQ7mKtEgntBj8WrClM6VTrkecRpdJ8yiZhLvgVhjK76JUFXlXNzYItTt-NvBKuXZ-869TmVrWADw6579M1IP2uCvQlE88v4e__LelTsipYfTzM_kN_Zl7hoDur57rTfzsbkipm5-szZ1W4wEksgtPybfnauIOJY3AtTQ1u6HSciCbGULA6N024tcnpDAJ2mOwYuYbMHlvVYpTv9AJ7P6WOuxaD9GQP5jDJb_AcmhzylwX6r7TQrHnPtbRfratJUOP85A9-5vgaKdLJ4QIPgVqPh4eyTR6xRWWkFnkJdUnJ8m8gshI0ZZI1KzM-ORzBPl8TrEFfqrHHVq1ujN7JQknxUgbrWQOU-jzncZnCYk6TBMbyFX7SdKMG8rgbLaDdnANGnoVCiuA%3D"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"352"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-a0e19ab5c251cf8a-01"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"5f473058bea374dc"}}]}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.996693919Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"IwEGClsNoWA=","parentSpanId":"YDVIt+KGpQQ=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025993892000","endTimeUnixNano":"1735335025994689000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"HLHEc+bHits=","parentSpanId":"YDVIt+KGpQQ=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025994653000","endTimeUnixNano":"1735335025996329000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"YDVIt+KGpQQ=","parentSpanId":"4mu5mTDooas=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025993722000","endTimeUnixNano":"1735335025996352000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"8fbba8a8-1d48-4c54-a298-e8193b239932"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=ZnX70Wyz8dCAymop0VMwrObf9dt9EeGcdvdSJMWerio%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-a0e19ab5c251cf8a-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5431e6c683c51b78-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"972"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5431e6c683c51b78-01"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-a0e19ab5c251cf8a-01"}}]}]}]}]}}, + {"timestamp":"2024-12-27T21:30:25.998791864Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"SRBjmOgMIao=","parentSpanId":"aOAQNAsmhVU=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025979443000","endTimeUnixNano":"1735335025983541000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_unauthorized"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"g8gyWKtu1KQ=","parentSpanId":"GeyplZHybww=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025997043000","endTimeUnixNano":"1735335025997853000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"c4K33/UOGkY=","parentSpanId":"GeyplZHybww=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025997706000","endTimeUnixNano":"1735335025998445000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]}]}]}]}}, + {"timestamp":"2024-12-27T21:30:26.00426142Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"GeyplZHybww=","parentSpanId":"fUtVfh0g50A=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025996849000","endTimeUnixNano":"1735335025998532000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"cf4aa962-5833-431a-9c34-698325c32dde"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/.pomerium/callback/?pomerium_expiry=1735335325\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_session_encrypted=gnpoJSWHmBQQkVfvjiOomyk9_F63PJnoFtYhIeue3g62S-0NF5d6BOpeOkgOwptEfJSQ_kKjqSYQSDT6kkqtKF1QDaX1fRCjLC9LHcOeioxnfLQ2rglnkAbs6XFscaRpZFstZ3TlJXUXf4oL9C-DwpXx96tlbHS9RzPLPijhEMnnLi3_zjw0hlHFmk8a6laRLJFuf-n9vtjh8TVnhmeiE7hic2GNbQjGfu8dqWjOvWDhN1XwmMzKGdp3RosO8xLozBy4i9CP1C35UgXeuoPvS8JPa4Y2d3V3YazDKutWLoMdmtZVicgWb80MJRuR9NX4vlscb12wNk2QpO-qxXeLn6L8UennLI8v2RM-G-TPmK7GDQ2Ihfd8ifmIdRNCHIIKpTpNGHuc6uwWmc86lQt6VP7yyn2-6F0lUtMRLSFZMmfaqcEM2I1C1XULxqi-R8W5aQgO5Wb1sFA4quUUNQy2lHanVtFceM8lOVHeIBuOaieLxFvKmf0TILOdlaYDoeNKMLk9Z_gPBEUyngeDwfD-2RimJTkeUJxQUpjCuMtfc_0aWfKg8PpeFrwnQhSBBhdasBEbplMrRZ-YwcC_IBHeLRqANNaW8pTpVaimIsTCDoMq0WQu5UXhGWT_Xbc2BQ0vNZVCKylD47DijPzYP8fQd5ibiJtElOmxwuG_hl_rPEcPQKMnoByUd6MtVvFdkfik-iGqMEEGbYfixzO3D4q5ahd13lWCkaFsajMJS9Mapde3c2V-DwS53stVswOYFnOetvMYt4koEE5mqg27zdXEUTNImTExWvpo5swL9JgGbWlGwJIrTsSLpWcD7cfb\u0026pomerium_signatur"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"66"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-a0e19ab5c251cf8a-01"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5431e6c683c51b78-01"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Vh7x9OYP6sg=","parentSpanId":"hKGR0btH9xI=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025998973000","endTimeUnixNano":"1735335026003511000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"xLaYTRAzDqQ=","parentSpanId":"hKGR0btH9xI=","name":"router route-29d1ce2f2cda20fb egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335026003383000","endTimeUnixNano":"1735335026003898000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:43211"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:43211"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"http.status_code","value":{"stringValue":"200"}},{"key":"response_flags","value":{"stringValue":"-"}}]}]}]}]}}, + {"timestamp":"2024-12-27T21:30:29.645550038Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:377"}},{"key":"service.name","value":{"stringValue":"Test Environment"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"PcivIA1AuLg=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"Start","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024645655072","endTimeUnixNano":"1735335024646175548","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:543"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"Ea263bNDAzA=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.config_source.rebuild","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024738945717","endTimeUnixNano":"1735335024739703437","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"Wz1aMEqBJfk=","parentSpanId":"e8L/2jnMPhI=","flags":256,"name":"envoyconfig.Builder.BuildBootstrapStaticResources","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024957225167","endTimeUnixNano":"1735335024957253369","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"e8L/2jnMPhI=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"envoyconfig.Builder.BuildBootstrap","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024957107426","endTimeUnixNano":"1735335024957297089","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"r2S68piHLi0=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"authorize.newPolicyEvaluator","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024965662429","endTimeUnixNano":"1735335024973816647","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"R41i4hg7Nqc=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"authorize.newPolicyEvaluator","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024974090360","endTimeUnixNano":"1735335024974398313","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"yfcZPaW83Xk=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"snippets.WaitStartupComplete","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024646193238","endTimeUnixNano":"1735335025791941061","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"s/6I32KmWeU=","flags":256,"name":"Authenticate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025791948461","endTimeUnixNano":"1735335026004177499","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:88"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"ELScfHlrO3c=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"State Change Callback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026004322061","endTimeUnixNano":"1735335026004363371","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:932"}},{"key":"state","value":{"stringValue":"Stopping"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"lCPvM1rPzvc=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"State Change Callback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026004331111","endTimeUnixNano":"1735335026004390611","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:932"}},{"key":"state","value":{"stringValue":"Stopping"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"/Z4/vZfyMIg=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025790056388","endTimeUnixNano":"1735335025791635986","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"rBnm2/0CUj0=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"registry.Registry/Report","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025975496615","endTimeUnixNano":"1735335025975526226","attributes":[{"key":"rpc.service","value":{"stringValue":"registry.Registry"}},{"key":"rpc.method","value":{"stringValue":"Report"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"TGyFsg9jcEc=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"registry.Registry/Report","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335026621252279","endTimeUnixNano":"1735335026622954109","attributes":[{"key":"rpc.service","value":{"stringValue":"registry.Registry"}},{"key":"rpc.method","value":{"stringValue":"Report"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, + {"timestamp":"2024-12-27T21:30:29.647499751Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/upstreams/http.go:223"}},{"key":"service.name","value":{"stringValue":"Upstream"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp","version":"0.57.0"},"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"uSbobbStGD4=","parentSpanId":"xLaYTRAzDqQ=","flags":768,"name":"Server: GET /foo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335026003679492","endTimeUnixNano":"1735335026003700273","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"127.0.0.1"}},{"key":"net.host.port","value":{"intValue":"43211"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"35630"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/foo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"2"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, + {"timestamp":"2024-12-27T21:30:29.649291053Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/upstreams/http.go:223"}},{"key":"service.name","value":{"stringValue":"IDP"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp","version":"0.57.0"},"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Y/2ikMC4YHI=","parentSpanId":"68TkOnpSKwY=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025811671136","endTimeUnixNano":"1735335025811745867","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"wGyDo99t63A=","parentSpanId":"AgIT8HoFaKI=","flags":768,"name":"Server: GET /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025814635582","endTimeUnixNano":"1735335025814710443","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"512"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"mDIE54IHDzU=","parentSpanId":"+bOSkydEmoU=","flags":768,"name":"Server: POST /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025815045916","endTimeUnixNano":"1735335025815166098","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"/eKkXKPf76U=","parentSpanId":"7uQdjOjS7cI=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025816774837","endTimeUnixNano":"1735335025816830008","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"RM7RlGkPMJ799QjggkXaDg==","spanId":"aQejqjlsLbY=","flags":256,"name":"Server: POST /oidc/token","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025817074821","endTimeUnixNano":"1735335025817277083","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/token"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"193"}},{"key":"http.response_content_length","value":{"intValue":"673"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"PGcngHRgqCX4iaoO2fEC5g==","spanId":"wT/nSaqRA9g=","parentSpanId":"ZsmRwKi3Ki4=","flags":768,"name":"Server: GET /.well-known/jwks.json","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025817664798","endTimeUnixNano":"1735335025817768259","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/jwks.json"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"237"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"izW4Mbss9bQpEKa6oaesqg==","spanId":"IziQ2Bt/qwU=","flags":256,"name":"Server: GET /oidc/userinfo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025818151274","endTimeUnixNano":"1735335025818204185","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/userinfo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"150"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"hblCu3rYdXg=","parentSpanId":"uMSJ2tNZc4o=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025824939145","endTimeUnixNano":"1735335025824985545","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"R+9qz/UwFgM=","parentSpanId":"jqKetiY96Q0=","flags":768,"name":"Server: GET /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025826074959","endTimeUnixNano":"1735335025826105089","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"512"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"STcXbMrVBjs=","parentSpanId":"X83U/uv8a/A=","flags":768,"name":"Server: POST /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025826410313","endTimeUnixNano":"1735335025826494124","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"xZa7EacaVP4=","parentSpanId":"d/S+GCMxW/g=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025828197924","endTimeUnixNano":"1735335025828253395","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"3yAOnptzzqQGSMsp5b9s/Q==","spanId":"sVuwHgVcAxw=","flags":256,"name":"Server: POST /oidc/token","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025828508488","endTimeUnixNano":"1735335025828682680","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/token"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"193"}},{"key":"http.response_content_length","value":{"intValue":"673"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"Cm2eM1rGKwOPUvNrZ3JnIQ==","spanId":"wUiYdnapEuA=","parentSpanId":"2vnGWHtaRpY=","flags":768,"name":"Server: GET /.well-known/jwks.json","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025828998383","endTimeUnixNano":"1735335025829035064","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/jwks.json"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"237"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"gX+ojROBZc82P4SUdheIqQ==","spanId":"AjkxbAMUXEE=","flags":256,"name":"Server: GET /oidc/userinfo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025829408768","endTimeUnixNano":"1735335025829437668","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/userinfo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"150"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"63DJFKv/oEw=","parentSpanId":"STVwwUHTbwU=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025837400994","endTimeUnixNano":"1735335025837448364","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"EcKjsRkfXgw=","parentSpanId":"pOYMzXr9GH0=","flags":768,"name":"Server: GET /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025838435756","endTimeUnixNano":"1735335025838474057","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"512"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"b+dMjtK4KO4=","parentSpanId":"iIl8PJ7gVyQ=","flags":768,"name":"Server: POST /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025838776380","endTimeUnixNano":"1735335025838842161","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"hIU1ya0kjXw=","parentSpanId":"rhwZXUEWDec=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025840392170","endTimeUnixNano":"1735335025840438940","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"v2TRBVlwtUQGp9pqypOIsg==","spanId":"B/CQbn5aNW0=","flags":256,"name":"Server: POST /oidc/token","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025840684803","endTimeUnixNano":"1735335025840820895","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/token"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"193"}},{"key":"http.response_content_length","value":{"intValue":"673"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"Ng1V9c7eduVtG8yBTNSUdw==","spanId":"KGHvI14mYcw=","parentSpanId":"uAH2dcySq2g=","flags":768,"name":"Server: GET /.well-known/jwks.json","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025841139289","endTimeUnixNano":"1735335025841186439","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/jwks.json"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"237"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"9iXAmWolAOXVMR/gXbFaDg==","spanId":"SAuhBchhq/U=","flags":256,"name":"Server: GET /oidc/userinfo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025841526593","endTimeUnixNano":"1735335025841551763","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/userinfo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"150"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"AmcPtu7pXmc=","parentSpanId":"KVfrvuRAMxg=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025850262848","endTimeUnixNano":"1735335025850319459","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"NQ1MMF1Ptvk=","parentSpanId":"yabq+U/JonU=","flags":768,"name":"Server: GET /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025851255979","endTimeUnixNano":"1735335025851289730","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"512"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Whn0hDahaBI=","parentSpanId":"zFaBQK23WvM=","flags":768,"name":"Server: POST /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025851556073","endTimeUnixNano":"1735335025851635154","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"V6EIVmKug9I=","parentSpanId":"Ua0wEfIKbXg=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025853150272","endTimeUnixNano":"1735335025853212453","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"jW95QpYR4b2l5wJxuDeadQ==","spanId":"WnrnskhtQlM=","flags":256,"name":"Server: POST /oidc/token","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025853460966","endTimeUnixNano":"1735335025853593097","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/token"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"193"}},{"key":"http.response_content_length","value":{"intValue":"673"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"1F2INkD1DM1Gd2zZRmjgeQ==","spanId":"fL9XHs7b2BM=","parentSpanId":"TOYv2v4FgWY=","flags":768,"name":"Server: GET /.well-known/jwks.json","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025853865651","endTimeUnixNano":"1735335025853898042","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/jwks.json"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"237"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"dVmW0pZN9Eh8Lwurw9K1WA==","spanId":"MAiXzy9HNN0=","flags":256,"name":"Server: GET /oidc/userinfo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025854295756","endTimeUnixNano":"1735335025854335637","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/userinfo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"150"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"3FKwAcrzrXo=","parentSpanId":"FBXEc0lb4OY=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025872366062","endTimeUnixNano":"1735335025872423993","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"BrGdyC2/FrU=","parentSpanId":"mOp99KCsZuE=","flags":768,"name":"Server: GET /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025873414044","endTimeUnixNano":"1735335025873448105","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"512"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"L97J195gDzQ=","parentSpanId":"xm+TiiRVATY=","flags":768,"name":"Server: POST /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025873705238","endTimeUnixNano":"1735335025873790809","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"DFKYB+M6GSA=","parentSpanId":"u5Nt8T6Ejo0=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025875318107","endTimeUnixNano":"1735335025875368818","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"tEoEJcw2IpRwMPyBWPFRNA==","spanId":"DrI0XThkTj0=","flags":256,"name":"Server: POST /oidc/token","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025875602100","endTimeUnixNano":"1735335025875774792","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/token"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"193"}},{"key":"http.response_content_length","value":{"intValue":"673"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"3QpZE6RgWdOi5jhjakP3iw==","spanId":"rk1WHi4jjew=","parentSpanId":"ieD2I+EcHco=","flags":768,"name":"Server: GET /.well-known/jwks.json","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025876076737","endTimeUnixNano":"1735335025876116057","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/jwks.json"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"237"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"knYBLzILdB/6QmbrWijZ7Q==","spanId":"gh51Il616ow=","flags":256,"name":"Server: GET /oidc/userinfo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025876497642","endTimeUnixNano":"1735335025876534372","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/userinfo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"150"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"C24Dm6NaHuw=","parentSpanId":"Vb1M0/gTV54=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025891402500","endTimeUnixNano":"1735335025891463470","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ouFveyHvX5w=","parentSpanId":"CMjIXZ4ROSY=","flags":768,"name":"Server: GET /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025892835007","endTimeUnixNano":"1735335025892863517","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"512"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"YkQT7tUnsR0=","parentSpanId":"HWuw3R1htnc=","flags":768,"name":"Server: POST /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025893167181","endTimeUnixNano":"1735335025893234171","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"I/3OAGfQkYM=","parentSpanId":"E3uSnGr1nuo=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025894728240","endTimeUnixNano":"1735335025894779730","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"eRji0KpJP6RxIM1Uo1uVWA==","spanId":"e3qZQ2rnVW8=","flags":256,"name":"Server: POST /oidc/token","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025895040823","endTimeUnixNano":"1735335025895203635","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/token"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"193"}},{"key":"http.response_content_length","value":{"intValue":"673"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"PgqwQu60TK/M1HnlytJ0OA==","spanId":"SBNFjFlUR6o=","parentSpanId":"bO5cO4l7/ZY=","flags":768,"name":"Server: GET /.well-known/jwks.json","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025895488488","endTimeUnixNano":"1735335025895524139","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/jwks.json"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"237"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"NePcUVjMrbGMdEnmUOzekA==","spanId":"h1Xyietj7ck=","flags":256,"name":"Server: GET /oidc/userinfo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025895906514","endTimeUnixNano":"1735335025895972025","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/userinfo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"150"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"z4aamFIOXuE=","parentSpanId":"s5LlPWRVkt0=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025936688012","endTimeUnixNano":"1735335025936746062","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"fk5+rIJna+I=","parentSpanId":"rjRgfziAtxI=","flags":768,"name":"Server: GET /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025937647773","endTimeUnixNano":"1735335025937705593","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"512"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"0IzKTkJIBnQ=","parentSpanId":"AxuIpomNTe4=","flags":768,"name":"Server: POST /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025940776870","endTimeUnixNano":"1735335025941095344","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"S3OrW/9qYDk=","parentSpanId":"MN31qbGJoAI=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025943250280","endTimeUnixNano":"1735335025943297390","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"RuF2XmPgNlsgjQjJ8yCiOw==","spanId":"qll3f4pm2tw=","flags":256,"name":"Server: POST /oidc/token","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025943549793","endTimeUnixNano":"1735335025943753606","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/token"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"193"}},{"key":"http.response_content_length","value":{"intValue":"673"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"Taxuplp6DB+VUFbngmAAPw==","spanId":"pEHc5RZ44Kk=","parentSpanId":"75tyAErilUw=","flags":768,"name":"Server: GET /.well-known/jwks.json","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025944066200","endTimeUnixNano":"1735335025944116931","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/jwks.json"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"237"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"zdmjmNjkYA1wcwCsZgX1hA==","spanId":"kzagdp0+10Y=","flags":256,"name":"Server: GET /oidc/userinfo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025944444054","endTimeUnixNano":"1735335025944473565","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/userinfo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"150"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"v6U7DP7doY8=","parentSpanId":"11tLsq+ze/k=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025985549525","endTimeUnixNano":"1735335025985615986","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"dWO7PrLMczE=","parentSpanId":"zKjX115N+9k=","flags":768,"name":"Server: GET /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025986659259","endTimeUnixNano":"1735335025986699100","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"512"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"GYkxU/Q8BWU=","parentSpanId":"Jz4teHxKmjI=","flags":768,"name":"Server: POST /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025986949583","endTimeUnixNano":"1735335025987023313","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"XtNQUxuKbrM=","parentSpanId":"EqhoNIgJKpM=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025988592292","endTimeUnixNano":"1735335025988645103","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"bnPd9sGAr+N9kLYSZbNJdw==","spanId":"SvEOChXT7SI=","flags":256,"name":"Server: POST /oidc/token","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025988865176","endTimeUnixNano":"1735335025989028967","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/token"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"193"}},{"key":"http.response_content_length","value":{"intValue":"673"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"u+VskjE7NJ5A/WdVWZgX0g==","spanId":"t7GKDNGmpVI=","parentSpanId":"eNxd/O1kD1M=","flags":768,"name":"Server: GET /.well-known/jwks.json","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025989337901","endTimeUnixNano":"1735335025989370341","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/jwks.json"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"237"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"9noCFY0zqk9a5LpKHBqUpg==","spanId":"UgilqIFLTj8=","flags":256,"name":"Server: GET /oidc/userinfo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025989668615","endTimeUnixNano":"1735335025989705785","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/userinfo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"150"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, + {"timestamp":"2024-12-27T21:30:29.649769198Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/upstreams/http.go:228"}},{"key":"service.name","value":{"stringValue":"HTTP Client"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp","version":"0.57.0"},"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"6nkCYk4L8AI=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /foo","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025792124133","endTimeUnixNano":"1735335025804742723","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"net.peer.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}},{"key":"http.response_content_length","value":{"intValue":"1450"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"kNePRE2AarU=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /.pomerium/sign_in","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025804767524","endTimeUnixNano":"1735335025813157644","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=Z1Fi-CygBcWOsGaemJ9Kjck8J5CFO2-Q0NaLJ_090KA%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-09f757396858b657-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-ea7902624e0bf002-01"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"AgIT8HoFaKI=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025813175384","endTimeUnixNano":"1735335025814805524","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=bVhDeVhqYXBxRzNLS0F5dm1LVXRzUTZrUHVtSDgrcjdhQSswdmZsT3o2eE81OGJsaEgvOEY4UUVZbFV6em95elFySUNmeFJYelJ4Y0RkRWErUVdtWnc9PXwxNzM1MzM1MDI1fFr-RTRGNqnLn2IMEbk7FiyJdpSLrfG2toEU04qhst7IcWtWqDDKpQf-rvb4AOVFEyb_DCJImrMZ_JKA0yiP0S_U54-vwXFi6BfICA4m3NAFaFycuWP-DbKAQAfk9Uba9ichigqepuu4l8w3EGZwlpOR_H98gkVp8i0wUsRSi0QvJExUdEYP2R6hvWCaEF24kJZGzEgc3K8tFLuKKXF51oZzS8uTXWxZKboeoN_QAE7OJ5s2oD_oUJeQW1mU7W5l1lslB214Zv24ft7fLAbVQYJBd14DDoHfg4WZpQE8v3qIuCCMNcgCod69B1Fa-h4Glz138FEfNbnOgFjiCkW63brqiutwbtIn_Voy034oAAK95qCQvwc0B5RCf2peV7-rb3IrZk6h_ABEeNeRZEkKZtsixVns0ew7rEzfUlcPFre-3D7RK0HfV66arIKuNr4qnMLBuIiDO9PuucWyY4Rft3VAp8Onv-ENIWwLVkrOGvnDFtqubfCnutmW1cqXzF1oVuckiEOwmG0k0vUCk7xR8LLEkNBkh3s3yF3F8LvZLlfesQIl-dNXAWWKIonfCyb_fLiciYxlROetfJXFkMr6LKvgCUHB4AAkx_X--KpHabCpBVnFrEIPv2E59D-eLo7J9Sf4Yhd6nDzdAgj5B4BgDPKSLWvAvjRe5-kD2AT2AGcG-pIBVuKZoI7avNO0GJM23k0-b6_djF13Cg8%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"512"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"+bOSkydEmoU=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: POST /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025814897615","endTimeUnixNano":"1735335025815243729","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=bVhDeVhqYXBxRzNLS0F5dm1LVXRzUTZrUHVtSDgrcjdhQSswdmZsT3o2eE81OGJsaEgvOEY4UUVZbFV6em95elFySUNmeFJYelJ4Y0RkRWErUVdtWnc9PXwxNzM1MzM1MDI1fFr-RTRGNqnLn2IMEbk7FiyJdpSLrfG2toEU04qhst7IcWtWqDDKpQf-rvb4AOVFEyb_DCJImrMZ_JKA0yiP0S_U54-vwXFi6BfICA4m3NAFaFycuWP-DbKAQAfk9Uba9ichigqepuu4l8w3EGZwlpOR_H98gkVp8i0wUsRSi0QvJExUdEYP2R6hvWCaEF24kJZGzEgc3K8tFLuKKXF51oZzS8uTXWxZKboeoN_QAE7OJ5s2oD_oUJeQW1mU7W5l1lslB214Zv24ft7fLAbVQYJBd14DDoHfg4WZpQE8v3qIuCCMNcgCod69B1Fa-h4Glz138FEfNbnOgFjiCkW63brqiutwbtIn_Voy034oAAK95qCQvwc0B5RCf2peV7-rb3IrZk6h_ABEeNeRZEkKZtsixVns0ew7rEzfUlcPFre-3D7RK0HfV66arIKuNr4qnMLBuIiDO9PuucWyY4Rft3VAp8Onv-ENIWwLVkrOGvnDFtqubfCnutmW1cqXzF1oVuckiEOwmG0k0vUCk7xR8LLEkNBkh3s3yF3F8LvZLlfesQIl-dNXAWWKIonfCyb_fLiciYxlROetfJXFkMr6LKvgCUHB4AAkx_X--KpHabCpBVnFrEIPv2E59D-eLo7J9Sf4Yhd6nDzdAgj5B4BgDPKSLWvAvjRe5-kD2AT2AGcG-pIBVuKZoI7avNO0GJM23k0-b6_djF13Cg8%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"M+elTBSUJY8=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oauth2/callback","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025815258469","endTimeUnixNano":"1735335025819513500","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=bVhDeVhqYXBxRzNLS0F5dm1LVXRzUTZrUHVtSDgrcjdhQSswdmZsT3o2eE81OGJsaEgvOEY4UUVZbFV6em95elFySUNmeFJYelJ4Y0RkRWErUVdtWnc9PXwxNzM1MzM1MDI1fFr-RTRGNqnLn2IMEbk7FiyJdpSLrfG2toEU04qhst7IcWtWqDDKpQf-rvb4AOVFEyb_DCJImrMZ_JKA0yiP0S_U54-vwXFi6BfICA4m3NAFaFycuWP-DbKAQAfk9Uba9ichigqepuu4l8w3EGZwlpOR_H98gkVp8i0wUsRSi0QvJExUdEYP2R6hvWCaEF24kJZGzEgc3K8tFLuKKXF51oZzS8uTXWxZKboeoN_QAE7OJ5s2oD_oUJeQW1mU7W5l1lslB214Zv24ft7fLAbVQYJBd14DDoHfg4WZpQE8v3qIuCCMNcgCod69B1Fa-h4Glz138FEfNbnOgFjiCkW63brqiutwbtIn_Voy034oAAK95qCQvwc0B5RCf2peV7-rb3IrZk6h_ABEeNeRZEkKZtsixVns0ew7rEzfUlcPFre-3D7RK0HfV66arIKuNr4qnMLBuIiDO9PuucWyY4Rft3VAp8Onv-ENIWwLVkrOGvnDFtqubfCnutmW1cqXzF1oVuckiEOwmG0k0vUCk7xR8LLEkNBkh3s3yF3F8LvZLlfesQIl-dNXAWWKIonfCyb_fLiciYxlROetfJXFkMr6LKvgCUHB4AAkx_X--KpHabCpBVnFrEIPv2E59D-eLo7J9Sf4Yhd6nDzdAgj5B4BgDPKSLWvAvjRe5-kD2AT2AGcG-pIBVuKZoI7avNO0GJM23k0-b6_djF13Cg8%3D"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"uV3raBlyRYI=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /foo","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025820715914","endTimeUnixNano":"1735335025823340045","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"net.peer.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}},{"key":"http.response_content_length","value":{"intValue":"1450"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"I9tnFqm43jY=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /.pomerium/sign_in","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025823362446","endTimeUnixNano":"1735335025825936897","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=PUoMiJkZMinjfOP-kixkfrqo8r54-oHR-9Z-N1_VU5I%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-985db05505a0159d-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-b95deb6819724582-01"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"jqKetiY96Q0=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025825951567","endTimeUnixNano":"1735335025826191870","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=TUQrZ3dFSUpycGJ5TFp4c09pdGkvMDB3RWhsUkdLVEZ4Ni9OYmZOMWJuTG5xTlI3OE4vNjdQd0I4cGFSUU1QOUFTWXVqOEs4Z3lMenJhaks4ejRIdVE9PXwxNzM1MzM1MDI1fNlsbQS7GploSTVB0WfdtdH7gi0aLdGkm-Zw-kVVwqJF9rJZSPHGDEYpgyyhScfl32qa-0GvpPpCEPxXfroMBfIx0VSczgLJty1vJ_7-Wvuc_6EdlHR9x544l5p1a7zUqdftP975Xmdcj7tkJwsje3ttJ5uVESkFOy6sN0PAyQihwcYPd0aCvVnLPgVqroEbrbkPcVavCRI8sVE0Awx0X2YzkgBPWzKHmH43iAie-kM0uMZYFPFRtEFxFvD43PscwW-prc3qShMzp4C5cNJt2TRQCRJlXcUr_ccnecAjs3jdys3da_nFHMsbhpDgDfHaYsnL86zq_QVdjWJZFMpIEHOXQUVi2Q7PTm_n9iymHah7FPcWDaAo5BRsKr7fM4_JaHT_BsnrvLMRSV8c61gtzpBoDnIKrrvr7hzua4i1OFkR5XNRSOIy0gh1Re2ZpBFUo0H8miVsqD8Hqx7N1rxFopl3R-spO3a9Arin74L-5gZu1DsgkxVRa9oF5ntJ9ZigAKCnFFRgrmHynjxD4QMxGKR0tFBrlLBYiZFlWpxMYD-bmq6jccxwA8491DQAePeeyWbLIL4-9OIsBoZqIRmJac1gou6EqxJKsECNxxLCCreNKPYcNJKWdY6ZERWT2rVYlzaYowFnfV9KDvQAoaGV_D7rg3Dw7XGnQvo5NHAYGXv5sW7LB-vlPU_5CuVjV2bwmwcy6Ybez22aD1g%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"512"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"X83U/uv8a/A=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: POST /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025826262341","endTimeUnixNano":"1735335025826593365","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=TUQrZ3dFSUpycGJ5TFp4c09pdGkvMDB3RWhsUkdLVEZ4Ni9OYmZOMWJuTG5xTlI3OE4vNjdQd0I4cGFSUU1QOUFTWXVqOEs4Z3lMenJhaks4ejRIdVE9PXwxNzM1MzM1MDI1fNlsbQS7GploSTVB0WfdtdH7gi0aLdGkm-Zw-kVVwqJF9rJZSPHGDEYpgyyhScfl32qa-0GvpPpCEPxXfroMBfIx0VSczgLJty1vJ_7-Wvuc_6EdlHR9x544l5p1a7zUqdftP975Xmdcj7tkJwsje3ttJ5uVESkFOy6sN0PAyQihwcYPd0aCvVnLPgVqroEbrbkPcVavCRI8sVE0Awx0X2YzkgBPWzKHmH43iAie-kM0uMZYFPFRtEFxFvD43PscwW-prc3qShMzp4C5cNJt2TRQCRJlXcUr_ccnecAjs3jdys3da_nFHMsbhpDgDfHaYsnL86zq_QVdjWJZFMpIEHOXQUVi2Q7PTm_n9iymHah7FPcWDaAo5BRsKr7fM4_JaHT_BsnrvLMRSV8c61gtzpBoDnIKrrvr7hzua4i1OFkR5XNRSOIy0gh1Re2ZpBFUo0H8miVsqD8Hqx7N1rxFopl3R-spO3a9Arin74L-5gZu1DsgkxVRa9oF5ntJ9ZigAKCnFFRgrmHynjxD4QMxGKR0tFBrlLBYiZFlWpxMYD-bmq6jccxwA8491DQAePeeyWbLIL4-9OIsBoZqIRmJac1gou6EqxJKsECNxxLCCreNKPYcNJKWdY6ZERWT2rVYlzaYowFnfV9KDvQAoaGV_D7rg3Dw7XGnQvo5NHAYGXv5sW7LB-vlPU_5CuVjV2bwmwcy6Ybez22aD1g%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"wsNTIF8DnsE=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oauth2/callback","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025826614025","endTimeUnixNano":"1735335025830887906","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=TUQrZ3dFSUpycGJ5TFp4c09pdGkvMDB3RWhsUkdLVEZ4Ni9OYmZOMWJuTG5xTlI3OE4vNjdQd0I4cGFSUU1QOUFTWXVqOEs4Z3lMenJhaks4ejRIdVE9PXwxNzM1MzM1MDI1fNlsbQS7GploSTVB0WfdtdH7gi0aLdGkm-Zw-kVVwqJF9rJZSPHGDEYpgyyhScfl32qa-0GvpPpCEPxXfroMBfIx0VSczgLJty1vJ_7-Wvuc_6EdlHR9x544l5p1a7zUqdftP975Xmdcj7tkJwsje3ttJ5uVESkFOy6sN0PAyQihwcYPd0aCvVnLPgVqroEbrbkPcVavCRI8sVE0Awx0X2YzkgBPWzKHmH43iAie-kM0uMZYFPFRtEFxFvD43PscwW-prc3qShMzp4C5cNJt2TRQCRJlXcUr_ccnecAjs3jdys3da_nFHMsbhpDgDfHaYsnL86zq_QVdjWJZFMpIEHOXQUVi2Q7PTm_n9iymHah7FPcWDaAo5BRsKr7fM4_JaHT_BsnrvLMRSV8c61gtzpBoDnIKrrvr7hzua4i1OFkR5XNRSOIy0gh1Re2ZpBFUo0H8miVsqD8Hqx7N1rxFopl3R-spO3a9Arin74L-5gZu1DsgkxVRa9oF5ntJ9ZigAKCnFFRgrmHynjxD4QMxGKR0tFBrlLBYiZFlWpxMYD-bmq6jccxwA8491DQAePeeyWbLIL4-9OIsBoZqIRmJac1gou6EqxJKsECNxxLCCreNKPYcNJKWdY6ZERWT2rVYlzaYowFnfV9KDvQAoaGV_D7rg3Dw7XGnQvo5NHAYGXv5sW7LB-vlPU_5CuVjV2bwmwcy6Ybez22aD1g%3D"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"wQTpo0tPYe8=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /foo","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025833577948","endTimeUnixNano":"1735335025835834705","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"net.peer.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}},{"key":"http.response_content_length","value":{"intValue":"1450"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"dQPCN0PsSTM=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /.pomerium/sign_in","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025835850156","endTimeUnixNano":"1735335025838314955","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=AwwHIua79vR9AvSBdLu7fXeMAAtUCvoAW35-81UtSuE%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-0f614b85a586f0da-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-c104e9a34b4f61ef-01"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"pOYMzXr9GH0=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025838326145","endTimeUnixNano":"1735335025838546248","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=djBzMm5teGFzNDQxSlorL2ZYVUN2Nk1XVHAraVFnSjhtU01IRGRlU0czSm8zRUlsM296bjlEc0o4VVhXSHFPOTd3QnlDVEhtSlp1dElXS3ExOWx5dVE9PXwxNzM1MzM1MDI1fPYv0au-VRFGwhaPnlvMz_uEJ22N0pzR8VwFreItQnBEo4C5PRNtMUFyKPuWzxHL67dfMAFNXNaYKrpHwE-RshBnJbE4oxQd6W705zHKAwm2lJsMMMHJkHZmBUk_Tghkuj7AsIlCKQHAG4Dd3NuGBjqd3Kqs9kwin5JnZat8aa2-5KX5Lk4QUFo_sRMebvIARsqK2QubYU114RUM62nzm_HmkKMcvh1QsGQ1s7pr_QepoDs7hSj_oXUs_Tc7c9n97sBGwWXfGc4yKLdodqyubc31qdSmHxcU5I8-FEFbbhMWZ3Pk51N7sMOjoYoP2oupL6d5DSf9bCermdQNAm4UElB1aVY0l_rmg282JhxP_5qBCLAml8xhGHT6JxMHAKA8H6MpNBusSbcfPmfuO_px6ipSnKE9ZsmY4y8_nbMZUaIotKED5NJn9YUlKDgrI58R050HTSbv-zy-rBVpLRwFhk2aC5dhYSyxVc6QxEYIrKvnVyouumm4bvB9xankZB_A1ESpBrAPszKOoznYh1y72LR8P18o7tOvUWxonfsKbnNeZ92RZOOIeUcaJ6-ZYUVO4Os5DrL4QSTByrikq4hdsABQHu89omt_6iS2t_salQ6eVrGQLI9E9yHzoDTVzWsYf8DZ9NbpI9OHMBFB7nJ8DDVm7XXvpgKtRezh2JVu9C3BZHLswq_0dJrE2hH30U1_pXy4mAOjddNCVt4%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"512"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"iIl8PJ7gVyQ=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: POST /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025838618409","endTimeUnixNano":"1735335025838948772","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=djBzMm5teGFzNDQxSlorL2ZYVUN2Nk1XVHAraVFnSjhtU01IRGRlU0czSm8zRUlsM296bjlEc0o4VVhXSHFPOTd3QnlDVEhtSlp1dElXS3ExOWx5dVE9PXwxNzM1MzM1MDI1fPYv0au-VRFGwhaPnlvMz_uEJ22N0pzR8VwFreItQnBEo4C5PRNtMUFyKPuWzxHL67dfMAFNXNaYKrpHwE-RshBnJbE4oxQd6W705zHKAwm2lJsMMMHJkHZmBUk_Tghkuj7AsIlCKQHAG4Dd3NuGBjqd3Kqs9kwin5JnZat8aa2-5KX5Lk4QUFo_sRMebvIARsqK2QubYU114RUM62nzm_HmkKMcvh1QsGQ1s7pr_QepoDs7hSj_oXUs_Tc7c9n97sBGwWXfGc4yKLdodqyubc31qdSmHxcU5I8-FEFbbhMWZ3Pk51N7sMOjoYoP2oupL6d5DSf9bCermdQNAm4UElB1aVY0l_rmg282JhxP_5qBCLAml8xhGHT6JxMHAKA8H6MpNBusSbcfPmfuO_px6ipSnKE9ZsmY4y8_nbMZUaIotKED5NJn9YUlKDgrI58R050HTSbv-zy-rBVpLRwFhk2aC5dhYSyxVc6QxEYIrKvnVyouumm4bvB9xankZB_A1ESpBrAPszKOoznYh1y72LR8P18o7tOvUWxonfsKbnNeZ92RZOOIeUcaJ6-ZYUVO4Os5DrL4QSTByrikq4hdsABQHu89omt_6iS2t_salQ6eVrGQLI9E9yHzoDTVzWsYf8DZ9NbpI9OHMBFB7nJ8DDVm7XXvpgKtRezh2JVu9C3BZHLswq_0dJrE2hH30U1_pXy4mAOjddNCVt4%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Z7eWk7+izdA=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oauth2/callback","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025838961582","endTimeUnixNano":"1735335025842657227","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=djBzMm5teGFzNDQxSlorL2ZYVUN2Nk1XVHAraVFnSjhtU01IRGRlU0czSm8zRUlsM296bjlEc0o4VVhXSHFPOTd3QnlDVEhtSlp1dElXS3ExOWx5dVE9PXwxNzM1MzM1MDI1fPYv0au-VRFGwhaPnlvMz_uEJ22N0pzR8VwFreItQnBEo4C5PRNtMUFyKPuWzxHL67dfMAFNXNaYKrpHwE-RshBnJbE4oxQd6W705zHKAwm2lJsMMMHJkHZmBUk_Tghkuj7AsIlCKQHAG4Dd3NuGBjqd3Kqs9kwin5JnZat8aa2-5KX5Lk4QUFo_sRMebvIARsqK2QubYU114RUM62nzm_HmkKMcvh1QsGQ1s7pr_QepoDs7hSj_oXUs_Tc7c9n97sBGwWXfGc4yKLdodqyubc31qdSmHxcU5I8-FEFbbhMWZ3Pk51N7sMOjoYoP2oupL6d5DSf9bCermdQNAm4UElB1aVY0l_rmg282JhxP_5qBCLAml8xhGHT6JxMHAKA8H6MpNBusSbcfPmfuO_px6ipSnKE9ZsmY4y8_nbMZUaIotKED5NJn9YUlKDgrI58R050HTSbv-zy-rBVpLRwFhk2aC5dhYSyxVc6QxEYIrKvnVyouumm4bvB9xankZB_A1ESpBrAPszKOoznYh1y72LR8P18o7tOvUWxonfsKbnNeZ92RZOOIeUcaJ6-ZYUVO4Os5DrL4QSTByrikq4hdsABQHu89omt_6iS2t_salQ6eVrGQLI9E9yHzoDTVzWsYf8DZ9NbpI9OHMBFB7nJ8DDVm7XXvpgKtRezh2JVu9C3BZHLswq_0dJrE2hH30U1_pXy4mAOjddNCVt4%3D"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Wmt35B+x3Uo=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /foo","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025846420282","endTimeUnixNano":"1735335025848745650","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"net.peer.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}},{"key":"http.response_content_length","value":{"intValue":"1450"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"yYgi/sxRPQ8=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /.pomerium/sign_in","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025848760220","endTimeUnixNano":"1735335025851122848","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=KuYlGzyljFFgiYixx3bBF6bdlA-O1S065hn1lQuRHgM%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-6e31980f6e94316e-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5a6b77e41fb1dd4a-01"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"yabq+U/JonU=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025851145408","endTimeUnixNano":"1735335025851369911","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=K05oR213S1ZvdUlZMlh0TDFaRlU3SUlZdmc4OGIxOUFoanBDaXkvNlpZc3ZUeklnc0VQMm1CYjFGYkYrK3ZYdXpnNkNtYS9MZUtleU9DY3NMN0VNUUE9PXwxNzM1MzM1MDI1fF-scOL_CXRp0kDbE4ZjdGwJnPC2MQY02ucYYTtu54IwwWu6QgQgYZ-L1nC0_8TPDlspFKyI_65zC2CAiEPwji6gGkC65hOD0mDcjWFyaMY8NgZJYvackeVgm0tLWY0KPOyJmJEk3oGmXGYlPoPJ_sz8ywot5peE8tV3_7FUgL2k61nEBRmqffg_ICtN21fxrk63NHmQZBw5E6_fj_DSwMT0XeLiu3I2sJSMP8gAXgZ-9KNAU34h9NmBwAqkyCJ5HP7N0cYwc0wCWoZVO8kgI_MrasJvIyvf6UKnp9oBDp8Ro4pTlrukgQgcj1-Wu2x6pCl9xpBT32Ftwl4Vuvr_zI7V6MoqZMKywDBNfsApHUkZTUedX0ZLK9A29RXEVkNqr9OevFVyVsMyOvTZ_tfS7_5hfwzHjyOnHq7q2COA1ocRzWCMaXtIeSk8WwL_K6hCf9vNrpSi0_agf2JWLbefcjy8AUAWjpp6jt4My9JhdRZ-qBmnL973LPUmyiIbYkC0Km1nIe71rItMLqx2SZCwuc8-9jq7DshSiSNzNqMD35PGdiiEawIhMFJwhWRWKCDMPBCYeAkYIODnfJWGBbOrD7kFReqvfO9Jm2zbuaw9tPw6HCHWY8XwIyAougw7FP15F9Vqtzc4imuTmsvkOYJNTvI8lz8Sk2ALwlPo63QbI9DFjlE_uDuGb-HqiUqe0eoIx4EqP9Ni_Qj_AkY%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"512"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"zFaBQK23WvM=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: POST /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025851442912","endTimeUnixNano":"1735335025851703335","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=K05oR213S1ZvdUlZMlh0TDFaRlU3SUlZdmc4OGIxOUFoanBDaXkvNlpZc3ZUeklnc0VQMm1CYjFGYkYrK3ZYdXpnNkNtYS9MZUtleU9DY3NMN0VNUUE9PXwxNzM1MzM1MDI1fF-scOL_CXRp0kDbE4ZjdGwJnPC2MQY02ucYYTtu54IwwWu6QgQgYZ-L1nC0_8TPDlspFKyI_65zC2CAiEPwji6gGkC65hOD0mDcjWFyaMY8NgZJYvackeVgm0tLWY0KPOyJmJEk3oGmXGYlPoPJ_sz8ywot5peE8tV3_7FUgL2k61nEBRmqffg_ICtN21fxrk63NHmQZBw5E6_fj_DSwMT0XeLiu3I2sJSMP8gAXgZ-9KNAU34h9NmBwAqkyCJ5HP7N0cYwc0wCWoZVO8kgI_MrasJvIyvf6UKnp9oBDp8Ro4pTlrukgQgcj1-Wu2x6pCl9xpBT32Ftwl4Vuvr_zI7V6MoqZMKywDBNfsApHUkZTUedX0ZLK9A29RXEVkNqr9OevFVyVsMyOvTZ_tfS7_5hfwzHjyOnHq7q2COA1ocRzWCMaXtIeSk8WwL_K6hCf9vNrpSi0_agf2JWLbefcjy8AUAWjpp6jt4My9JhdRZ-qBmnL973LPUmyiIbYkC0Km1nIe71rItMLqx2SZCwuc8-9jq7DshSiSNzNqMD35PGdiiEawIhMFJwhWRWKCDMPBCYeAkYIODnfJWGBbOrD7kFReqvfO9Jm2zbuaw9tPw6HCHWY8XwIyAougw7FP15F9Vqtzc4imuTmsvkOYJNTvI8lz8Sk2ALwlPo63QbI9DFjlE_uDuGb-HqiUqe0eoIx4EqP9Ni_Qj_AkY%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"xa5A0dmcNOo=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oauth2/callback","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025851714055","endTimeUnixNano":"1735335025856269410","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=K05oR213S1ZvdUlZMlh0TDFaRlU3SUlZdmc4OGIxOUFoanBDaXkvNlpZc3ZUeklnc0VQMm1CYjFGYkYrK3ZYdXpnNkNtYS9MZUtleU9DY3NMN0VNUUE9PXwxNzM1MzM1MDI1fF-scOL_CXRp0kDbE4ZjdGwJnPC2MQY02ucYYTtu54IwwWu6QgQgYZ-L1nC0_8TPDlspFKyI_65zC2CAiEPwji6gGkC65hOD0mDcjWFyaMY8NgZJYvackeVgm0tLWY0KPOyJmJEk3oGmXGYlPoPJ_sz8ywot5peE8tV3_7FUgL2k61nEBRmqffg_ICtN21fxrk63NHmQZBw5E6_fj_DSwMT0XeLiu3I2sJSMP8gAXgZ-9KNAU34h9NmBwAqkyCJ5HP7N0cYwc0wCWoZVO8kgI_MrasJvIyvf6UKnp9oBDp8Ro4pTlrukgQgcj1-Wu2x6pCl9xpBT32Ftwl4Vuvr_zI7V6MoqZMKywDBNfsApHUkZTUedX0ZLK9A29RXEVkNqr9OevFVyVsMyOvTZ_tfS7_5hfwzHjyOnHq7q2COA1ocRzWCMaXtIeSk8WwL_K6hCf9vNrpSi0_agf2JWLbefcjy8AUAWjpp6jt4My9JhdRZ-qBmnL973LPUmyiIbYkC0Km1nIe71rItMLqx2SZCwuc8-9jq7DshSiSNzNqMD35PGdiiEawIhMFJwhWRWKCDMPBCYeAkYIODnfJWGBbOrD7kFReqvfO9Jm2zbuaw9tPw6HCHWY8XwIyAougw7FP15F9Vqtzc4imuTmsvkOYJNTvI8lz8Sk2ALwlPo63QbI9DFjlE_uDuGb-HqiUqe0eoIx4EqP9Ni_Qj_AkY%3D"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"iwk4YAWR/kU=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /foo","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025868112882","endTimeUnixNano":"1735335025870819804","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"net.peer.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}},{"key":"http.response_content_length","value":{"intValue":"1450"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ChvrQ2kjOv4=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /.pomerium/sign_in","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025870834974","endTimeUnixNano":"1735335025873259603","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=NfuC-nJofCctq_72Lvl602hVWbnwI3s-MmMAYICg8RM%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-9690e8cd7247b772-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-8b0938600591fe45-01"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"mOp99KCsZuE=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025873277763","endTimeUnixNano":"1735335025873525176","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=NlpEbWhsdDRlbm1nNEVtSWR0U1BsY3RWU3kyRTNvR09DK3o1NnRLUEtGNCtCNUk5NmE0dUE2N01KM0xkdnk2WGgwTjN1eGQ2cG1rLzdweE4wc1JCbFE9PXwxNzM1MzM1MDI1fNPhIc3ToLHpvS_Gvc8aQRaU88cZYSsnQK8fP27HBXGJKzuRaASTXkCUMwIy2bfmT5GrsBxx8-MmupZH8PE_9xtv96G9GEbRgrrFdukuhrQei_80gU6OgUoTwTtCC9jC8T37myP2Oy72fpFGDoXxzEzRGhSyMAHQJy_HBv3PLX-UxHuResc5NH_Yj7aDx16xDkDAWmc-S1VeeUW0zqT8HgnzfXFYdZqf2_tnV12NUp-CLGXLRsU-3XJIsSwy5CRy1mfIGspxD-SxBHxPaRjKMY9FMxc4CsDmR4eiF28glstJrheqExh9wDhG0TFmgwLtEJ1Vq_hLTmV3RZO8jx8ZtdV1id1slzj8fwzRXH-IVKgTJ1uLcA28LLI1ZzJzD2PrTkfISjg6-5nUJv-OYQcQlQI21pxL1s4ICAa0l3KzE5i5_yeymQaGeAfSLIJ3-5RTTwBMrx5zgMlco96WRrJZ604XqQSOgiGoxSLA9aHu__QOK9OU7GFdga92Y67T3vkKwvWXeX3w2EAF__tzDF5dy--oGborqKGcv0fkKeMgRyfxkxJ-E1w0Z7ppcwL6LL1q78zKI8JcLzgV8MyVDcVnHRrkIToVZ5YYavQnbXK_7lfo_XbKyifs4ZqPmwNY0WyGZ0j_ZZTOLRo8lQR2y5NOlTZRWCz3sqkmOJ3SeAoGrHiQ0tZDfFgvO99ucbtQj1_PCyO6u2-JaIpH6fg%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"512"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"xm+TiiRVATY=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: POST /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025873587166","endTimeUnixNano":"1735335025873885981","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=NlpEbWhsdDRlbm1nNEVtSWR0U1BsY3RWU3kyRTNvR09DK3o1NnRLUEtGNCtCNUk5NmE0dUE2N01KM0xkdnk2WGgwTjN1eGQ2cG1rLzdweE4wc1JCbFE9PXwxNzM1MzM1MDI1fNPhIc3ToLHpvS_Gvc8aQRaU88cZYSsnQK8fP27HBXGJKzuRaASTXkCUMwIy2bfmT5GrsBxx8-MmupZH8PE_9xtv96G9GEbRgrrFdukuhrQei_80gU6OgUoTwTtCC9jC8T37myP2Oy72fpFGDoXxzEzRGhSyMAHQJy_HBv3PLX-UxHuResc5NH_Yj7aDx16xDkDAWmc-S1VeeUW0zqT8HgnzfXFYdZqf2_tnV12NUp-CLGXLRsU-3XJIsSwy5CRy1mfIGspxD-SxBHxPaRjKMY9FMxc4CsDmR4eiF28glstJrheqExh9wDhG0TFmgwLtEJ1Vq_hLTmV3RZO8jx8ZtdV1id1slzj8fwzRXH-IVKgTJ1uLcA28LLI1ZzJzD2PrTkfISjg6-5nUJv-OYQcQlQI21pxL1s4ICAa0l3KzE5i5_yeymQaGeAfSLIJ3-5RTTwBMrx5zgMlco96WRrJZ604XqQSOgiGoxSLA9aHu__QOK9OU7GFdga92Y67T3vkKwvWXeX3w2EAF__tzDF5dy--oGborqKGcv0fkKeMgRyfxkxJ-E1w0Z7ppcwL6LL1q78zKI8JcLzgV8MyVDcVnHRrkIToVZ5YYavQnbXK_7lfo_XbKyifs4ZqPmwNY0WyGZ0j_ZZTOLRo8lQR2y5NOlTZRWCz3sqkmOJ3SeAoGrHiQ0tZDfFgvO99ucbtQj1_PCyO6u2-JaIpH6fg%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"DtfBvkQHXZs=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oauth2/callback","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025873931101","endTimeUnixNano":"1735335025877578894","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=NlpEbWhsdDRlbm1nNEVtSWR0U1BsY3RWU3kyRTNvR09DK3o1NnRLUEtGNCtCNUk5NmE0dUE2N01KM0xkdnk2WGgwTjN1eGQ2cG1rLzdweE4wc1JCbFE9PXwxNzM1MzM1MDI1fNPhIc3ToLHpvS_Gvc8aQRaU88cZYSsnQK8fP27HBXGJKzuRaASTXkCUMwIy2bfmT5GrsBxx8-MmupZH8PE_9xtv96G9GEbRgrrFdukuhrQei_80gU6OgUoTwTtCC9jC8T37myP2Oy72fpFGDoXxzEzRGhSyMAHQJy_HBv3PLX-UxHuResc5NH_Yj7aDx16xDkDAWmc-S1VeeUW0zqT8HgnzfXFYdZqf2_tnV12NUp-CLGXLRsU-3XJIsSwy5CRy1mfIGspxD-SxBHxPaRjKMY9FMxc4CsDmR4eiF28glstJrheqExh9wDhG0TFmgwLtEJ1Vq_hLTmV3RZO8jx8ZtdV1id1slzj8fwzRXH-IVKgTJ1uLcA28LLI1ZzJzD2PrTkfISjg6-5nUJv-OYQcQlQI21pxL1s4ICAa0l3KzE5i5_yeymQaGeAfSLIJ3-5RTTwBMrx5zgMlco96WRrJZ604XqQSOgiGoxSLA9aHu__QOK9OU7GFdga92Y67T3vkKwvWXeX3w2EAF__tzDF5dy--oGborqKGcv0fkKeMgRyfxkxJ-E1w0Z7ppcwL6LL1q78zKI8JcLzgV8MyVDcVnHRrkIToVZ5YYavQnbXK_7lfo_XbKyifs4ZqPmwNY0WyGZ0j_ZZTOLRo8lQR2y5NOlTZRWCz3sqkmOJ3SeAoGrHiQ0tZDfFgvO99ucbtQj1_PCyO6u2-JaIpH6fg%3D"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"YEKY6t1fm8A=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /foo","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025887526993","endTimeUnixNano":"1735335025889954213","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"net.peer.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}},{"key":"http.response_content_length","value":{"intValue":"1450"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"X170o5Ll3/I=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /.pomerium/sign_in","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025889968233","endTimeUnixNano":"1735335025892715716","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=dqScDf0iAzuxtV8kOVFuK6LkRdrngMgT7ZF5WZSWxrM%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-75518e62ed861ee4-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-604298eadd5f9bc0-01"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"CMjIXZ4ROSY=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025892732676","endTimeUnixNano":"1735335025892959438","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=eE9BRU5YQUZpTzFaV0pPK28rTmF0NU1xempEV1NzU3hhM1lDL1I2Zm5KNFRkM0NPd3RQY2wxZDAvVVFJaVB1MTN6enlwa1h1NDFaZmRHZGFIdFQxVlE9PXwxNzM1MzM1MDI1fEw2htjJsKXWR0ubmlecrlxxyWaM5lyrfQKVov8UW14H8_5YI7twjd0qGPWd8uLPpHaUpE5zR5PVk37RnPpkNC2J5NteJv5IN51QbBerrsuGuDZ650Sbqr9-p9FXOKmBBcoKRPW_0N-B7PWMWkQIsY5rVbjYeXgfOjuadl0PNkCA0awKZQhOwfJwr8waX88ypxsRszpU26pXB75zl5tO-AOtrROKkjsv_Y0i-Vvvhj7VGlRPsJ2xvNqK9DuJLbdjmyKrSutz1IsGwhs6NWyd82yaid7x7qCob3VkStVsNeLhjcniP9xjD22Hbi50itMD1HyXCTu_hAVyEYw2SHC842nad-4tP1EEvQkB05PBwTT5G324_lbicp6lFCSzGT9rS2MPEuz12m8Uq9hdJEj18am1pZM7vBXgevH6K0mwlpb4eIgqceOwxbgSHeGw07op-0jdeGnbz1REdKGGui4pWW5PmxcCjDiAG6g6kP6DrXUhPvAVXPK3GhpK4lxuLfU84zuQr81Xo0EmYOxQGjvdcYepBDn9anS1tEghjCk7iptxM6zO11Q7vMFI9wcVHrNl3sSCAM2k21BFmPEyWcnWvPFsQMsegbSznjD2X52WjB7k19H406-bHN-d5Py0dcMo185Y1aAIZ3m2CyXNMgPlrBPT-5XDSvf8113VVpTk8Yazr1rx0DzKlmLhWaUo0Nxm9mPwxK17XRvpdSg%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"512"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"HWuw3R1htnc=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: POST /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025893017369","endTimeUnixNano":"1735335025893320422","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=eE9BRU5YQUZpTzFaV0pPK28rTmF0NU1xempEV1NzU3hhM1lDL1I2Zm5KNFRkM0NPd3RQY2wxZDAvVVFJaVB1MTN6enlwa1h1NDFaZmRHZGFIdFQxVlE9PXwxNzM1MzM1MDI1fEw2htjJsKXWR0ubmlecrlxxyWaM5lyrfQKVov8UW14H8_5YI7twjd0qGPWd8uLPpHaUpE5zR5PVk37RnPpkNC2J5NteJv5IN51QbBerrsuGuDZ650Sbqr9-p9FXOKmBBcoKRPW_0N-B7PWMWkQIsY5rVbjYeXgfOjuadl0PNkCA0awKZQhOwfJwr8waX88ypxsRszpU26pXB75zl5tO-AOtrROKkjsv_Y0i-Vvvhj7VGlRPsJ2xvNqK9DuJLbdjmyKrSutz1IsGwhs6NWyd82yaid7x7qCob3VkStVsNeLhjcniP9xjD22Hbi50itMD1HyXCTu_hAVyEYw2SHC842nad-4tP1EEvQkB05PBwTT5G324_lbicp6lFCSzGT9rS2MPEuz12m8Uq9hdJEj18am1pZM7vBXgevH6K0mwlpb4eIgqceOwxbgSHeGw07op-0jdeGnbz1REdKGGui4pWW5PmxcCjDiAG6g6kP6DrXUhPvAVXPK3GhpK4lxuLfU84zuQr81Xo0EmYOxQGjvdcYepBDn9anS1tEghjCk7iptxM6zO11Q7vMFI9wcVHrNl3sSCAM2k21BFmPEyWcnWvPFsQMsegbSznjD2X52WjB7k19H406-bHN-d5Py0dcMo185Y1aAIZ3m2CyXNMgPlrBPT-5XDSvf8113VVpTk8Yazr1rx0DzKlmLhWaUo0Nxm9mPwxK17XRvpdSg%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"BDPsEOd6+nI=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oauth2/callback","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025893331733","endTimeUnixNano":"1735335025896943046","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=eE9BRU5YQUZpTzFaV0pPK28rTmF0NU1xempEV1NzU3hhM1lDL1I2Zm5KNFRkM0NPd3RQY2wxZDAvVVFJaVB1MTN6enlwa1h1NDFaZmRHZGFIdFQxVlE9PXwxNzM1MzM1MDI1fEw2htjJsKXWR0ubmlecrlxxyWaM5lyrfQKVov8UW14H8_5YI7twjd0qGPWd8uLPpHaUpE5zR5PVk37RnPpkNC2J5NteJv5IN51QbBerrsuGuDZ650Sbqr9-p9FXOKmBBcoKRPW_0N-B7PWMWkQIsY5rVbjYeXgfOjuadl0PNkCA0awKZQhOwfJwr8waX88ypxsRszpU26pXB75zl5tO-AOtrROKkjsv_Y0i-Vvvhj7VGlRPsJ2xvNqK9DuJLbdjmyKrSutz1IsGwhs6NWyd82yaid7x7qCob3VkStVsNeLhjcniP9xjD22Hbi50itMD1HyXCTu_hAVyEYw2SHC842nad-4tP1EEvQkB05PBwTT5G324_lbicp6lFCSzGT9rS2MPEuz12m8Uq9hdJEj18am1pZM7vBXgevH6K0mwlpb4eIgqceOwxbgSHeGw07op-0jdeGnbz1REdKGGui4pWW5PmxcCjDiAG6g6kP6DrXUhPvAVXPK3GhpK4lxuLfU84zuQr81Xo0EmYOxQGjvdcYepBDn9anS1tEghjCk7iptxM6zO11Q7vMFI9wcVHrNl3sSCAM2k21BFmPEyWcnWvPFsQMsegbSznjD2X52WjB7k19H406-bHN-d5Py0dcMo185Y1aAIZ3m2CyXNMgPlrBPT-5XDSvf8113VVpTk8Yazr1rx0DzKlmLhWaUo0Nxm9mPwxK17XRvpdSg%3D"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"YelxH0HhuCs=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /foo","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025932967647","endTimeUnixNano":"1735335025935240514","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"net.peer.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}},{"key":"http.response_content_length","value":{"intValue":"1450"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"NEEa6gf13GY=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /.pomerium/sign_in","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025935254354","endTimeUnixNano":"1735335025937528801","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=1DclRi_1pUHTYKNXCgmLO7xIgxsFHQ6PnZo6vg2QNpc%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-e310a150afa55100-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-61e9711f41e1b82b-01"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"rjRgfziAtxI=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025937538181","endTimeUnixNano":"1735335025937811784","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=Vld4RWw1Q2pKNkZ2NXNST3pzVTdFMi9qSTNMY0E3bm91bDNYUDNyNWVJbUMrekFzSW5WejIySEtxclJscnBvUkkvVWY1RStubmcrT1g3S1llcklSUWc9PXwxNzM1MzM1MDI1fCZPSA2y7mrm6xOOpuWELdtqzK8AmnUFMhpMijy4u4IE5SwEboZafPLVOuG6XLO-hnn5Xf2lTXraFGd6UpqWGelZT7VfHlpWPhDdf2uCYIAj1u6TYRgPS3_hMilmsO6AvSWmX37-XsPL4I78VhCqbL2FoEKz_CgyCAVhowdooxX6INcvZ7-G8BCJX16jnZKu7o3gX8xFl0mrHYKV1EAPY1V2YsxRTGRS2ptkcqEuxXw9YVOQIIsC6OsphlbN91V6qHGhzBRcLX8CZ-8E_2D9Osgiy4-C-koOCtzRLLFJy9S6oq-8Z1c3PK_wGhTu-syftdKFKjKjdRo97DWgjVz_lnKWeYC2usX32gaExGKkVkMERCaQ6JKoi-TXo4kY1g-AywEfn_gehBqAqzlXQ5VHEDdCth5C2BHjOhgUKcDv7W4Oqmp_93_U28elEz4Q72JUpSaDDe1fKA_eArNUrQBJTJ6N6bABEwL52xkh0ifRwWcERMCwnc1bOlAIDs2T1yWW27hdYrpSOZbAwdKuSji2x1ut0-DqgnGoDMMqzgS98MUPsC9I3PNltfu_DgC959VjH7H6_HmE2q1nPELeZUXRCGzAZTOFE23bD0ks6uyipS3jaVb1qxVClG3VMuSLf9SdV2ESH7MWJaXGyvhN8c1KBsiZp5L-7gqCnsZOMYQrJNQIsbdic-DkeRlmQYbs8iZykp-cADl2uTKek-0%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"512"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"AxuIpomNTe4=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: POST /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025939701887","endTimeUnixNano":"1735335025941702821","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=Vld4RWw1Q2pKNkZ2NXNST3pzVTdFMi9qSTNMY0E3bm91bDNYUDNyNWVJbUMrekFzSW5WejIySEtxclJscnBvUkkvVWY1RStubmcrT1g3S1llcklSUWc9PXwxNzM1MzM1MDI1fCZPSA2y7mrm6xOOpuWELdtqzK8AmnUFMhpMijy4u4IE5SwEboZafPLVOuG6XLO-hnn5Xf2lTXraFGd6UpqWGelZT7VfHlpWPhDdf2uCYIAj1u6TYRgPS3_hMilmsO6AvSWmX37-XsPL4I78VhCqbL2FoEKz_CgyCAVhowdooxX6INcvZ7-G8BCJX16jnZKu7o3gX8xFl0mrHYKV1EAPY1V2YsxRTGRS2ptkcqEuxXw9YVOQIIsC6OsphlbN91V6qHGhzBRcLX8CZ-8E_2D9Osgiy4-C-koOCtzRLLFJy9S6oq-8Z1c3PK_wGhTu-syftdKFKjKjdRo97DWgjVz_lnKWeYC2usX32gaExGKkVkMERCaQ6JKoi-TXo4kY1g-AywEfn_gehBqAqzlXQ5VHEDdCth5C2BHjOhgUKcDv7W4Oqmp_93_U28elEz4Q72JUpSaDDe1fKA_eArNUrQBJTJ6N6bABEwL52xkh0ifRwWcERMCwnc1bOlAIDs2T1yWW27hdYrpSOZbAwdKuSji2x1ut0-DqgnGoDMMqzgS98MUPsC9I3PNltfu_DgC959VjH7H6_HmE2q1nPELeZUXRCGzAZTOFE23bD0ks6uyipS3jaVb1qxVClG3VMuSLf9SdV2ESH7MWJaXGyvhN8c1KBsiZp5L-7gqCnsZOMYQrJNQIsbdic-DkeRlmQYbs8iZykp-cADl2uTKek-0%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Sn6DR8I82JI=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oauth2/callback","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025941731151","endTimeUnixNano":"1735335025945621198","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=Vld4RWw1Q2pKNkZ2NXNST3pzVTdFMi9qSTNMY0E3bm91bDNYUDNyNWVJbUMrekFzSW5WejIySEtxclJscnBvUkkvVWY1RStubmcrT1g3S1llcklSUWc9PXwxNzM1MzM1MDI1fCZPSA2y7mrm6xOOpuWELdtqzK8AmnUFMhpMijy4u4IE5SwEboZafPLVOuG6XLO-hnn5Xf2lTXraFGd6UpqWGelZT7VfHlpWPhDdf2uCYIAj1u6TYRgPS3_hMilmsO6AvSWmX37-XsPL4I78VhCqbL2FoEKz_CgyCAVhowdooxX6INcvZ7-G8BCJX16jnZKu7o3gX8xFl0mrHYKV1EAPY1V2YsxRTGRS2ptkcqEuxXw9YVOQIIsC6OsphlbN91V6qHGhzBRcLX8CZ-8E_2D9Osgiy4-C-koOCtzRLLFJy9S6oq-8Z1c3PK_wGhTu-syftdKFKjKjdRo97DWgjVz_lnKWeYC2usX32gaExGKkVkMERCaQ6JKoi-TXo4kY1g-AywEfn_gehBqAqzlXQ5VHEDdCth5C2BHjOhgUKcDv7W4Oqmp_93_U28elEz4Q72JUpSaDDe1fKA_eArNUrQBJTJ6N6bABEwL52xkh0ifRwWcERMCwnc1bOlAIDs2T1yWW27hdYrpSOZbAwdKuSji2x1ut0-DqgnGoDMMqzgS98MUPsC9I3PNltfu_DgC959VjH7H6_HmE2q1nPELeZUXRCGzAZTOFE23bD0ks6uyipS3jaVb1qxVClG3VMuSLf9SdV2ESH7MWJaXGyvhN8c1KBsiZp5L-7gqCnsZOMYQrJNQIsbdic-DkeRlmQYbs8iZykp-cADl2uTKek-0%3D"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"VDHmxoPFG3g=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /foo","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025979196630","endTimeUnixNano":"1735335025983688133","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"net.peer.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}},{"key":"http.response_content_length","value":{"intValue":"1450"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"V4KDwU2b7No=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /.pomerium/sign_in","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025983702733","endTimeUnixNano":"1735335025986526338","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=ZnX70Wyz8dCAymop0VMwrObf9dt9EeGcdvdSJMWerio%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-a0e19ab5c251cf8a-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5431e6c683c51b78-01"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"zKjX115N+9k=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025986536548","endTimeUnixNano":"1735335025986763250","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=TlVucmJGSVM1TTkxQjdVeFpFTndpblBPMUpqdjBoOTBCTkN1NVBPa1RWWGkzcC9YNE1Td3RYc3IyOHZQS05HSVA5am9EbngyT0pNdzBzdEQ4Kzhrbmc9PXwxNzM1MzM1MDI1fGFr6SjNR6oY3LbAckTb1zOQwi4DF7qp73tBPnxNCM9uAN-E7benf9dM4rTw1MXWLEUpA1XTw-NdsI5DXa5mG-guOz-4MjnubljhndMmKy6H-mdPjrJwqpirLHSjGNIOMh6prOpDlt1i-RnnOB_wtIY1wxUCPntd-aCiBGfViIgB0aCMQq6YcqcJYohYyPYYILz6IczkvpPa5cevuvnCJ20J7X69_6AJ0tYzwo9PVmYrGFyTJULaNfw-2G-nUAFuP4-FHv4PexN-NB_Qn0kM4KsX6482RPpNetBM8hHfHbdlr6J_018NOc7Jpv092cxDmMpwzeTQ7mKtEgntBj8WrClM6VTrkecRpdJ8yiZhLvgVhjK76JUFXlXNzYItTt-NvBKuXZ-869TmVrWADw6579M1IP2uCvQlE88v4e__LelTsipYfTzM_kN_Zl7hoDur57rTfzsbkipm5-szZ1W4wEksgtPybfnauIOJY3AtTQ1u6HSciCbGULA6N024tcnpDAJ2mOwYuYbMHlvVYpTv9AJ7P6WOuxaD9GQP5jDJb_AcmhzylwX6r7TQrHnPtbRfratJUOP85A9-5vgaKdLJ4QIPgVqPh4eyTR6xRWWkFnkJdUnJ8m8gshI0ZZI1KzM-ORzBPl8TrEFfqrHHVq1ujN7JQknxUgbrWQOU-jzncZnCYk6TBMbyFX7SdKMG8rgbLaDdnANGnoVCiuA%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"512"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Jz4teHxKmjI=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: POST /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025986818651","endTimeUnixNano":"1735335025987117384","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=TlVucmJGSVM1TTkxQjdVeFpFTndpblBPMUpqdjBoOTBCTkN1NVBPa1RWWGkzcC9YNE1Td3RYc3IyOHZQS05HSVA5am9EbngyT0pNdzBzdEQ4Kzhrbmc9PXwxNzM1MzM1MDI1fGFr6SjNR6oY3LbAckTb1zOQwi4DF7qp73tBPnxNCM9uAN-E7benf9dM4rTw1MXWLEUpA1XTw-NdsI5DXa5mG-guOz-4MjnubljhndMmKy6H-mdPjrJwqpirLHSjGNIOMh6prOpDlt1i-RnnOB_wtIY1wxUCPntd-aCiBGfViIgB0aCMQq6YcqcJYohYyPYYILz6IczkvpPa5cevuvnCJ20J7X69_6AJ0tYzwo9PVmYrGFyTJULaNfw-2G-nUAFuP4-FHv4PexN-NB_Qn0kM4KsX6482RPpNetBM8hHfHbdlr6J_018NOc7Jpv092cxDmMpwzeTQ7mKtEgntBj8WrClM6VTrkecRpdJ8yiZhLvgVhjK76JUFXlXNzYItTt-NvBKuXZ-869TmVrWADw6579M1IP2uCvQlE88v4e__LelTsipYfTzM_kN_Zl7hoDur57rTfzsbkipm5-szZ1W4wEksgtPybfnauIOJY3AtTQ1u6HSciCbGULA6N024tcnpDAJ2mOwYuYbMHlvVYpTv9AJ7P6WOuxaD9GQP5jDJb_AcmhzylwX6r7TQrHnPtbRfratJUOP85A9-5vgaKdLJ4QIPgVqPh4eyTR6xRWWkFnkJdUnJ8m8gshI0ZZI1KzM-ORzBPl8TrEFfqrHHVq1ujN7JQknxUgbrWQOU-jzncZnCYk6TBMbyFX7SdKMG8rgbLaDdnANGnoVCiuA%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"X0cwWL6jdNw=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oauth2/callback","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025987145055","endTimeUnixNano":"1735335025993611052","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=TlVucmJGSVM1TTkxQjdVeFpFTndpblBPMUpqdjBoOTBCTkN1NVBPa1RWWGkzcC9YNE1Td3RYc3IyOHZQS05HSVA5am9EbngyT0pNdzBzdEQ4Kzhrbmc9PXwxNzM1MzM1MDI1fGFr6SjNR6oY3LbAckTb1zOQwi4DF7qp73tBPnxNCM9uAN-E7benf9dM4rTw1MXWLEUpA1XTw-NdsI5DXa5mG-guOz-4MjnubljhndMmKy6H-mdPjrJwqpirLHSjGNIOMh6prOpDlt1i-RnnOB_wtIY1wxUCPntd-aCiBGfViIgB0aCMQq6YcqcJYohYyPYYILz6IczkvpPa5cevuvnCJ20J7X69_6AJ0tYzwo9PVmYrGFyTJULaNfw-2G-nUAFuP4-FHv4PexN-NB_Qn0kM4KsX6482RPpNetBM8hHfHbdlr6J_018NOc7Jpv092cxDmMpwzeTQ7mKtEgntBj8WrClM6VTrkecRpdJ8yiZhLvgVhjK76JUFXlXNzYItTt-NvBKuXZ-869TmVrWADw6579M1IP2uCvQlE88v4e__LelTsipYfTzM_kN_Zl7hoDur57rTfzsbkipm5-szZ1W4wEksgtPybfnauIOJY3AtTQ1u6HSciCbGULA6N024tcnpDAJ2mOwYuYbMHlvVYpTv9AJ7P6WOuxaD9GQP5jDJb_AcmhzylwX6r7TQrHnPtbRfratJUOP85A9-5vgaKdLJ4QIPgVqPh4eyTR6xRWWkFnkJdUnJ8m8gshI0ZZI1KzM-ORzBPl8TrEFfqrHHVq1ujN7JQknxUgbrWQOU-jzncZnCYk6TBMbyFX7SdKMG8rgbLaDdnANGnoVCiuA%3D"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"4mu5mTDooas=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /.pomerium/sign_in","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025993631222","endTimeUnixNano":"1735335025996706079","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=ZnX70Wyz8dCAymop0VMwrObf9dt9EeGcdvdSJMWerio%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-a0e19ab5c251cf8a-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5431e6c683c51b78-01"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"fUtVfh0g50A=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /.pomerium/callback/","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025996719560","endTimeUnixNano":"1735335025998707173","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/.pomerium/callback/?pomerium_expiry=1735335325\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_session_encrypted=gnpoJSWHmBQQkVfvjiOomyk9_F63PJnoFtYhIeue3g62S-0NF5d6BOpeOkgOwptEfJSQ_kKjqSYQSDT6kkqtKF1QDaX1fRCjLC9LHcOeioxnfLQ2rglnkAbs6XFscaRpZFstZ3TlJXUXf4oL9C-DwpXx96tlbHS9RzPLPijhEMnnLi3_zjw0hlHFmk8a6laRLJFuf-n9vtjh8TVnhmeiE7hic2GNbQjGfu8dqWjOvWDhN1XwmMzKGdp3RosO8xLozBy4i9CP1C35UgXeuoPvS8JPa4Y2d3V3YazDKutWLoMdmtZVicgWb80MJRuR9NX4vlscb12wNk2QpO-qxXeLn6L8UennLI8v2RM-G-TPmK7GDQ2Ihfd8ifmIdRNCHIIKpTpNGHuc6uwWmc86lQt6VP7yyn2-6F0lUtMRLSFZMmfaqcEM2I1C1XULxqi-R8W5aQgO5Wb1sFA4quUUNQy2lHanVtFceM8lOVHeIBuOaieLxFvKmf0TILOdlaYDoeNKMLk9Z_gPBEUyngeDwfD-2RimJTkeUJxQUpjCuMtfc_0aWfKg8PpeFrwnQhSBBhdasBEbplMrRZ-YwcC_IBHeLRqANNaW8pTpVaimIsTCDoMq0WQu5UXhGWT_Xbc2BQ0vNZVCKylD47DijPzYP8fQd5ibiJtElOmxwuG_hl_rPEcPQKMnoByUd6MtVvFdkfik-iGqMEEGbYfixzO3D4q5ahd13lWCkaFsajMJS9Mapde3c2V-DwS53stVswOYFnOetvMYt4koEE5mqg27zdXEUTNImTExWvpo5swL9JgGbWlGwJIrTsSLpWcD7cfb\u0026pomerium_signature=ylPA2kChpnYPWI9WUZDIU5mnENgGeREsecrSWxxxN9g%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-a0e19ab5c251cf8a-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5431e6c683c51b78-01"}},{"key":"net.peer.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}},{"key":"http.response_content_length","value":{"intValue":"66"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"fxjsKYWTWL4=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /foo","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025998722713","endTimeUnixNano":"1735335026004195719","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"net.peer.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"2"}}],"status":{}}]},{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"X5SXMIPajt8=","parentSpanId":"s/6I32KmWeU=","flags":256,"name":"httpUpstream.Do","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025791973471","endTimeUnixNano":"1735335026004164859","attributes":[{"key":"method","value":{"stringValue":"GET"}},{"key":"url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/upstreams/http.go:288"}},{"key":"http.request.resend_count","value":{"intValue":"7"}}],"events":[{"timeUnixNano":"1735335025791981701","name":"creating new http client"},{"timeUnixNano":"1735335025814881765","name":"submitting form","attributes":[{"key":"location","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=bVhDeVhqYXBxRzNLS0F5dm1LVXRzUTZrUHVtSDgrcjdhQSswdmZsT3o2eE81OGJsaEgvOEY4UUVZbFV6em95elFySUNmeFJYelJ4Y0RkRWErUVdtWnc9PXwxNzM1MzM1MDI1fFr-RTRGNqnLn2IMEbk7FiyJdpSLrfG2toEU04qhst7IcWtWqDDKpQf-rvb4AOVFEyb_DCJImrMZ_JKA0yiP0S_U54-vwXFi6BfICA4m3NAFaFycuWP-DbKAQAfk9Uba9ichigqepuu4l8w3EGZwlpOR_H98gkVp8i0wUsRSi0QvJExUdEYP2R6hvWCaEF24kJZGzEgc3K8tFLuKKXF51oZzS8uTXWxZKboeoN_QAE7OJ5s2oD_oUJeQW1mU7W5l1lslB214Zv24ft7fLAbVQYJBd14DDoHfg4WZpQE8v3qIuCCMNcgCod69B1Fa-h4Glz138FEfNbnOgFjiCkW63brqiutwbtIn_Voy034oAAK95qCQvwc0B5RCf2peV7-rb3IrZk6h_ABEeNeRZEkKZtsixVns0ew7rEzfUlcPFre-3D7RK0HfV66arIKuNr4qnMLBuIiDO9PuucWyY4Rft3VAp8Onv-ENIWwLVkrOGvnDFtqubfCnutmW1cqXzF1oVuckiEOwmG0k0vUCk7xR8LLEkNBkh3s3yF3F8LvZLlfesQIl-dNXAWWKIonfCyb_fLiciYxlROetfJXFkMr6LKvgCUHB4AAkx_X--KpHabCpBVnFrEIPv2E59D-eLo7J9Sf4Yhd6nDzdAgj5B4BgDPKSLWvAvjRe5-kD2AT2AGcG-pIBVuKZoI7avNO0GJM23k0-b6_djF13Cg8%3D"}}]},{"timeUnixNano":"1735335025819520690","name":"Retrying on 5xx error","attributes":[{"key":"status","value":{"stringValue":"500 Internal Server Error"}}]},{"timeUnixNano":"1735335025819540450","name":"backing off for 1.122949ms..."},{"timeUnixNano":"1735335025826246421","name":"submitting form","attributes":[{"key":"location","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=TUQrZ3dFSUpycGJ5TFp4c09pdGkvMDB3RWhsUkdLVEZ4Ni9OYmZOMWJuTG5xTlI3OE4vNjdQd0I4cGFSUU1QOUFTWXVqOEs4Z3lMenJhaks4ejRIdVE9PXwxNzM1MzM1MDI1fNlsbQS7GploSTVB0WfdtdH7gi0aLdGkm-Zw-kVVwqJF9rJZSPHGDEYpgyyhScfl32qa-0GvpPpCEPxXfroMBfIx0VSczgLJty1vJ_7-Wvuc_6EdlHR9x544l5p1a7zUqdftP975Xmdcj7tkJwsje3ttJ5uVESkFOy6sN0PAyQihwcYPd0aCvVnLPgVqroEbrbkPcVavCRI8sVE0Awx0X2YzkgBPWzKHmH43iAie-kM0uMZYFPFRtEFxFvD43PscwW-prc3qShMzp4C5cNJt2TRQCRJlXcUr_ccnecAjs3jdys3da_nFHMsbhpDgDfHaYsnL86zq_QVdjWJZFMpIEHOXQUVi2Q7PTm_n9iymHah7FPcWDaAo5BRsKr7fM4_JaHT_BsnrvLMRSV8c61gtzpBoDnIKrrvr7hzua4i1OFkR5XNRSOIy0gh1Re2ZpBFUo0H8miVsqD8Hqx7N1rxFopl3R-spO3a9Arin74L-5gZu1DsgkxVRa9oF5ntJ9ZigAKCnFFRgrmHynjxD4QMxGKR0tFBrlLBYiZFlWpxMYD-bmq6jccxwA8491DQAePeeyWbLIL4-9OIsBoZqIRmJac1gou6EqxJKsECNxxLCCreNKPYcNJKWdY6ZERWT2rVYlzaYowFnfV9KDvQAoaGV_D7rg3Dw7XGnQvo5NHAYGXv5sW7LB-vlPU_5CuVjV2bwmwcy6Ybez22aD1g%3D"}}]},{"timeUnixNano":"1735335025830902106","name":"Retrying on 5xx error","attributes":[{"key":"status","value":{"stringValue":"500 Internal Server Error"}}]},{"timeUnixNano":"1735335025830940047","name":"backing off for 2.414787ms..."},{"timeUnixNano":"1735335025838599218","name":"submitting form","attributes":[{"key":"location","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=djBzMm5teGFzNDQxSlorL2ZYVUN2Nk1XVHAraVFnSjhtU01IRGRlU0czSm8zRUlsM296bjlEc0o4VVhXSHFPOTd3QnlDVEhtSlp1dElXS3ExOWx5dVE9PXwxNzM1MzM1MDI1fPYv0au-VRFGwhaPnlvMz_uEJ22N0pzR8VwFreItQnBEo4C5PRNtMUFyKPuWzxHL67dfMAFNXNaYKrpHwE-RshBnJbE4oxQd6W705zHKAwm2lJsMMMHJkHZmBUk_Tghkuj7AsIlCKQHAG4Dd3NuGBjqd3Kqs9kwin5JnZat8aa2-5KX5Lk4QUFo_sRMebvIARsqK2QubYU114RUM62nzm_HmkKMcvh1QsGQ1s7pr_QepoDs7hSj_oXUs_Tc7c9n97sBGwWXfGc4yKLdodqyubc31qdSmHxcU5I8-FEFbbhMWZ3Pk51N7sMOjoYoP2oupL6d5DSf9bCermdQNAm4UElB1aVY0l_rmg282JhxP_5qBCLAml8xhGHT6JxMHAKA8H6MpNBusSbcfPmfuO_px6ipSnKE9ZsmY4y8_nbMZUaIotKED5NJn9YUlKDgrI58R050HTSbv-zy-rBVpLRwFhk2aC5dhYSyxVc6QxEYIrKvnVyouumm4bvB9xankZB_A1ESpBrAPszKOoznYh1y72LR8P18o7tOvUWxonfsKbnNeZ92RZOOIeUcaJ6-ZYUVO4Os5DrL4QSTByrikq4hdsABQHu89omt_6iS2t_salQ6eVrGQLI9E9yHzoDTVzWsYf8DZ9NbpI9OHMBFB7nJ8DDVm7XXvpgKtRezh2JVu9C3BZHLswq_0dJrE2hH30U1_pXy4mAOjddNCVt4%3D"}}]},{"timeUnixNano":"1735335025842671447","name":"Retrying on 5xx error","attributes":[{"key":"status","value":{"stringValue":"500 Internal Server Error"}}]},{"timeUnixNano":"1735335025842692477","name":"backing off for 3.385134ms..."},{"timeUnixNano":"1735335025851420141","name":"submitting form","attributes":[{"key":"location","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=K05oR213S1ZvdUlZMlh0TDFaRlU3SUlZdmc4OGIxOUFoanBDaXkvNlpZc3ZUeklnc0VQMm1CYjFGYkYrK3ZYdXpnNkNtYS9MZUtleU9DY3NMN0VNUUE9PXwxNzM1MzM1MDI1fF-scOL_CXRp0kDbE4ZjdGwJnPC2MQY02ucYYTtu54IwwWu6QgQgYZ-L1nC0_8TPDlspFKyI_65zC2CAiEPwji6gGkC65hOD0mDcjWFyaMY8NgZJYvackeVgm0tLWY0KPOyJmJEk3oGmXGYlPoPJ_sz8ywot5peE8tV3_7FUgL2k61nEBRmqffg_ICtN21fxrk63NHmQZBw5E6_fj_DSwMT0XeLiu3I2sJSMP8gAXgZ-9KNAU34h9NmBwAqkyCJ5HP7N0cYwc0wCWoZVO8kgI_MrasJvIyvf6UKnp9oBDp8Ro4pTlrukgQgcj1-Wu2x6pCl9xpBT32Ftwl4Vuvr_zI7V6MoqZMKywDBNfsApHUkZTUedX0ZLK9A29RXEVkNqr9OevFVyVsMyOvTZ_tfS7_5hfwzHjyOnHq7q2COA1ocRzWCMaXtIeSk8WwL_K6hCf9vNrpSi0_agf2JWLbefcjy8AUAWjpp6jt4My9JhdRZ-qBmnL973LPUmyiIbYkC0Km1nIe71rItMLqx2SZCwuc8-9jq7DshSiSNzNqMD35PGdiiEawIhMFJwhWRWKCDMPBCYeAkYIODnfJWGBbOrD7kFReqvfO9Jm2zbuaw9tPw6HCHWY8XwIyAougw7FP15F9Vqtzc4imuTmsvkOYJNTvI8lz8Sk2ALwlPo63QbI9DFjlE_uDuGb-HqiUqe0eoIx4EqP9Ni_Qj_AkY%3D"}}]},{"timeUnixNano":"1735335025856282990","name":"Retrying on 5xx error","attributes":[{"key":"status","value":{"stringValue":"500 Internal Server Error"}}]},{"timeUnixNano":"1735335025856302840","name":"backing off for 11.041868ms..."},{"timeUnixNano":"1735335025873572446","name":"submitting form","attributes":[{"key":"location","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=NlpEbWhsdDRlbm1nNEVtSWR0U1BsY3RWU3kyRTNvR09DK3o1NnRLUEtGNCtCNUk5NmE0dUE2N01KM0xkdnk2WGgwTjN1eGQ2cG1rLzdweE4wc1JCbFE9PXwxNzM1MzM1MDI1fNPhIc3ToLHpvS_Gvc8aQRaU88cZYSsnQK8fP27HBXGJKzuRaASTXkCUMwIy2bfmT5GrsBxx8-MmupZH8PE_9xtv96G9GEbRgrrFdukuhrQei_80gU6OgUoTwTtCC9jC8T37myP2Oy72fpFGDoXxzEzRGhSyMAHQJy_HBv3PLX-UxHuResc5NH_Yj7aDx16xDkDAWmc-S1VeeUW0zqT8HgnzfXFYdZqf2_tnV12NUp-CLGXLRsU-3XJIsSwy5CRy1mfIGspxD-SxBHxPaRjKMY9FMxc4CsDmR4eiF28glstJrheqExh9wDhG0TFmgwLtEJ1Vq_hLTmV3RZO8jx8ZtdV1id1slzj8fwzRXH-IVKgTJ1uLcA28LLI1ZzJzD2PrTkfISjg6-5nUJv-OYQcQlQI21pxL1s4ICAa0l3KzE5i5_yeymQaGeAfSLIJ3-5RTTwBMrx5zgMlco96WRrJZ604XqQSOgiGoxSLA9aHu__QOK9OU7GFdga92Y67T3vkKwvWXeX3w2EAF__tzDF5dy--oGborqKGcv0fkKeMgRyfxkxJ-E1w0Z7ppcwL6LL1q78zKI8JcLzgV8MyVDcVnHRrkIToVZ5YYavQnbXK_7lfo_XbKyifs4ZqPmwNY0WyGZ0j_ZZTOLRo8lQR2y5NOlTZRWCz3sqkmOJ3SeAoGrHiQ0tZDfFgvO99ucbtQj1_PCyO6u2-JaIpH6fg%3D"}}]},{"timeUnixNano":"1735335025877583754","name":"Retrying on 5xx error","attributes":[{"key":"status","value":{"stringValue":"500 Internal Server Error"}}]},{"timeUnixNano":"1735335025877603214","name":"backing off for 8.858289ms..."},{"timeUnixNano":"1735335025892996459","name":"submitting form","attributes":[{"key":"location","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=eE9BRU5YQUZpTzFaV0pPK28rTmF0NU1xempEV1NzU3hhM1lDL1I2Zm5KNFRkM0NPd3RQY2wxZDAvVVFJaVB1MTN6enlwa1h1NDFaZmRHZGFIdFQxVlE9PXwxNzM1MzM1MDI1fEw2htjJsKXWR0ubmlecrlxxyWaM5lyrfQKVov8UW14H8_5YI7twjd0qGPWd8uLPpHaUpE5zR5PVk37RnPpkNC2J5NteJv5IN51QbBerrsuGuDZ650Sbqr9-p9FXOKmBBcoKRPW_0N-B7PWMWkQIsY5rVbjYeXgfOjuadl0PNkCA0awKZQhOwfJwr8waX88ypxsRszpU26pXB75zl5tO-AOtrROKkjsv_Y0i-Vvvhj7VGlRPsJ2xvNqK9DuJLbdjmyKrSutz1IsGwhs6NWyd82yaid7x7qCob3VkStVsNeLhjcniP9xjD22Hbi50itMD1HyXCTu_hAVyEYw2SHC842nad-4tP1EEvQkB05PBwTT5G324_lbicp6lFCSzGT9rS2MPEuz12m8Uq9hdJEj18am1pZM7vBXgevH6K0mwlpb4eIgqceOwxbgSHeGw07op-0jdeGnbz1REdKGGui4pWW5PmxcCjDiAG6g6kP6DrXUhPvAVXPK3GhpK4lxuLfU84zuQr81Xo0EmYOxQGjvdcYepBDn9anS1tEghjCk7iptxM6zO11Q7vMFI9wcVHrNl3sSCAM2k21BFmPEyWcnWvPFsQMsegbSznjD2X52WjB7k19H406-bHN-d5Py0dcMo185Y1aAIZ3m2CyXNMgPlrBPT-5XDSvf8113VVpTk8Yazr1rx0DzKlmLhWaUo0Nxm9mPwxK17XRvpdSg%3D"}}]},{"timeUnixNano":"1735335025896949436","name":"Retrying on 5xx error","attributes":[{"key":"status","value":{"stringValue":"500 Internal Server Error"}}]},{"timeUnixNano":"1735335025896969616","name":"backing off for 35.491179ms..."},{"timeUnixNano":"1735335025939610306","name":"submitting form","attributes":[{"key":"location","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=Vld4RWw1Q2pKNkZ2NXNST3pzVTdFMi9qSTNMY0E3bm91bDNYUDNyNWVJbUMrekFzSW5WejIySEtxclJscnBvUkkvVWY1RStubmcrT1g3S1llcklSUWc9PXwxNzM1MzM1MDI1fCZPSA2y7mrm6xOOpuWELdtqzK8AmnUFMhpMijy4u4IE5SwEboZafPLVOuG6XLO-hnn5Xf2lTXraFGd6UpqWGelZT7VfHlpWPhDdf2uCYIAj1u6TYRgPS3_hMilmsO6AvSWmX37-XsPL4I78VhCqbL2FoEKz_CgyCAVhowdooxX6INcvZ7-G8BCJX16jnZKu7o3gX8xFl0mrHYKV1EAPY1V2YsxRTGRS2ptkcqEuxXw9YVOQIIsC6OsphlbN91V6qHGhzBRcLX8CZ-8E_2D9Osgiy4-C-koOCtzRLLFJy9S6oq-8Z1c3PK_wGhTu-syftdKFKjKjdRo97DWgjVz_lnKWeYC2usX32gaExGKkVkMERCaQ6JKoi-TXo4kY1g-AywEfn_gehBqAqzlXQ5VHEDdCth5C2BHjOhgUKcDv7W4Oqmp_93_U28elEz4Q72JUpSaDDe1fKA_eArNUrQBJTJ6N6bABEwL52xkh0ifRwWcERMCwnc1bOlAIDs2T1yWW27hdYrpSOZbAwdKuSji2x1ut0-DqgnGoDMMqzgS98MUPsC9I3PNltfu_DgC959VjH7H6_HmE2q1nPELeZUXRCGzAZTOFE23bD0ks6uyipS3jaVb1qxVClG3VMuSLf9SdV2ESH7MWJaXGyvhN8c1KBsiZp5L-7gqCnsZOMYQrJNQIsbdic-DkeRlmQYbs8iZykp-cADl2uTKek-0%3D"}}]},{"timeUnixNano":"1735335025945633478","name":"Retrying on 5xx error","attributes":[{"key":"status","value":{"stringValue":"500 Internal Server Error"}}]},{"timeUnixNano":"1735335025945650428","name":"backing off for 33.510651ms..."},{"timeUnixNano":"1735335025986804741","name":"submitting form","attributes":[{"key":"location","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=TlVucmJGSVM1TTkxQjdVeFpFTndpblBPMUpqdjBoOTBCTkN1NVBPa1RWWGkzcC9YNE1Td3RYc3IyOHZQS05HSVA5am9EbngyT0pNdzBzdEQ4Kzhrbmc9PXwxNzM1MzM1MDI1fGFr6SjNR6oY3LbAckTb1zOQwi4DF7qp73tBPnxNCM9uAN-E7benf9dM4rTw1MXWLEUpA1XTw-NdsI5DXa5mG-guOz-4MjnubljhndMmKy6H-mdPjrJwqpirLHSjGNIOMh6prOpDlt1i-RnnOB_wtIY1wxUCPntd-aCiBGfViIgB0aCMQq6YcqcJYohYyPYYILz6IczkvpPa5cevuvnCJ20J7X69_6AJ0tYzwo9PVmYrGFyTJULaNfw-2G-nUAFuP4-FHv4PexN-NB_Qn0kM4KsX6482RPpNetBM8hHfHbdlr6J_018NOc7Jpv092cxDmMpwzeTQ7mKtEgntBj8WrClM6VTrkecRpdJ8yiZhLvgVhjK76JUFXlXNzYItTt-NvBKuXZ-869TmVrWADw6579M1IP2uCvQlE88v4e__LelTsipYfTzM_kN_Zl7hoDur57rTfzsbkipm5-szZ1W4wEksgtPybfnauIOJY3AtTQ1u6HSciCbGULA6N024tcnpDAJ2mOwYuYbMHlvVYpTv9AJ7P6WOuxaD9GQP5jDJb_AcmhzylwX6r7TQrHnPtbRfratJUOP85A9-5vgaKdLJ4QIPgVqPh4eyTR6xRWWkFnkJdUnJ8m8gshI0ZZI1KzM-ORzBPl8TrEFfqrHHVq1ujN7JQknxUgbrWQOU-jzncZnCYk6TBMbyFX7SdKMG8rgbLaDdnANGnoVCiuA%3D"}}]}],"status":{"code":"STATUS_CODE_OK"}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, + {"timestamp":"2024-12-27T21:30:29.754253667Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/server.go:85"}},{"key":"service.name","value":{"stringValue":"Control Plane"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"v60d668rsFI=","parentSpanId":"sQT1yEWt9Cs=","flags":256,"name":"envoyconfig.Builder.BuildClusters","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024752937074","endTimeUnixNano":"1735335024753567272","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"1yZo4AZ6R6s=","parentSpanId":"sQT1yEWt9Cs=","flags":256,"name":"envoyconfig.Builder.BuildRouteConfigurations","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024752902344","endTimeUnixNano":"1735335024753657353","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"hwlquCtBU6I=","parentSpanId":"sQT1yEWt9Cs=","flags":256,"name":"envoyconfig.Builder.BuildListeners","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024752986625","endTimeUnixNano":"1735335024756961172","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"sQT1yEWt9Cs=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"controlplane.Server.buildDiscoveryResources","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024752828213","endTimeUnixNano":"1735335024757486459","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/xds.go:23"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"uUk5i2AWBI8=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"controlplane.Server.OnConfigChange","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024757502610","endTimeUnixNano":"1735335024757514710","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/server.go:255"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"m0X4qa9wV2c=","parentSpanId":"ULuo/pZ48II=","flags":256,"name":"envoyconfig.Builder.BuildClusters","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024981645690","endTimeUnixNano":"1735335024981731451","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"G1RZ+p9n3wg=","parentSpanId":"ULuo/pZ48II=","flags":256,"name":"envoyconfig.Builder.BuildRouteConfigurations","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024981640880","endTimeUnixNano":"1735335024981808162","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"lEwSFwmbsuk=","parentSpanId":"ULuo/pZ48II=","flags":256,"name":"envoyconfig.Builder.BuildListeners","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024981666531","endTimeUnixNano":"1735335024982687002","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"ULuo/pZ48II=","parentSpanId":"6TmKHMpFyR8=","flags":256,"name":"controlplane.Server.buildDiscoveryResources","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024981577879","endTimeUnixNano":"1735335024982789743","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/xds.go:23"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"6TmKHMpFyR8=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"controlplane.Server.update","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024979194300","endTimeUnixNano":"1735335024982795253","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/server.go:279"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Sq0l7QhDyGw=","parentSpanId":"Wh6E3dnkilQ=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025810188179","endTimeUnixNano":"1735335025812103302","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"IFI+Jjnxj/k=","parentSpanId":"6gWh1J+1CTs=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025816575605","endTimeUnixNano":"1735335025818844242","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"374SofEc5m0=","parentSpanId":"KCsnd9DyaCk=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025824689532","endTimeUnixNano":"1735335025825267179","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"nMxMJeHK43k=","parentSpanId":"n9+R8Z02ndA=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025827930971","endTimeUnixNano":"1735335025830137867","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"MZXyiOTh9Eg=","parentSpanId":"QAxhOEu2VQU=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025837118560","endTimeUnixNano":"1735335025837672777","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"HAIPnqctQV0=","parentSpanId":"LUc3vGeZwMU=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025840145587","endTimeUnixNano":"1735335025842017740","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"niDmR26iJ04=","parentSpanId":"kFGSZ3gb9bk=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025849973685","endTimeUnixNano":"1735335025850509481","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"SKktgOzuqY8=","parentSpanId":"yp5RxlvZiO8=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025852883819","endTimeUnixNano":"1735335025855193017","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"NrLjbmN+dFg=","parentSpanId":"skvhJjM1/7g=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025872067819","endTimeUnixNano":"1735335025872665856","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"+1faJLPAIcA=","parentSpanId":"KS2iwg6Ulrk=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025875076544","endTimeUnixNano":"1735335025877003927","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"1wVqzZAVYVo=","parentSpanId":"UvKTIVQP9uU=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025891138737","endTimeUnixNano":"1735335025892143109","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"74NUTYEb5Bg=","parentSpanId":"tqKzCJwTic4=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025894506047","endTimeUnixNano":"1735335025896375430","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Lg6KRq4pqLI=","parentSpanId":"KmOo7wve0b8=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025936456229","endTimeUnixNano":"1735335025936996765","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"6ssvTNGwuSI=","parentSpanId":"V/8X7Bvg6MQ=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025943015547","endTimeUnixNano":"1735335025945046211","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"81+d+tfVjk8=","parentSpanId":"b8TIlWwstzc=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025985231592","endTimeUnixNano":"1735335025985897359","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"gxVvt4Qw3j4=","parentSpanId":"NPwFGMaPhF0=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025988379350","endTimeUnixNano":"1735335025992807653","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Sakh5mjAWmc=","parentSpanId":"4uLVZ3lujSQ=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025994883677","endTimeUnixNano":"1735335025996036772","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"c61OVvg7XN8=","parentSpanId":"/Z4/vZfyMIg=","flags":768,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025791087760","endTimeUnixNano":"1735335025791265402","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52360"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"qxEQ86FHF6s=","parentSpanId":"P2zcRg+jsFU=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025801616696","endTimeUnixNano":"1735335025804016115","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52362"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"82uwbqPPsyw=","parentSpanId":"RaoMfsRKjao=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025808896723","endTimeUnixNano":"1735335025809411459","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"+Esu9QHJMcINMk5i34yYkQ==","spanId":"agu8Rnp2lTY=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025812994992","endTimeUnixNano":"1735335025813461367","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"heQjAAl6dfU=","parentSpanId":"tjexoDV7/Hw=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025815769595","endTimeUnixNano":"1735335025816184490","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"7w6Ze9HZfEVHHUuDVdMu6A==","spanId":"YScrBlT6wM8=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025819523040","endTimeUnixNano":"1735335025819673762","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"3Y8oPmEgblM=","parentSpanId":"moqlVoGI3u8=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025821290081","endTimeUnixNano":"1735335025822550546","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52362"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"ahfKuiWUwf4UL01+76Wbzw==","spanId":"kF6eC5N4U6Q=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025823305055","endTimeUnixNano":"1735335025823437187","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52388"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"PiagOqYTpmM=","parentSpanId":"d+7biQunPZg=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025823865962","endTimeUnixNano":"1735335025824300178","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"PI8a76D3GRaQcVMVvKuhxA==","spanId":"aLFJpoBTHWU=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025825930037","endTimeUnixNano":"1735335025826102579","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"KKk10SNGrmc=","parentSpanId":"12NNbneNbUM=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025827104331","endTimeUnixNano":"1735335025827553596","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"hvwtVhD28xiIg6jGUsGaUA==","spanId":"zmCkOSt/uoU=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025830854846","endTimeUnixNano":"1735335025831018477","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"9g8u2zSWvxM=","parentSpanId":"Cw7qX+Mq85E=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025834037494","endTimeUnixNano":"1735335025835353929","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52362"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"+MYYvMbuVt4g7WuhUNPW/g==","spanId":"dz5uWpV6YNA=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025835827455","endTimeUnixNano":"1735335025836006738","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52388"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"86mKxrPrdQQ=","parentSpanId":"aQWRcZQDEL0=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025836369052","endTimeUnixNano":"1735335025836705056","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"TUv6UTv0VXh0kfLsYrzVKQ==","spanId":"4o3f+Kc15eg=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025838285165","endTimeUnixNano":"1735335025838413676","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"VchSVDx2hio=","parentSpanId":"zZCpocublXw=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025839411308","endTimeUnixNano":"1735335025839735711","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mpWzym14/n5SACh89xs85A==","spanId":"nzL1FznBWHc=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025842625817","endTimeUnixNano":"1735335025842773378","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"04nIoKZ7Rn8=","parentSpanId":"pMCRYOIpDEU=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025846889047","endTimeUnixNano":"1735335025848342995","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52362"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ZaPF80+dmj0=","parentSpanId":"ix5QFxi6p84=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025849258506","endTimeUnixNano":"1735335025849554229","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"BmZa5YHnHlYQiR+IrkT47Q==","spanId":"A/QKUn1/j7Y=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025851141728","endTimeUnixNano":"1735335025851285560","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"4k00P8wSf5A=","parentSpanId":"RjOOCHrl7Pg=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025852180531","endTimeUnixNano":"1735335025852530975","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"r24gxKcPFjoUmOrv5DxJhw==","spanId":"Fx9e2eE/2tk=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025856341321","endTimeUnixNano":"1735335025856488572","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"AWRkklChgtM=","parentSpanId":"AxcQF3bUAPA=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025868616247","endTimeUnixNano":"1735335025870334218","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52362"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"FV+HlSQF4YQXvthFn+vPRA==","spanId":"zqQnLUpZJCY=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025870825104","endTimeUnixNano":"1735335025871009936","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52388"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"egVsvBSrdgA=","parentSpanId":"zprk0rw2m8Q=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025871292749","endTimeUnixNano":"1735335025871653633","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"Sxyz3wluzqbxA7WEDH0jmQ==","spanId":"JmqOMx1Ebj0=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025873224052","endTimeUnixNano":"1735335025873363724","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"QdUTNSa6qh0=","parentSpanId":"zsr1/Xcbq2A=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025874360096","endTimeUnixNano":"1735335025874704660","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"vWoVZuYHE7KlLY8HQtSswg==","spanId":"QDkd/nq3ZF0=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025877585554","endTimeUnixNano":"1735335025877748386","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"gaEA+BN+3kI=","parentSpanId":"VlLCvKW20c8=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025887963709","endTimeUnixNano":"1735335025889467206","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52362"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"C+l5gQuBOqvHCrVNu+ScZQ==","spanId":"GqwwAwMakis=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025889952343","endTimeUnixNano":"1735335025890103245","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52388"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"tUL29k86aA0=","parentSpanId":"TnTIhQSeD8s=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025890415708","endTimeUnixNano":"1735335025890740692","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"1ZAKVivbLPa8QTyRp/cohA==","spanId":"GjA6x3sP32c=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025892627675","endTimeUnixNano":"1735335025892774436","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ojzIzoRpdkE=","parentSpanId":"fpe5wrNxals=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025893747717","endTimeUnixNano":"1735335025894140383","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"5qSLUDyjtQlRUAUnOy376Q==","spanId":"MVnTad8KraA=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025896950346","endTimeUnixNano":"1735335025897145708","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"iI6JC+zkuuI=","parentSpanId":"HG6gDV9Mh1I=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025933475133","endTimeUnixNano":"1735335025934864520","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52362"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"jozir3kDgsM=","parentSpanId":"x4/fCcrN2l0=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025935692199","endTimeUnixNano":"1735335025936067784","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"u6fr0Pm2yZsJdFtLxvYY6A==","spanId":"I/1Y/o3LJKE=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025937522891","endTimeUnixNano":"1735335025937665513","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"HSPi+PbWXh0=","parentSpanId":"Ktn4PSE/Eio=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025942261708","endTimeUnixNano":"1735335025942572452","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"qcVMFqgVWF0zPlPHBnnkTw==","spanId":"u4M218qJeLE=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025945601248","endTimeUnixNano":"1735335025945778270","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"vZpvKbrCQqQ=","parentSpanId":"xC/VB6ObsY8=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025980994651","endTimeUnixNano":"1735335025981239534","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52390"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"2yTflfwGhOI=","parentSpanId":"zQwYJxI53tM=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025981996074","endTimeUnixNano":"1735335025982197246","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52390"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"FtTGqe5YvtE=","parentSpanId":"SRBjmOgMIao=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025979676595","endTimeUnixNano":"1735335025983226488","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52362"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"e3JgaULd+gGFz/8A4w5e8g==","spanId":"+HmVp1qzIts=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025983732184","endTimeUnixNano":"1735335025983996588","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52388"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"owAwqwUWHX4=","parentSpanId":"U+zO7kgs2kw=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025984297291","endTimeUnixNano":"1735335025984776537","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"fjpGU6n1puaDxNOl3amsAw==","spanId":"LEmR4prBl5E=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025986530558","endTimeUnixNano":"1735335025986712400","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ayYTEGRLA/4=","parentSpanId":"pqDfjSA5zY8=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025987598210","endTimeUnixNano":"1735335025988006776","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"cifxyFmv4dY=","parentSpanId":"Hh70Fz863sw=","flags":768,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025990548786","endTimeUnixNano":"1735335025990652057","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52402"}},{"key":"rpc.grpc.status_code","value":{"intValue":"5"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"vP37Rm86XmY=","parentSpanId":"JywV84LM8IA=","flags":768,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025991322805","endTimeUnixNano":"1735335025991453716","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52402"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"3ijdqATnWA0=","parentSpanId":"8GtaEDC3sm8=","flags":768,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025992226656","endTimeUnixNano":"1735335025992349167","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52402"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"ubxh4ifDIe2okDP6OtRYFA==","spanId":"ESGoMZpHB88=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025993673843","endTimeUnixNano":"1735335025993818894","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"2cTcRlCzz7A=","parentSpanId":"IwEGClsNoWA=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025994191250","endTimeUnixNano":"1735335025994511013","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"WgzfdQgrifQ=","parentSpanId":"TjF0fvq4Ahs=","flags":768,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025995387233","endTimeUnixNano":"1735335025995508385","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52402"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"BPwmhsyPx+Hs4Aq/WNwlfw==","spanId":"QPU+lU7U14g=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025996637579","endTimeUnixNano":"1735335025996759610","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"jn2jF/UGEwI=","parentSpanId":"g8gyWKtu1KQ=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025997215675","endTimeUnixNano":"1735335025997552799","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52362"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"wBTyiupIuPiO4BLtsY0qUA==","spanId":"D+teV707Wvg=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025998741624","endTimeUnixNano":"1735335025998851195","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52388"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"AWyn5ABOBS0=","parentSpanId":"qo+2ljnLo3s=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025999800756","endTimeUnixNano":"1735335025999986049","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52390"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"QBjGZUW54F0=","parentSpanId":"niGr/qWcz4g=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335026000647497","endTimeUnixNano":"1735335026000798758","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52390"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"yHJAjGq7Cgg=","parentSpanId":"yo8tJLvB7F8=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335026001480356","endTimeUnixNano":"1735335026001645488","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52390"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"DC1wSzzAIIQ=","parentSpanId":"GZGIXC4RfsY=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335026002212135","endTimeUnixNano":"1735335026002391097","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52390"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"kKNQsnG+JKs=","parentSpanId":"Vh7x9OYP6sg=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025999199329","endTimeUnixNano":"1735335026003181657","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52362"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"3JQRBybzzOfTYzQQxN38pg==","spanId":"U1wYLMi29Kw=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335026004202599","endTimeUnixNano":"1735335026004326451","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52388"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"CGVc3FYZu4Q=","parentSpanId":"TGyFsg9jcEc=","flags":768,"name":"registry.Registry/Report","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335026622400462","endTimeUnixNano":"1735335026622594285","attributes":[{"key":"rpc.service","value":{"stringValue":"registry.Registry"}},{"key":"rpc.method","value":{"stringValue":"Report"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52414"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp","version":"0.57.0"},"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Wh6E3dnkilQ=","parentSpanId":"dq09RB/npOA=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025810161779","endTimeUnixNano":"1735335025812111242","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1109"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"6gWh1J+1CTs=","parentSpanId":"dMDfBRWWNfg=","flags":768,"name":"Server: GET /oauth2/callback","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025816557095","endTimeUnixNano":"1735335025818859832","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/oauth2/callback"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1286"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"KCsnd9DyaCk=","parentSpanId":"K5Za89EMcFE=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025824673852","endTimeUnixNano":"1735335025825279409","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1109"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"n9+R8Z02ndA=","parentSpanId":"ETof3b34RNg=","flags":768,"name":"Server: GET /oauth2/callback","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025827892501","endTimeUnixNano":"1735335025830155978","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/oauth2/callback"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1286"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"QAxhOEu2VQU=","parentSpanId":"JMTzABiGm6s=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025837101150","endTimeUnixNano":"1735335025837680307","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1109"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"LUc3vGeZwMU=","parentSpanId":"rLKtQLC36X0=","flags":768,"name":"Server: GET /oauth2/callback","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025840115597","endTimeUnixNano":"1735335025842033750","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/oauth2/callback"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1286"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"kFGSZ3gb9bk=","parentSpanId":"UBrFUNOcDBc=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025849955795","endTimeUnixNano":"1735335025850520191","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1109"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"yp5RxlvZiO8=","parentSpanId":"B7oHC/7Ojus=","flags":768,"name":"Server: GET /oauth2/callback","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025852864029","endTimeUnixNano":"1735335025855209937","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/oauth2/callback"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1286"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"skvhJjM1/7g=","parentSpanId":"fOxEGb9Booo=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025872050999","endTimeUnixNano":"1735335025872673316","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1109"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"KS2iwg6Ulrk=","parentSpanId":"rid7A7oKsBM=","flags":768,"name":"Server: GET /oauth2/callback","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025875060084","endTimeUnixNano":"1735335025877018588","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/oauth2/callback"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1286"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"UvKTIVQP9uU=","parentSpanId":"jh13uEWyq54=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025891112586","endTimeUnixNano":"1735335025892150419","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1109"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"tqKzCJwTic4=","parentSpanId":"zaz8bXwr3x0=","flags":768,"name":"Server: GET /oauth2/callback","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025894483737","endTimeUnixNano":"1735335025896388580","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/oauth2/callback"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1286"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"KmOo7wve0b8=","parentSpanId":"8j7gcdiyUN4=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025936439719","endTimeUnixNano":"1735335025937002515","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1109"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"V/8X7Bvg6MQ=","parentSpanId":"PbLOU7xbMgE=","flags":768,"name":"Server: GET /oauth2/callback","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025942995277","endTimeUnixNano":"1735335025945061772","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/oauth2/callback"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1286"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"b8TIlWwstzc=","parentSpanId":"C27ROXjc3Mg=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025985214352","endTimeUnixNano":"1735335025985923170","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1109"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"NPwFGMaPhF0=","parentSpanId":"rn0P4iWyrGg=","flags":768,"name":"Server: GET /oauth2/callback","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025988363160","endTimeUnixNano":"1735335025992825383","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/oauth2/callback"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"545"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"4uLVZ3lujSQ=","parentSpanId":"HLHEc+bHits=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025994865237","endTimeUnixNano":"1735335025996042312","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1334"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"rGSqePbCvq8=","parentSpanId":"c4K33/UOGkY=","flags":768,"name":"Server: GET /","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025998120747","endTimeUnixNano":"1735335025998238578","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34938"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/callback/"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"66"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, + {"timestamp":"2024-12-27T21:30:29.961108701Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/authenticate.go:54"}},{"key":"service.name","value":{"stringValue":"Authenticate"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp","version":"0.57.0"},"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"68TkOnpSKwY=","parentSpanId":"Sq0l7QhDyGw=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025810351341","endTimeUnixNano":"1735335025811858969","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"7uQdjOjS7cI=","parentSpanId":"2OM054Y6ekI=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025816661726","endTimeUnixNano":"1735335025816945089","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"PGcngHRgqCX4iaoO2fEC5g==","spanId":"ZsmRwKi3Ki4=","flags":256,"name":"OAuth2 Client: GET /.well-known/jwks.json","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025817551966","endTimeUnixNano":"1735335025817837991","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/jwks.json"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"237"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"uMSJ2tNZc4o=","parentSpanId":"374SofEc5m0=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025824809913","endTimeUnixNano":"1735335025825054246","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"d/S+GCMxW/g=","parentSpanId":"keERcjTx5f8=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025828015602","endTimeUnixNano":"1735335025828350076","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"Cm2eM1rGKwOPUvNrZ3JnIQ==","spanId":"2vnGWHtaRpY=","flags":256,"name":"OAuth2 Client: GET /.well-known/jwks.json","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025828864322","endTimeUnixNano":"1735335025829104825","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/jwks.json"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"237"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"STVwwUHTbwU=","parentSpanId":"MZXyiOTh9Eg=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025837271002","endTimeUnixNano":"1735335025837536305","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"rhwZXUEWDec=","parentSpanId":"+q/dL49IE64=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025840238718","endTimeUnixNano":"1735335025840538652","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"Ng1V9c7eduVtG8yBTNSUdw==","spanId":"uAH2dcySq2g=","flags":256,"name":"OAuth2 Client: GET /.well-known/jwks.json","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025841008297","endTimeUnixNano":"1735335025841249770","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/jwks.json"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"237"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"KVfrvuRAMxg=","parentSpanId":"niDmR26iJ04=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025850109416","endTimeUnixNano":"1735335025850425100","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Ua0wEfIKbXg=","parentSpanId":"Gt2TeC1aqPg=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025853004610","endTimeUnixNano":"1735335025853297644","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"1F2INkD1DM1Gd2zZRmjgeQ==","spanId":"TOYv2v4FgWY=","flags":256,"name":"OAuth2 Client: GET /.well-known/jwks.json","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025853745289","endTimeUnixNano":"1735335025853992803","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/jwks.json"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"237"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"FBXEc0lb4OY=","parentSpanId":"NrLjbmN+dFg=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025872225141","endTimeUnixNano":"1735335025872507544","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"u5Nt8T6Ejo0=","parentSpanId":"DIniBfO6UmU=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025875180866","endTimeUnixNano":"1735335025875464629","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"3QpZE6RgWdOi5jhjakP3iw==","spanId":"ieD2I+EcHco=","flags":256,"name":"OAuth2 Client: GET /.well-known/jwks.json","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025875966256","endTimeUnixNano":"1735335025876204678","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/jwks.json"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"237"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Vb1M0/gTV54=","parentSpanId":"1wVqzZAVYVo=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025891274498","endTimeUnixNano":"1735335025891560921","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"E3uSnGr1nuo=","parentSpanId":"OrWTQ/P6cUg=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025894580388","endTimeUnixNano":"1735335025894879101","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"PgqwQu60TK/M1HnlytJ0OA==","spanId":"bO5cO4l7/ZY=","flags":256,"name":"OAuth2 Client: GET /.well-known/jwks.json","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025895363437","endTimeUnixNano":"1735335025895629050","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/jwks.json"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"237"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"s5LlPWRVkt0=","parentSpanId":"Lg6KRq4pqLI=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025936568330","endTimeUnixNano":"1735335025936827293","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"MN31qbGJoAI=","parentSpanId":"Dtl5Ryn+L04=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025943121088","endTimeUnixNano":"1735335025943398532","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"Taxuplp6DB+VUFbngmAAPw==","spanId":"75tyAErilUw=","flags":256,"name":"OAuth2 Client: GET /.well-known/jwks.json","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025943941569","endTimeUnixNano":"1735335025944189992","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/jwks.json"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"237"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"11tLsq+ze/k=","parentSpanId":"81+d+tfVjk8=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025985356943","endTimeUnixNano":"1735335025985697997","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"EqhoNIgJKpM=","parentSpanId":"JmRWK9vBQms=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025988459931","endTimeUnixNano":"1735335025988721994","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"u+VskjE7NJ5A/WdVWZgX0g==","spanId":"eNxd/O1kD1M=","flags":256,"name":"OAuth2 Client: GET /.well-known/jwks.json","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025989222180","endTimeUnixNano":"1735335025989419262","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/jwks.json"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"237"}}],"status":{}}]},{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"fwpT5InD5fo=","parentSpanId":"Sq0l7QhDyGw=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025810285030","endTimeUnixNano":"1735335025812091962","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"2OM054Y6ekI=","parentSpanId":"IFI+Jjnxj/k=","flags":256,"name":"authenticate.getOAuthCallback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025816621876","endTimeUnixNano":"1735335025818676870","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:327"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"rfYpAPj+Z5Y=","parentSpanId":"374SofEc5m0=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025824756963","endTimeUnixNano":"1735335025825255419","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"keERcjTx5f8=","parentSpanId":"nMxMJeHK43k=","flags":256,"name":"authenticate.getOAuthCallback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025827976542","endTimeUnixNano":"1735335025829830903","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:327"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"mLi3S4al7/Q=","parentSpanId":"MZXyiOTh9Eg=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025837219552","endTimeUnixNano":"1735335025837669477","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"+q/dL49IE64=","parentSpanId":"HAIPnqctQV0=","flags":256,"name":"authenticate.getOAuthCallback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025840195818","endTimeUnixNano":"1735335025841862858","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:327"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"TXUsos5kBAU=","parentSpanId":"niDmR26iJ04=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025850054386","endTimeUnixNano":"1735335025850506761","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Gt2TeC1aqPg=","parentSpanId":"SKktgOzuqY8=","flags":256,"name":"authenticate.getOAuthCallback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025852956420","endTimeUnixNano":"1735335025854634200","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:327"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"B/8GRqmLiD0=","parentSpanId":"NrLjbmN+dFg=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025872170720","endTimeUnixNano":"1735335025872662356","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"DIniBfO6UmU=","parentSpanId":"+1faJLPAIcA=","flags":256,"name":"authenticate.getOAuthCallback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025875137845","endTimeUnixNano":"1735335025876866526","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:327"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"qM9apFAszkU=","parentSpanId":"1wVqzZAVYVo=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025891216407","endTimeUnixNano":"1735335025892125099","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"OrWTQ/P6cUg=","parentSpanId":"74NUTYEb5Bg=","flags":256,"name":"authenticate.getOAuthCallback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025894542748","endTimeUnixNano":"1735335025896299149","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:327"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"v29TkK6tEd8=","parentSpanId":"Lg6KRq4pqLI=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025936521940","endTimeUnixNano":"1735335025936988485","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Dtl5Ryn+L04=","parentSpanId":"6ssvTNGwuSI=","flags":256,"name":"authenticate.getOAuthCallback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025943072718","endTimeUnixNano":"1735335025944817659","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:327"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"2ZQdrlp4PQA=","parentSpanId":"81+d+tfVjk8=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025985306413","endTimeUnixNano":"1735335025985888449","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"JmRWK9vBQms=","parentSpanId":"gxVvt4Qw3j4=","flags":256,"name":"authenticate.getOAuthCallback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025988424971","endTimeUnixNano":"1735335025992705111","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:327"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"/vivTqGooJY=","parentSpanId":"P8FQuKylCa4=","flags":256,"name":"authenticate.SignIn","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025995755978","endTimeUnixNano":"1735335025996025632","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:162"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"P8FQuKylCa4=","parentSpanId":"Sakh5mjAWmc=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025995020979","endTimeUnixNano":"1735335025996034412","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"TBBw9HlazRs=","parentSpanId":"2OM054Y6ekI=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025818473748","endTimeUnixNano":"1735335025818497298","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"S28y0oRNdMQ=","parentSpanId":"2OM054Y6ekI=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025818637590","endTimeUnixNano":"1735335025818656600","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"RAQ9e4c2I1o=","parentSpanId":"keERcjTx5f8=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025829704651","endTimeUnixNano":"1735335025829733222","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"YoI+tVmeBU8=","parentSpanId":"keERcjTx5f8=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025829802983","endTimeUnixNano":"1735335025829819013","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"qpHhwhJ2jAg=","parentSpanId":"+q/dL49IE64=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025841747185","endTimeUnixNano":"1735335025841773156","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"TUjoaJC9HOc=","parentSpanId":"+q/dL49IE64=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025841839637","endTimeUnixNano":"1735335025841853178","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"hyl5oOC3YzM=","parentSpanId":"Gt2TeC1aqPg=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025854524469","endTimeUnixNano":"1735335025854547439","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"TMGWDzqOHlQ=","parentSpanId":"Gt2TeC1aqPg=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025854606140","endTimeUnixNano":"1735335025854625060","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ZU5kNBztLjY=","parentSpanId":"DIniBfO6UmU=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025876733454","endTimeUnixNano":"1735335025876758805","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"k7H+ovBso5U=","parentSpanId":"DIniBfO6UmU=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025876834766","endTimeUnixNano":"1735335025876854736","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"PSwEM35MNTk=","parentSpanId":"OrWTQ/P6cUg=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025896197728","endTimeUnixNano":"1735335025896218918","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"oOGdHZ1iuAA=","parentSpanId":"OrWTQ/P6cUg=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025896277748","endTimeUnixNano":"1735335025896290689","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"oXhIZii2q6U=","parentSpanId":"Dtl5Ryn+L04=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025944684407","endTimeUnixNano":"1735335025944716418","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"6qc4irrqh+0=","parentSpanId":"Dtl5Ryn+L04=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025944789678","endTimeUnixNano":"1735335025944804859","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Hh70Fz863sw=","parentSpanId":"JmRWK9vBQms=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025989899467","endTimeUnixNano":"1735335025990944830","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"5"}}],"status":{"message":"record not found","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"JywV84LM8IA=","parentSpanId":"JmRWK9vBQms=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025991011171","endTimeUnixNano":"1735335025991662729","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"8GtaEDC3sm8=","parentSpanId":"JmRWK9vBQms=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025991930462","endTimeUnixNano":"1735335025992593130","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"TjF0fvq4Ahs=","parentSpanId":"P8FQuKylCa4=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025995100900","endTimeUnixNano":"1735335025995725367","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, + {"timestamp":"2024-12-27T21:30:29.966934441Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/authorize/authorize.go:48"}},{"key":"service.name","value":{"stringValue":"Authorize"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"+QZw+I9tpmY=","parentSpanId":"9zbnaSilf64=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025802086372","endTimeUnixNano":"1735335025802275004","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"K5lGkBkLfGA=","parentSpanId":"5PoR/9GtyZ8=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025802546737","endTimeUnixNano":"1735335025802838961","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"WVzMc/GgbKk=","parentSpanId":"5PoR/9GtyZ8=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025802869671","endTimeUnixNano":"1735335025803008213","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Ht0evUW5Oi8=","parentSpanId":"5PoR/9GtyZ8=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025803026513","endTimeUnixNano":"1735335025803153644","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"5PoR/9GtyZ8=","parentSpanId":"9zbnaSilf64=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025802107732","endTimeUnixNano":"1735335025803553979","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"9zbnaSilf64=","parentSpanId":"CfdXOWhYtlc=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025802041861","endTimeUnixNano":"1735335025803563059","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"rDn9JOsOTU4=","parentSpanId":"CfdXOWhYtlc=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025803784751","endTimeUnixNano":"1735335025803813032","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"CfdXOWhYtlc=","parentSpanId":"qxEQ86FHF6s=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025802002621","endTimeUnixNano":"1735335025803821102","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ZYwpZQe8Ypc=","parentSpanId":"AavMnl5RXEU=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025809118426","endTimeUnixNano":"1735335025809252097","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"AavMnl5RXEU=","parentSpanId":"gsWnxw7YYZ4=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025809084905","endTimeUnixNano":"1735335025809265737","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"P7rFWZXQDtU=","parentSpanId":"gsWnxw7YYZ4=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025809271637","endTimeUnixNano":"1735335025809305098","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"gsWnxw7YYZ4=","parentSpanId":"82uwbqPPsyw=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025809017044","endTimeUnixNano":"1735335025809311668","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"uS0c/bpC1QE=","parentSpanId":"89ZZ3/tdVtE=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025815968698","endTimeUnixNano":"1735335025816070779","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"89ZZ3/tdVtE=","parentSpanId":"vMeSVkU8tvE=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025815939228","endTimeUnixNano":"1735335025816085939","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"sMMZrzVKBwo=","parentSpanId":"vMeSVkU8tvE=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025816091679","endTimeUnixNano":"1735335025816137810","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"vMeSVkU8tvE=","parentSpanId":"heQjAAl6dfU=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025815832295","endTimeUnixNano":"1735335025816145190","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"IsHgfMSwOFc=","parentSpanId":"BjvlEOqqoYQ=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025821396242","endTimeUnixNano":"1735335025821479763","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"trf8MoiBLtw=","parentSpanId":"6j75o2UhjsM=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025821599005","endTimeUnixNano":"1735335025821762686","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"l0zVW4Qbs5Q=","parentSpanId":"6j75o2UhjsM=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025821775127","endTimeUnixNano":"1735335025821865809","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"uKeJNfsaBgU=","parentSpanId":"6j75o2UhjsM=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025821877069","endTimeUnixNano":"1735335025821985160","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"6j75o2UhjsM=","parentSpanId":"BjvlEOqqoYQ=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025821413642","endTimeUnixNano":"1735335025822306794","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"BjvlEOqqoYQ=","parentSpanId":"mF2wVQWgFZ0=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025821370152","endTimeUnixNano":"1735335025822315724","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"WB1ZRIGSTkY=","parentSpanId":"mF2wVQWgFZ0=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025822479446","endTimeUnixNano":"1735335025822512356","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"mF2wVQWgFZ0=","parentSpanId":"3Y8oPmEgblM=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025821345842","endTimeUnixNano":"1735335025822520676","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"en/si12hPAA=","parentSpanId":"955DafiANbI=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025824068025","endTimeUnixNano":"1735335025824176806","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"955DafiANbI=","parentSpanId":"xhQqyPAaa3s=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025824035004","endTimeUnixNano":"1735335025824191396","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"JgVQu1FL8Xg=","parentSpanId":"xhQqyPAaa3s=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025824196046","endTimeUnixNano":"1735335025824227327","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"xhQqyPAaa3s=","parentSpanId":"PiagOqYTpmM=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025823978244","endTimeUnixNano":"1735335025824229907","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"dQH9gToiFwg=","parentSpanId":"LbjnoYA5zpo=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025827339963","endTimeUnixNano":"1735335025827462485","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"LbjnoYA5zpo=","parentSpanId":"WyPW5d+ni+k=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025827307573","endTimeUnixNano":"1735335025827476625","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"BO5fc2gjmE4=","parentSpanId":"WyPW5d+ni+k=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025827482245","endTimeUnixNano":"1735335025827510355","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"WyPW5d+ni+k=","parentSpanId":"KKk10SNGrmc=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025827222892","endTimeUnixNano":"1735335025827517415","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"WCM+K0l/Fh8=","parentSpanId":"JNt2DRr7/Bg=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025834189906","endTimeUnixNano":"1735335025834295387","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"p2l78TY+CMQ=","parentSpanId":"nGIqcHMvUqs=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025834402658","endTimeUnixNano":"1735335025834580890","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"CrmdWPLP2Ss=","parentSpanId":"nGIqcHMvUqs=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025834594621","endTimeUnixNano":"1735335025834742182","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ZtLWldKyeFY=","parentSpanId":"nGIqcHMvUqs=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025834753742","endTimeUnixNano":"1735335025834837043","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"nGIqcHMvUqs=","parentSpanId":"JNt2DRr7/Bg=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025834221616","endTimeUnixNano":"1735335025835139387","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"JNt2DRr7/Bg=","parentSpanId":"D2FLhaWG8No=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025834158286","endTimeUnixNano":"1735335025835148707","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"pjk3dOwVGKU=","parentSpanId":"D2FLhaWG8No=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025835289348","endTimeUnixNano":"1735335025835314379","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"D2FLhaWG8No=","parentSpanId":"9g8u2zSWvxM=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025834117995","endTimeUnixNano":"1735335025835322209","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"1bOsLY65o2g=","parentSpanId":"QSVgmuuQlyk=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025836527134","endTimeUnixNano":"1735335025836617435","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"QSVgmuuQlyk=","parentSpanId":"hf9k8TOUFdQ=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025836499553","endTimeUnixNano":"1735335025836631825","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"w9ssqcECh8E=","parentSpanId":"hf9k8TOUFdQ=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025836637165","endTimeUnixNano":"1735335025836663395","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"hf9k8TOUFdQ=","parentSpanId":"86mKxrPrdQQ=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025836444483","endTimeUnixNano":"1735335025836665285","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"m3ofV4AgdzU=","parentSpanId":"zd/6MXSPyNg=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025839567369","endTimeUnixNano":"1735335025839665131","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"zd/6MXSPyNg=","parentSpanId":"gpBYkLZrl4c=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025839541359","endTimeUnixNano":"1735335025839679011","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"IHKu0mcP9wE=","parentSpanId":"gpBYkLZrl4c=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025839683671","endTimeUnixNano":"1735335025839707591","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"gpBYkLZrl4c=","parentSpanId":"VchSVDx2hio=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025839471088","endTimeUnixNano":"1735335025839709111","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"hlJITdy5GbM=","parentSpanId":"nmW6dmDbCZ0=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025847066409","endTimeUnixNano":"1735335025847203041","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"YKX2PFK9kUo=","parentSpanId":"RTL/QJCpqKo=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025847364843","endTimeUnixNano":"1735335025847548445","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"2tmsOuk2trk=","parentSpanId":"RTL/QJCpqKo=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025847564305","endTimeUnixNano":"1735335025847650146","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"q4oGWBE7P2o=","parentSpanId":"RTL/QJCpqKo=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025847661786","endTimeUnixNano":"1735335025847729647","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"RTL/QJCpqKo=","parentSpanId":"nmW6dmDbCZ0=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025847087100","endTimeUnixNano":"1735335025848095482","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"nmW6dmDbCZ0=","parentSpanId":"bjGYD26UMW4=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025847024189","endTimeUnixNano":"1735335025848104842","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Z2cxpjgDXKo=","parentSpanId":"bjGYD26UMW4=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025848274294","endTimeUnixNano":"1735335025848301155","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"bjGYD26UMW4=","parentSpanId":"04nIoKZ7Rn8=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025846988918","endTimeUnixNano":"1735335025848309595","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"RhqhM2Lw8Cw=","parentSpanId":"Jpo9gKiZZbA=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025849399137","endTimeUnixNano":"1735335025849478718","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Jpo9gKiZZbA=","parentSpanId":"AuMbefBUznA=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025849372607","endTimeUnixNano":"1735335025849492258","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Nf4cA//0tWc=","parentSpanId":"AuMbefBUznA=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025849496388","endTimeUnixNano":"1735335025849519459","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"AuMbefBUznA=","parentSpanId":"ZaPF80+dmj0=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025849324586","endTimeUnixNano":"1735335025849520989","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"CU1kvqJd8pY=","parentSpanId":"qv941ZWIXdc=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025852347363","endTimeUnixNano":"1735335025852451264","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"qv941ZWIXdc=","parentSpanId":"AkCsHLC/QWw=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025852318633","endTimeUnixNano":"1735335025852466974","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"M8M06S5Ax0U=","parentSpanId":"AkCsHLC/QWw=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025852471954","endTimeUnixNano":"1735335025852501705","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"AkCsHLC/QWw=","parentSpanId":"4k00P8wSf5A=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025852241682","endTimeUnixNano":"1735335025852503465","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Esv4dxLKSzg=","parentSpanId":"KTbmH0YYMbs=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025868750089","endTimeUnixNano":"1735335025868853830","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"iPrUuthF+So=","parentSpanId":"r65t/8B85Eo=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025869072113","endTimeUnixNano":"1735335025869308355","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"15r71FlGjP0=","parentSpanId":"r65t/8B85Eo=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025869326245","endTimeUnixNano":"1735335025869420507","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ZFDnmObzkoY=","parentSpanId":"r65t/8B85Eo=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025869433287","endTimeUnixNano":"1735335025869522318","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"r65t/8B85Eo=","parentSpanId":"KTbmH0YYMbs=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025868781859","endTimeUnixNano":"1735335025870036635","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"KTbmH0YYMbs=","parentSpanId":"lpDozXJHt3I=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025868718519","endTimeUnixNano":"1735335025870047135","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"dGyTLkjazfI=","parentSpanId":"lpDozXJHt3I=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025870231497","endTimeUnixNano":"1735335025870272947","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"lpDozXJHt3I=","parentSpanId":"AWRkklChgtM=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025868684858","endTimeUnixNano":"1735335025870289748","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ukceDzC3mkU=","parentSpanId":"kzEZGsLQ6Kk=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025871459001","endTimeUnixNano":"1735335025871560392","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"kzEZGsLQ6Kk=","parentSpanId":"JBX+Bty4YDA=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025871436931","endTimeUnixNano":"1735335025871576662","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Dqi6tJi5ct4=","parentSpanId":"JBX+Bty4YDA=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025871581612","endTimeUnixNano":"1735335025871611953","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"JBX+Bty4YDA=","parentSpanId":"egVsvBSrdgA=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025871373260","endTimeUnixNano":"1735335025871613623","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Hbe4hwbmNH8=","parentSpanId":"N3lmTyy+O4Q=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025874539408","endTimeUnixNano":"1735335025874617299","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"N3lmTyy+O4Q=","parentSpanId":"3Nk2CzroXhE=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025874497358","endTimeUnixNano":"1735335025874630559","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"V9Iz75CIWkI=","parentSpanId":"3Nk2CzroXhE=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025874641509","endTimeUnixNano":"1735335025874666730","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"3Nk2CzroXhE=","parentSpanId":"QdUTNSa6qh0=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025874420637","endTimeUnixNano":"1735335025874672780","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"baJUDiZjW24=","parentSpanId":"1/N7WxCrg+A=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025888077280","endTimeUnixNano":"1735335025888180602","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"PdGcA51RQic=","parentSpanId":"feitCe313qQ=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025888339613","endTimeUnixNano":"1735335025888562636","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"UovJyUrDUj8=","parentSpanId":"feitCe313qQ=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025888581186","endTimeUnixNano":"1735335025888663237","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"GkjNF0cAMyw=","parentSpanId":"feitCe313qQ=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025888674147","endTimeUnixNano":"1735335025888761718","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"feitCe313qQ=","parentSpanId":"1/N7WxCrg+A=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025888088331","endTimeUnixNano":"1735335025889199503","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"1/N7WxCrg+A=","parentSpanId":"dVGOYu2GHuQ=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025888050010","endTimeUnixNano":"1735335025889209203","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"qZB8lZItX/8=","parentSpanId":"dVGOYu2GHuQ=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025889399106","endTimeUnixNano":"1735335025889425796","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"dVGOYu2GHuQ=","parentSpanId":"gaEA+BN+3kI=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025888027330","endTimeUnixNano":"1735335025889434086","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"X6t41m2jxuQ=","parentSpanId":"UQgm8R8CPu4=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025890572690","endTimeUnixNano":"1735335025890655181","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"UQgm8R8CPu4=","parentSpanId":"6j4JzApql/8=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025890531350","endTimeUnixNano":"1735335025890668221","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"mYGGhy9JPXM=","parentSpanId":"6j4JzApql/8=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025890678251","endTimeUnixNano":"1735335025890701492","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"6j4JzApql/8=","parentSpanId":"tUL29k86aA0=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025890483229","endTimeUnixNano":"1735335025890707882","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"0p9gZYL0WMw=","parentSpanId":"Ak/AhdqIt1Q=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025893972201","endTimeUnixNano":"1735335025894060642","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Ak/AhdqIt1Q=","parentSpanId":"mM6hh4zyNaY=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025893943961","endTimeUnixNano":"1735335025894074652","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"2Pjk6OMh7QY=","parentSpanId":"mM6hh4zyNaY=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025894079212","endTimeUnixNano":"1735335025894103792","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"mM6hh4zyNaY=","parentSpanId":"ojzIzoRpdkE=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025893823268","endTimeUnixNano":"1735335025894105402","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Ph5DuK/hPIs=","parentSpanId":"VDjrSWqMTyo=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025933615094","endTimeUnixNano":"1735335025933712166","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Jk23ZxyiZAI=","parentSpanId":"Pv/OUxSoLt0=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025933842257","endTimeUnixNano":"1735335025934053800","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"22nudVC7nbI=","parentSpanId":"Pv/OUxSoLt0=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025934070661","endTimeUnixNano":"1735335025934172422","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"tE5jOpi7xag=","parentSpanId":"Pv/OUxSoLt0=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025934185102","endTimeUnixNano":"1735335025934273633","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Pv/OUxSoLt0=","parentSpanId":"VDjrSWqMTyo=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025933641165","endTimeUnixNano":"1735335025934632597","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"VDjrSWqMTyo=","parentSpanId":"4xChUK+lUQA=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025933581614","endTimeUnixNano":"1735335025934642447","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"LeP3Wa8uaUQ=","parentSpanId":"4xChUK+lUQA=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025934788779","endTimeUnixNano":"1735335025934814629","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"4xChUK+lUQA=","parentSpanId":"iI6JC+zkuuI=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025933548594","endTimeUnixNano":"1735335025934830159","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"UQmJPJkHcto=","parentSpanId":"ooJJdyUu9E0=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025935853461","endTimeUnixNano":"1735335025935972563","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ooJJdyUu9E0=","parentSpanId":"jDMcjJG1E0g=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025935821601","endTimeUnixNano":"1735335025935985504","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"tRutovykdVQ=","parentSpanId":"jDMcjJG1E0g=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025935990094","endTimeUnixNano":"1735335025936014614","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"jDMcjJG1E0g=","parentSpanId":"jozir3kDgsM=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025935760090","endTimeUnixNano":"1735335025936016204","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Xx9BJ7LoYww=","parentSpanId":"YSKgfHDwYVU=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025942413670","endTimeUnixNano":"1735335025942497731","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"YSKgfHDwYVU=","parentSpanId":"wytgul5s1Bw=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025942395640","endTimeUnixNano":"1735335025942511661","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"8fSOqkwi94o=","parentSpanId":"wytgul5s1Bw=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025942515701","endTimeUnixNano":"1735335025942543032","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"wytgul5s1Bw=","parentSpanId":"HSPi+PbWXh0=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025942324049","endTimeUnixNano":"1735335025942544302","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"PlKDELGMRME=","parentSpanId":"FDNGCAVYL6s=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025979828997","endTimeUnixNano":"1735335025979952199","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"A5Eu+VH/j/k=","parentSpanId":"zqN5at4EmC8=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025980145902","endTimeUnixNano":"1735335025981619709","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Hr6dVe1Xl7k=","parentSpanId":"zqN5at4EmC8=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025981641269","endTimeUnixNano":"1735335025982429099","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"oaUyaHxddFI=","parentSpanId":"zqN5at4EmC8=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025982455469","endTimeUnixNano":"1735335025982481589","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"zqN5at4EmC8=","parentSpanId":"FDNGCAVYL6s=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025979848637","endTimeUnixNano":"1735335025982943405","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"FDNGCAVYL6s=","parentSpanId":"oOGatcJRz4o=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025979790987","endTimeUnixNano":"1735335025982953885","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"/7q/6uvS7PQ=","parentSpanId":"oOGatcJRz4o=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025983155277","endTimeUnixNano":"1735335025983184887","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"oOGatcJRz4o=","parentSpanId":"FtTGqe5YvtE=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025979755566","endTimeUnixNano":"1735335025983192618","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ddsiV4yPi00=","parentSpanId":"EVCUlNnGnZA=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025984529224","endTimeUnixNano":"1735335025984663775","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"EVCUlNnGnZA=","parentSpanId":"CaypUioEVro=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025984489183","endTimeUnixNano":"1735335025984678796","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"j4auuJdL6Fc=","parentSpanId":"CaypUioEVro=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025984685216","endTimeUnixNano":"1735335025984723086","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"CaypUioEVro=","parentSpanId":"owAwqwUWHX4=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025984410812","endTimeUnixNano":"1735335025984729996","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"C3GISENbQC4=","parentSpanId":"nKI+9NglJjs=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025987822973","endTimeUnixNano":"1735335025987930624","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"nKI+9NglJjs=","parentSpanId":"WYB/uQwJqxw=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025987793652","endTimeUnixNano":"1735335025987943574","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"9RtAyEpXyF8=","parentSpanId":"WYB/uQwJqxw=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025987948154","endTimeUnixNano":"1735335025987974775","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"WYB/uQwJqxw=","parentSpanId":"ayYTEGRLA/4=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025987708481","endTimeUnixNano":"1735335025987977925","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"AWsYWu+Xc0I=","parentSpanId":"Ze5RueXjySs=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025994341641","endTimeUnixNano":"1735335025994428252","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Ze5RueXjySs=","parentSpanId":"Od0H5CjLwYU=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025994316841","endTimeUnixNano":"1735335025994441292","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"MN1XjKdi+4U=","parentSpanId":"Od0H5CjLwYU=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025994445612","endTimeUnixNano":"1735335025994471673","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Od0H5CjLwYU=","parentSpanId":"2cTcRlCzz7A=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025994259650","endTimeUnixNano":"1735335025994477013","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"0y4T4EjPqoA=","parentSpanId":"P3sUoSO2QW8=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025997382847","endTimeUnixNano":"1735335025997472788","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"P3sUoSO2QW8=","parentSpanId":"9xP3tf6dYZc=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025997355987","endTimeUnixNano":"1735335025997486558","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"YMIGvR9FWUI=","parentSpanId":"9xP3tf6dYZc=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025997491278","endTimeUnixNano":"1735335025997516819","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"9xP3tf6dYZc=","parentSpanId":"jn2jF/UGEwI=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025997284916","endTimeUnixNano":"1735335025997519729","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"1DHjc96hDgo=","parentSpanId":"fKTN7FMA9bA=","flags":256,"name":"authorize.getDataBrokerSessionOrServiceAccount","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025999408481","endTimeUnixNano":"1735335026000304333","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/databroker.go:64"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"A+TRUlQLrHs=","parentSpanId":"fKTN7FMA9bA=","flags":256,"name":"authorize.getDataBrokerUser","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026000307673","endTimeUnixNano":"1735335026001064951","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/databroker.go:97"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"qPBApgBMYDo=","parentSpanId":"H3n1ziGO7oI=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026001326014","endTimeUnixNano":"1735335026001992103","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"815f0b5e-a8a0-48d0-9ece-e1a8a1ed9e03\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"sC9Owg0pKCg=","parentSpanId":"H3n1ziGO7oI=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026002013393","endTimeUnixNano":"1735335026002274936","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"815f0b5e-a8a0-48d0-9ece-e1a8a1ed9e03\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"KBPImvf4CCY=","parentSpanId":"H3n1ziGO7oI=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026002369397","endTimeUnixNano":"1735335026002500159","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"815f0b5e-a8a0-48d0-9ece-e1a8a1ed9e03\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"STC1CzycLEI=","parentSpanId":"H3n1ziGO7oI=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026002553109","endTimeUnixNano":"1735335026002670921","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.User\""}},{"key":"record_id","value":{"stringValue":"\"623709e6-f5b8-445c-b93b-641d72e148a6\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"/7zHyVfkJXI=","parentSpanId":"rU3EQcGjQag=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026001109672","endTimeUnixNano":"1735335026002707651","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"H3n1ziGO7oI=","parentSpanId":"rU3EQcGjQag=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026001117002","endTimeUnixNano":"1735335026003087055","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"rU3EQcGjQag=","parentSpanId":"fKTN7FMA9bA=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026001079121","endTimeUnixNano":"1735335026003097476","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"L+E5uS0Pmj8=","parentSpanId":"fKTN7FMA9bA=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026003101526","endTimeUnixNano":"1735335026003142976","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"fKTN7FMA9bA=","parentSpanId":"kKNQsnG+JKs=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025999274240","endTimeUnixNano":"1735335026003151306","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"N+v2AAwKVfI=","parentSpanId":"K5lGkBkLfGA=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025802655949","endTimeUnixNano":"1735335025802690289","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"E93VUaFf2Js=","parentSpanId":"WVzMc/GgbKk=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025802953572","endTimeUnixNano":"1735335025802978622","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Z7SxNTRbIlI=","parentSpanId":"Ht0evUW5Oi8=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025803094304","endTimeUnixNano":"1735335025803117944","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"9DQy+AMvVdU=","parentSpanId":"trf8MoiBLtw=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025821664945","endTimeUnixNano":"1735335025821682636","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ivezjXbKh00=","parentSpanId":"l0zVW4Qbs5Q=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025821824757","endTimeUnixNano":"1735335025821841718","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"+Y+OAZu4k5E=","parentSpanId":"uKeJNfsaBgU=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025821946920","endTimeUnixNano":"1735335025821961990","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"yP28QGB/eIQ=","parentSpanId":"p2l78TY+CMQ=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025834474299","endTimeUnixNano":"1735335025834498179","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"3gTVWLuOnUA=","parentSpanId":"CrmdWPLP2Ss=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025834698192","endTimeUnixNano":"1735335025834715032","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"kgdUpDdt3Zo=","parentSpanId":"ZtLWldKyeFY=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025834802283","endTimeUnixNano":"1735335025834817913","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"VsDbNMPWFyo=","parentSpanId":"YKX2PFK9kUo=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025847440004","endTimeUnixNano":"1735335025847459274","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"NpHyYPqtEcw=","parentSpanId":"2tmsOuk2trk=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025847614826","endTimeUnixNano":"1735335025847629396","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"iimzfcbaGXw=","parentSpanId":"q4oGWBE7P2o=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025847704227","endTimeUnixNano":"1735335025847717107","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"NG0e34bILr0=","parentSpanId":"iPrUuthF+So=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025869171194","endTimeUnixNano":"1735335025869191354","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"WR68p7UO4YI=","parentSpanId":"15r71FlGjP0=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025869382826","endTimeUnixNano":"1735335025869402586","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"GWfj0lLpLIo=","parentSpanId":"ZFDnmObzkoY=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025869475787","endTimeUnixNano":"1735335025869497057","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"o/4RdylU04s=","parentSpanId":"PdGcA51RQic=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025888424664","endTimeUnixNano":"1735335025888452865","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"vV8NB5ruoOU=","parentSpanId":"UovJyUrDUj8=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025888627447","endTimeUnixNano":"1735335025888640797","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"h2MSFkLt9p4=","parentSpanId":"GkjNF0cAMyw=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025888727348","endTimeUnixNano":"1735335025888745088","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"2CeZdqym8xg=","parentSpanId":"Jk23ZxyiZAI=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025933940429","endTimeUnixNano":"1735335025933958539","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"TDCb5mGNTys=","parentSpanId":"22nudVC7nbI=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025934122361","endTimeUnixNano":"1735335025934147932","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"C42DkoQ/1Ic=","parentSpanId":"tE5jOpi7xag=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025934234913","endTimeUnixNano":"1735335025934254703","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"xC/VB6ObsY8=","parentSpanId":"A5Eu+VH/j/k=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025980218182","endTimeUnixNano":"1735335025981570378","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"zQwYJxI53tM=","parentSpanId":"Hr6dVe1Xl7k=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025981702279","endTimeUnixNano":"1735335025982399588","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"qo+2ljnLo3s=","parentSpanId":"1DHjc96hDgo=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025999465152","endTimeUnixNano":"1735335026000214772","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"niGr/qWcz4g=","parentSpanId":"A+TRUlQLrHs=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335026000369583","endTimeUnixNano":"1735335026001014421","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"yo8tJLvB7F8=","parentSpanId":"/7zHyVfkJXI=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335026001200583","endTimeUnixNano":"1735335026001827350","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"GZGIXC4RfsY=","parentSpanId":"/7zHyVfkJXI=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335026001960572","endTimeUnixNano":"1735335026002569590","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, + {"timestamp":"2024-12-27T21:30:29.976009599Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/databroker/cache.go:94"}},{"key":"service.name","value":{"stringValue":"Data Broker"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"ba01ZDTHdtk=","parentSpanId":"wASe7nky7+s=","flags":256,"name":"databroker.grpc.AcquireLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024979877729","endTimeUnixNano":"1735335024979946490","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:83"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"BpM7OEx5blE=","parentSpanId":"PZh68vG8OFw=","flags":256,"name":"databroker.grpc.SyncLatest","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024980693738","endTimeUnixNano":"1735335024980859720","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:393"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"uSSBr2vDr+k=","parentSpanId":"gmAAOaMKlhw=","flags":256,"name":"databroker.grpc.SyncLatest","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024980710859","endTimeUnixNano":"1735335024980930291","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:393"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"JwezNJcX+Sw=","parentSpanId":"hcyPk5piJ2w=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025481032712","endTimeUnixNano":"1735335025481045612","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"Rt5U410X8d8=","parentSpanId":"c61OVvg7XN8=","flags":256,"name":"databroker.grpc.SyncLatest","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025791172121","endTimeUnixNano":"1735335025791253352","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:393"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"Gq0XIVz0V8w=","parentSpanId":"iVo7IHe6x6A=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025980612787","endTimeUnixNano":"1735335025980623447","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"6X2Mt+eDjks=","parentSpanId":"vZpvKbrCQqQ=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025981082192","endTimeUnixNano":"1735335025981188064","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"FbAKR5t/VF8=","parentSpanId":"2yTflfwGhOI=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025982084235","endTimeUnixNano":"1735335025982169766","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"uRXsqpb8sac=","parentSpanId":"cifxyFmv4dY=","flags":256,"name":"databroker.grpc.Get","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025990608827","endTimeUnixNano":"1735335025990630187","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:110"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"F3yNlNUGRtw=","parentSpanId":"vP37Rm86XmY=","flags":256,"name":"databroker.grpc.Put","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025991377145","endTimeUnixNano":"1735335025991412456","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:207"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ymstrLvsdeM=","parentSpanId":"3ijdqATnWA0=","flags":256,"name":"databroker.grpc.Put","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025992282787","endTimeUnixNano":"1735335025992317287","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:207"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"JS26p672Ew8=","parentSpanId":"WgzfdQgrifQ=","flags":256,"name":"databroker.grpc.Get","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025995441274","endTimeUnixNano":"1735335025995463904","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:110"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"7yYzlt2jIZw=","parentSpanId":"AWyn5ABOBS0=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025999878447","endTimeUnixNano":"1735335025999959508","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"TinKSNbNkto=","parentSpanId":"QBjGZUW54F0=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026000717487","endTimeUnixNano":"1735335026000775288","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"mX+fLnOfT9c=","parentSpanId":"yHJAjGq7Cgg=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026001565227","endTimeUnixNano":"1735335026001621198","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"63xkmInB/OQ=","parentSpanId":"DC1wSzzAIIQ=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026002316647","endTimeUnixNano":"1735335026002366377","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"uGIKXrnoZgI=","parentSpanId":"48tM9guCB7U=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026481483556","endTimeUnixNano":"1735335026481500596","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"kUBB4g/3mLw=","parentSpanId":"CGVc3FYZu4Q=","flags":256,"name":"databroker.grpc.Report","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026622495634","endTimeUnixNano":"1735335026622541525","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/registry.go:27"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"AFuw9JL5Xes=","parentSpanId":"IPKDPNoyYVI=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026981558406","endTimeUnixNano":"1735335026981568306","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"ONbNFwPfwZM=","parentSpanId":"534wdGGTsVw=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335027481416914","endTimeUnixNano":"1735335027481433145","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"QJOnObIqIf8=","parentSpanId":"Kt6HdK0x1SI=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335027981632535","endTimeUnixNano":"1735335027981643605","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"GRV9N0Uu6WI=","parentSpanId":"s/79OOiIyW8=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335028481339059","endTimeUnixNano":"1735335028481354230","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"YpZ4gH4pj1s=","parentSpanId":"pNRMXAPmD0o=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335028981482941","endTimeUnixNano":"1735335028981492081","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"/BP6SqKMjSo=","parentSpanId":"u3CptJkv/oQ=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335029481050441","endTimeUnixNano":"1735335029481060561","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"wASe7nky7+s=","parentSpanId":"RExgtixuvIo=","flags":768,"name":"databroker.DataBrokerService/AcquireLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335024979764968","endTimeUnixNano":"1735335024980042561","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"AcquireLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"RExgtixuvIo=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/AcquireLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335024979268002","endTimeUnixNano":"1735335024980212473","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"AcquireLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"PZh68vG8OFw=","parentSpanId":"yecKEUDPiwE=","flags":768,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335024980602177","endTimeUnixNano":"1735335024980873570","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"gmAAOaMKlhw=","parentSpanId":"uTXSTu1o/4w=","flags":768,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335024980622358","endTimeUnixNano":"1735335024980940641","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"yecKEUDPiwE=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335024980357704","endTimeUnixNano":"1735335024981013972","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"uTXSTu1o/4w=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335024980347634","endTimeUnixNano":"1735335024981035892","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"hcyPk5piJ2w=","parentSpanId":"Jf7MJl1DrwM=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025480950041","endTimeUnixNano":"1735335025481084252","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"Jf7MJl1DrwM=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025480640167","endTimeUnixNano":"1735335025481266504","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"iVo7IHe6x6A=","parentSpanId":"bcj0AK8Y/vs=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025980551466","endTimeUnixNano":"1735335025980651767","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"bcj0AK8Y/vs=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025980403345","endTimeUnixNano":"1735335025980758629","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"48tM9guCB7U=","parentSpanId":"P8qyVdH0S0I=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335026481393325","endTimeUnixNano":"1735335026481539477","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"P8qyVdH0S0I=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335026481185033","endTimeUnixNano":"1735335026481632248","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"IPKDPNoyYVI=","parentSpanId":"0saA2tMcvZk=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335026981468595","endTimeUnixNano":"1735335026981600897","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"0saA2tMcvZk=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335026981228112","endTimeUnixNano":"1735335026981720128","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"534wdGGTsVw=","parentSpanId":"XoADz2djL1c=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335027481298093","endTimeUnixNano":"1735335027481487875","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"XoADz2djL1c=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335027481013320","endTimeUnixNano":"1735335027481626977","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"Kt6HdK0x1SI=","parentSpanId":"e04LxDH4n8U=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335027981534604","endTimeUnixNano":"1735335027981689146","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"e04LxDH4n8U=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335027981301780","endTimeUnixNano":"1735335027981813137","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"s/79OOiIyW8=","parentSpanId":"Ok+FyABLV8k=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335028481247548","endTimeUnixNano":"1735335028481393530","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"Ok+FyABLV8k=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335028481041156","endTimeUnixNano":"1735335028481531892","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"pNRMXAPmD0o=","parentSpanId":"aSHOPBsOYTU=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335028981413940","endTimeUnixNano":"1735335028981515991","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"aSHOPBsOYTU=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335028981164807","endTimeUnixNano":"1735335028981576862","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"u3CptJkv/oQ=","parentSpanId":"gxf2r5UfW94=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335029480982791","endTimeUnixNano":"1735335029481085622","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"gxf2r5UfW94=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335029480740818","endTimeUnixNano":"1735335029481166113","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, + {"timestamp":"2024-12-27T21:30:30.080468097Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"hKGR0btH9xI=","parentSpanId":"fxjsKYWTWL4=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025998816000","endTimeUnixNano":"1735335026004007000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"d9cf85d0-6ce9-93d2-83f4-31e616491757"}},{"key":"http.url","value":{"stringValue":"https://127.0.0.1:43211/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"2"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"http.status_code","value":{"stringValue":"200"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"7f18ec29859358be"}}]}]}]}]}}, + {"timestamp":"2024-12-27T21:30:30.193443488Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:377"}},{"key":"service.name","value":{"stringValue":"Test Environment"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"FRYjgKjC2fg=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025791678197","endTimeUnixNano":"1735335030170817557","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"error reading from server: EOF","code":"STATUS_CODE_ERROR"}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"eTl2DY0T39Y=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335030177708480","endTimeUnixNano":"1735335030180906089","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"error reading server preface: read tcp 127.0.0.1:33216-\u003e127.0.0.1:41075: read: connection reset by peer\"","code":"STATUS_CODE_ERROR"}}]},{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"wX7NySRpuEk=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"State Change Callback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026004298180","endTimeUnixNano":"1735335030193020283","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:932"}},{"key":"state","value":{"stringValue":"Stopping"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"sXmaUcEg8Zo=","flags":256,"name":"TestOTLPTracing","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024644383476","endTimeUnixNano":"1735335030193209966","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:379"}}],"events":[{"timeUnixNano":"1735335024645032905","name":"Add: *scenarios.OTLPTraceReceiver from /home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:66"},{"timeUnixNano":"1735335024645076815","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735335024645146246","name":"AddUpstream: *upstreams.grpcUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:71"},{"timeUnixNano":"1735335024645174036","name":"Add: *upstreams.grpcUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:71"},{"timeUnixNano":"1735335024645180996","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735335024645195116","name":"AddTask: *upstreams.grpcUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:71"},{"timeUnixNano":"1735335024645213057","name":"AddUpstream: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:72"},{"timeUnixNano":"1735335024645222787","name":"Add: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:72"},{"timeUnixNano":"1735335024645229277","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735335024645240347","name":"AddTask: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:72"},{"timeUnixNano":"1735335024645326608","name":"Add: *scenarios.IDP from /home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:72"},{"timeUnixNano":"1735335024645338678","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735335024645456839","name":"AddUpstream: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/mock_idp.go:97"},{"timeUnixNano":"1735335024645490060","name":"Add: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/mock_idp.go:97"},{"timeUnixNano":"1735335024645496940","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735335024645508010","name":"AddTask: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/mock_idp.go:97"},{"timeUnixNano":"1735335024645584241","name":"AddUpstream: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:84"},{"timeUnixNano":"1735335024645606441","name":"Add: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:84"},{"timeUnixNano":"1735335024645613641","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735335024645624181","name":"Add: *testenv.PolicyRoute from /home/ubuntu/pomerium/internal/testenv/types.go:125"},{"timeUnixNano":"1735335024645635562","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735335024645645462","name":"AddTask: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:84"},{"timeUnixNano":"1735335024645664252","name":"Start()"},{"timeUnixNano":"1735335024645670342","name":"state NotRunning -\u003e Starting"},{"timeUnixNano":"1735335024645684982","name":"temp dir: /tmp/TestOTLPTracing380461907/001"},{"timeUnixNano":"1735335024646028336","name":"AddTask: *testenv.taskFunc from /home/ubuntu/pomerium/internal/testenv/environment.go:608"},{"timeUnixNano":"1735335024646118027","name":"state Starting -\u003e Running"},{"timeUnixNano":"1735335024738510112","name":"envoy profiling not available"},{"timeUnixNano":"1735335026004215959","name":"stop: Stop() called manually"},{"timeUnixNano":"1735335026004266750","name":"state Running -\u003e Stopping"},{"timeUnixNano":"1735335026004276110","name":"notifying 3 listeners of state change"},{"timeUnixNano":"1735335026004308931","name":"invoking state change callback (caller: /home/ubuntu/pomerium/internal/testenv/environment.go:659)"},{"timeUnixNano":"1735335026004339961","name":"invoking state change callback (caller: /home/ubuntu/pomerium/internal/testenv/snippets/shutdown.go:17)"},{"timeUnixNano":"1735335026004372111","name":"invoking state change callback (caller: /home/ubuntu/pomerium/internal/testenv/snippets/shutdown.go:17)"},{"timeUnixNano":"1735335030192986643","name":"pomerium server shut down without error"},{"timeUnixNano":"1735335030193042454","name":"done notifying state change listeners"}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, + {"timestamp":"2024-12-27T21:30:30.193784142Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/server.go:85"}},{"key":"service.name","value":{"stringValue":"Control Plane"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"Kcsl2I0u90IL3RjX9ECWQw==","spanId":"vdSorNgp3tM=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335030080404696","endTimeUnixNano":"1735335030080571098","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52388"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"c2UxzUHBdbEMq0ms+REjYg==","spanId":"k6YpnvzKBjQ=","flags":768,"name":"envoy.service.accesslog.v3.AccessLogService/StreamAccessLogs","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335026081822587","endTimeUnixNano":"1735335030118825295","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.accesslog.v3.AccessLogService"}},{"key":"rpc.method","value":{"stringValue":"StreamAccessLogs"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}},{"traceId":"z+C0XOvX6j/JSUWh8/UHWA==","spanId":"2Hpg8GvNW0k=","flags":768,"name":"envoy.service.accesslog.v3.AccessLogService/StreamAccessLogs","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025856359311","endTimeUnixNano":"1735335030123684784","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.accesslog.v3.AccessLogService"}},{"key":"rpc.method","value":{"stringValue":"StreamAccessLogs"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"gwpAbVXcwdw=","parentSpanId":"FRYjgKjC2fg=","flags":768,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025792058502","endTimeUnixNano":"1735335030171437535","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}},{"traceId":"1s487gww7zxt1qWc+o35qA==","spanId":"vOl62ewaIDc=","flags":768,"name":"envoy.service.discovery.v3.AggregatedDiscoveryService/DeltaAggregatedResources","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025036599817","endTimeUnixNano":"1735335030174370480","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.discovery.v3.AggregatedDiscoveryService"}},{"key":"rpc.method","value":{"stringValue":"DeltaAggregatedResources"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52346"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, + {"timestamp":"2024-12-27T21:30:30.194141156Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/databroker/cache.go:94"}},{"key":"service.name","value":{"stringValue":"Data Broker"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"CLyx0UG93i0=","parentSpanId":"xPK4CiYdt1w=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335029981041359","endTimeUnixNano":"1735335029981050839","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"qG+NVOCrqCo=","parentSpanId":"gwpAbVXcwdw=","flags":256,"name":"databroker.grpc.Sync","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025792173903","endTimeUnixNano":"1735335030171424275","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:355"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"NS5usMrn7yI=","parentSpanId":"EF1NpNpk1bM=","flags":256,"name":"databroker.grpc.Sync","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024981421088","endTimeUnixNano":"1735335030192340375","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:355"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"1JC9QOassrE=","parentSpanId":"/+qBjKQTDHQ=","flags":256,"name":"databroker.grpc.Sync","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024981366457","endTimeUnixNano":"1735335030192345195","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:355"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"xPK4CiYdt1w=","parentSpanId":"6mMToadBqf4=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335029980977879","endTimeUnixNano":"1735335029981082870","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"6mMToadBqf4=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335029980793386","endTimeUnixNano":"1735335029981161621","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"mqSZh+et1KQ=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335024981134203","endTimeUnixNano":"1735335030192211813","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{"message":"context canceled","code":"STATUS_CODE_ERROR"}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"mA2k4+Vryho=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335024981065413","endTimeUnixNano":"1735335030192267074","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{"message":"context canceled","code":"STATUS_CODE_ERROR"}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"/+qBjKQTDHQ=","parentSpanId":"mA2k4+Vryho=","flags":768,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335024981276086","endTimeUnixNano":"1735335030192360775","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"EF1NpNpk1bM=","parentSpanId":"mqSZh+et1KQ=","flags":768,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335024981311536","endTimeUnixNano":"1735335030192363645","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"/luU/UTJghg=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/ReleaseLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335030192357005","endTimeUnixNano":"1735335030192453166","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"ReleaseLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"the connection is draining","code":"STATUS_CODE_ERROR"}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"mwChaFmJX8g=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/ReleaseLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335030192555787","endTimeUnixNano":"1735335030192839370","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"ReleaseLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:45869: connect: connection refused\"","code":"STATUS_CODE_ERROR"}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}} +] diff --git a/internal/telemetry/trace/testdata/recording_02_multi_trace.json b/internal/telemetry/trace/testdata/recording_02_multi_trace.json new file mode 100644 index 000000000..a5241ed76 --- /dev/null +++ b/internal/telemetry/trace/testdata/recording_02_multi_trace.json @@ -0,0 +1,20 @@ +[ + {"timestamp":"2024-12-30T22:09:12.0668664Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"+R7poiK6/239h1nEy5hIrA==","spanId":"7pbeHJTanQw=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552060390000","endTimeUnixNano":"1735596552063277000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"d359b246-91e9-9c73-9975-8fd66089853f"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:35065/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"0"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"+R7poiK6/239h1nEy5hIrA==","spanId":"ynwZv1w6sTA=","parentSpanId":"7pbeHJTanQw=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552060836000","endTimeUnixNano":"1735596552063332000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"2"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"BI0pXOVFSFDjDh4HlytjyA==","spanId":"INSLthkL30A=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552064753000","endTimeUnixNano":"1735596552065999000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"52243523-9ac3-9bc0-bfa0-49407482f874"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:35065/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"0"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, + {"timestamp":"2024-12-30T22:09:12.071787916Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"BI0pXOVFSFDjDh4HlytjyA==","spanId":"7hYMiT5DOHg=","parentSpanId":"INSLthkL30A=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552064892000","endTimeUnixNano":"1735596552066228000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"2"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"nD9dzNOnTKOIJMxmlWZ9bw==","spanId":"1mtI7TODihI=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552070193000","endTimeUnixNano":"1735596552071393000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"b0ee2c5d-dbbb-9e99-a761-52475c57d45b"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:35065/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"0"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"nD9dzNOnTKOIJMxmlWZ9bw==","spanId":"sIew416JLXc=","parentSpanId":"1mtI7TODihI=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552070307000","endTimeUnixNano":"1735596552071417000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"2"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, + {"timestamp":"2024-12-30T22:09:12.088508581Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"b0XQMNrGpRts3otatJ4VkQ==","spanId":"Fry+Em1HWnk=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552077561000","endTimeUnixNano":"1735596552078549000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"e8b5646e-6da4-951e-9e39-79ee2ad7f54f"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:35065/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"0"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"b0XQMNrGpRts3otatJ4VkQ==","spanId":"poak1tCu/Ds=","parentSpanId":"Fry+Em1HWnk=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552077719000","endTimeUnixNano":"1735596552078573000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"2"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"lCsEeGUuNYemOgQ6EAqw/Q==","spanId":"lVjYK8BjGEA=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552087015000","endTimeUnixNano":"1735596552088139000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"2d9063c8-0eb2-9b72-a077-759abcfaeec3"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:35065/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"0"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, + {"timestamp":"2024-12-30T22:09:12.098955033Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"lCsEeGUuNYemOgQ6EAqw/Q==","spanId":"BI5T6olEsIc=","parentSpanId":"lVjYK8BjGEA=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552087136000","endTimeUnixNano":"1735596552088265000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"2"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"69JHBTXAPq7zDqo19nNt3g==","spanId":"/8VC2W2xsyA=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552097531000","endTimeUnixNano":"1735596552098644000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"bc305fba-3cf8-96c4-b6d2-f4210a62866c"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:35065/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"0"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"69JHBTXAPq7zDqo19nNt3g==","spanId":"rlZX67RjnyY=","parentSpanId":"/8VC2W2xsyA=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552097660000","endTimeUnixNano":"1735596552098666000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"2"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, + {"timestamp":"2024-12-30T22:09:12.18635478Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"qGCrQnZvKm9AiV3UG+9y4A==","spanId":"5bcR4af/ywE=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552123774000","endTimeUnixNano":"1735596552124933000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"cfae5ccd-c26f-9e1f-b2e1-1e98200ec5b9"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:35065/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"0"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"qGCrQnZvKm9AiV3UG+9y4A==","spanId":"rNEObiMfLz0=","parentSpanId":"5bcR4af/ywE=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552123869000","endTimeUnixNano":"1735596552124967000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"2"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"oboPZ53IIkU+AOAisyhydQ==","spanId":"Z7HMwkovokw=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552184826000","endTimeUnixNano":"1735596552185961000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"8addfd98-9521-9b15-9f5a-50b3bb8ebe83"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:35065/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"0"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, + {"timestamp":"2024-12-30T22:09:12.253542922Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"oboPZ53IIkU+AOAisyhydQ==","spanId":"ws87AD1cVDE=","parentSpanId":"Z7HMwkovokw=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552184938000","endTimeUnixNano":"1735596552186090000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"2"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"+OVHuklKQMo=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552248874000","endTimeUnixNano":"1735596552253121000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"be91c88e-0c02-9e51-b7ef-8de7c22ab1bb"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:35065/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"1450"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"UAEX"}}]},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"xCIoDfKUEmA=","parentSpanId":"+OVHuklKQMo=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552248989000","endTimeUnixNano":"1735596552253152000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_unauthorized"}}],"status":{}}]}]}]}}, + {"timestamp":"2024-12-30T22:09:12.262728948Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"DfyMBC/GxYWDl+LE0WQyhQ==","spanId":"acI32UNM7h4=","parentSpanId":"j8D1S9l24K0=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552257242000","endTimeUnixNano":"1735596552258519000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"DfyMBC/GxYWDl+LE0WQyhQ==","spanId":"FBPvHWyFn6I=","parentSpanId":"j8D1S9l24K0=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552258384000","endTimeUnixNano":"1735596552262124000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:37809"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:37809"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"DfyMBC/GxYWDl+LE0WQyhQ==","spanId":"j8D1S9l24K0=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552256907000","endTimeUnixNano":"1735596552262163000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"acad727c-3b98-42c2-8ef5-4a344b18be16"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:35065/.pomerium/sign_in?pomerium_expiry=1735596852\u0026pomerium_idp_id=C44yiF1cj9B5ky7OuS1Le2l5g8fulMelHIX6d6E4dCeB\u0026pomerium_issued=1735596552\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A35065%2Ffoo\u0026pomerium_signature=nQpfEBF3IsHFTzzpL1D7sFy-CHxB1sJTeBATVXukUeQ%3D\u0026pomerium_traceparent=00-0deff952685913fe4df3ac83a5efbb83-1dcc8cb5972746ae-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"792"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-0deff952685913fe4df3ac83a5efbb83-1dcc8cb5972746ae-01"}}]}]}]}]}}, + {"timestamp":"2024-12-30T22:09:12.272346001Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"E9PYtbtULsI=","parentSpanId":"QCoXgtwGqF0=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552264909000","endTimeUnixNano":"1735596552265740000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"qZJUemZrh0U=","parentSpanId":"QCoXgtwGqF0=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552265713000","endTimeUnixNano":"1735596552271945000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:37809"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:37809"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"QCoXgtwGqF0=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552264741000","endTimeUnixNano":"1735596552271969000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"ab366edc-1a2f-9ed9-851b-d982a3396ba9"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:35065/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=dEVxQThvQ0k4amQwQWc1VExVN2JJS3I0N1FPd3JQc1l4NW5xSC9nSFZaTkFJaEJwN011RVptTm5oK0NVZ1Nya1dkNis1Z1VoUytjZGFlU1dGNFBoSUE9PXwxNzM1NTk2NTUyfDwRWSChr4x26k5xhXcIAFuQ49JDHDh0PW_YYl2LORSWykRnWoJEuD8UwPKxXsP_HF70JYwmOVesuA17GdKHTFWswy_ggJVJaKJgcWNQL2v-GcbmXWUcOnYi80bud98SpVaii44ozEbwJAkKuF8ZiUGGUBXtvqlazXUmyMWUgujOrXEnr8l-xQL5dMuj1b0szz3q8W3SrMc0wV_sSuGuV3d8Vi9KVvVvuJ3vtheLZWHvUtlppZLT2yyRUIwCyO6RCIZxY6huIPa8Sxs3Lgg61jFbpvWxbNNFq12Ux1Z4zIt2bhdVD9TFOv9QoBXhpZPHOYru0w2k7JfED7Iio9MPOjparZCsOZTzgs5j4eDYbpudwuJRdcPQ4tCZP_3M4WpUKzS8X_V56YewQM-tkk_DHqH7YdoU3RcNr4NC9G0kbp7mmtUQEVgZ1ZahtgfPAZKRGWFF4VFM-_O1gyGJUmPU6VW3QgeVKPyEUzlrZAeWgMmDr3SGIxX3RYnr8e0ttGzUzhv5WiVuMQJaAoQQPTJ6nxjBtzV13NFx1AnPDE5clRQEpWEiTr0z45NSWtfhLQE9LN4BkDLRTr0r"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"322"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-0deff952685913fe4df3ac83a5efbb83-1dcc8cb5972746ae-01"}}]}]}]}]}}, + {"timestamp":"2024-12-30T22:09:12.279487054Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"60XX4iML+BmEG3gZjznr1g==","spanId":"lWFmfug8YNY=","parentSpanId":"KE354oTf2M0=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552272538000","endTimeUnixNano":"1735596552273360000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"60XX4iML+BmEG3gZjznr1g==","spanId":"uuvSf1QFkxA=","parentSpanId":"KE354oTf2M0=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552273329000","endTimeUnixNano":"1735596552279103000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:37809"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:37809"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"60XX4iML+BmEG3gZjznr1g==","spanId":"KE354oTf2M0=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552272386000","endTimeUnixNano":"1735596552279120000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"e96483ca-4bae-483e-9240-30e79bc4b0b7"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:35065/.pomerium/sign_in?pomerium_expiry=1735596852\u0026pomerium_idp_id=C44yiF1cj9B5ky7OuS1Le2l5g8fulMelHIX6d6E4dCeB\u0026pomerium_issued=1735596552\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A35065%2Ffoo\u0026pomerium_signature=nQpfEBF3IsHFTzzpL1D7sFy-CHxB1sJTeBATVXukUeQ%3D\u0026pomerium_traceparent=00-0deff952685913fe4df3ac83a5efbb83-1dcc8cb5972746ae-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"941"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-0deff952685913fe4df3ac83a5efbb83-1dcc8cb5972746ae-01"}}]}]}]}]}}, + {"timestamp":"2024-12-30T22:09:12.281856012Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"PAOA0r+dYsQn8fEvj1rlOQ==","spanId":"5ThhIloR5KY=","parentSpanId":"IfligTWwckk=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552279794000","endTimeUnixNano":"1735596552280758000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"PAOA0r+dYsQn8fEvj1rlOQ==","spanId":"LNr+ATKO4qk=","parentSpanId":"IfligTWwckk=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552280625000","endTimeUnixNano":"1735596552281367000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:37809"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:37809"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"PAOA0r+dYsQn8fEvj1rlOQ==","spanId":"IfligTWwckk=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552279582000","endTimeUnixNano":"1735596552281384000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"65383ba8-61d9-4cbb-bdb3-650b70eacedb"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:35065/.pomerium/callback/?pomerium_expiry=1735596852\u0026pomerium_issued=1735596552\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A35065%2Ffoo\u0026pomerium_session_encrypted=EWV_6baRK5R5-pUgJFqfUJK13XziMrN6yew9hwsfF_MX5DDsbz3LeQrQ7VpyWA4RWATPOqoOObC0Fjqac3VdaiFkpbTuiXsU8OzOU3m2iU01xkG0nGsgaz8gSfOsNH9vtfSlg73G8NjyelubtY6OGRHw64tHdy-BoGwp68m6bocVbrzUQgOHPXbn7vuFnZ5pj9NdAHQwEZOKNouE6kG6fuz2oZL6eLk9E-uPef5G0OV6Ewl6oNz71kR8f0Y-vdPwE9xDEuN6tWQP6yPeMT87I7KhIAhgwhbm6k_keZm6c3Bz6Mo10WevLJYhNLDQdkgJ0tU5sYObYp5gXMAK1w7lMviZ5bzrjC4asP2oeWaU4ZTg_l_mC0lsu7ztlcDlo1x3ILNuucABtMD1jIgE2H2-BWqGjd_oAxF-53SPNu251S-ciO8Sq1E-T3KP3UKAif9Y2qmVxEntQqxAKs3YzbIEaCwUB4mpr9aLY1SnBN9N2O2KpzLQs3OKdcaLHKv0_K675SaJJR9MUcqjSpUxlHv4BM3NQbA9p56wTCqqG7TJD9DTTCuxIDvLSQRimPI1mJsQic8f9ZjFLjkD_CdHnM1jy2ckoVvYwVELOa6-pPFfnlcqy93Gx7tBGYB7K8_OzAarv7YK54QiRykTpNBf-mymLuRtQcLf9hHsFKsF-Gfx_aforvpZhK4L9EqLE3fZZ3Y0LIAlbjbJzxuip0i4JoudSpbZuL7w2roxYj6dkSxJ_fj1HJVj47wbxPoU6TMqEG1Yu5t14oG3SrtRMxoVyBzwPClS2Hv8EnnizDUKr12uVVN9dux7P1xZ7Ve5qhJMrx4%3D\u0026pomerium_si"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"66"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-0deff952685913fe4df3ac83a5efbb83-1dcc8cb5972746ae-01"}}]}]}]}]}}, + {"timestamp":"2024-12-30T22:09:12.287783501Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"6LX3bJmVP78=","parentSpanId":"HxQN6e2rYXU=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552281917000","endTimeUnixNano":"1735596552286888000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"Nnmvh2NOoyE=","parentSpanId":"HxQN6e2rYXU=","name":"router route-8054bb0274b8b3dd egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552286739000","endTimeUnixNano":"1735596552287352000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:44525"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:44525"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"200"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"HxQN6e2rYXU=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552281793000","endTimeUnixNano":"1735596552287370000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"21cb5717-0472-9782-9751-6cac2d39ac3e"}},{"key":"http.url","value":{"stringValue":"https://127.0.0.1:44525/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"2"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"200"}},{"key":"response_flags","value":{"stringValue":"-"}}]}]}]}]}}, + {"timestamp":"2024-12-30T22:09:12.400423432Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:377"}},{"key":"service.name","value":{"stringValue":"Test Environment"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"MfzSNZegoFc=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"Start","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596550908732917","endTimeUnixNano":"1735596550909168452","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:543"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"DSJCIaOGrEM=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.config_source.rebuild","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551000633187","endTimeUnixNano":"1735596551001673799","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"6Qd1AEzhdis=","parentSpanId":"Woul7Heu4HE=","flags":256,"name":"envoyconfig.Builder.BuildBootstrapStaticResources","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551223409140","endTimeUnixNano":"1735596551223438210","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"Woul7Heu4HE=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"envoyconfig.Builder.BuildBootstrap","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551223351599","endTimeUnixNano":"1735596551223487891","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"cFxn05+6kBk=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"authorize.newPolicyEvaluator","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551233260585","endTimeUnixNano":"1735596551241016875","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"KTlvZ+eAgZg=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"authorize.newPolicyEvaluator","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551241293388","endTimeUnixNano":"1735596551241567001","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"6QO6cyUuZo8=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"snippets.WaitStartupComplete","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596550909198712","endTimeUnixNano":"1735596552053766546","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"HNOD2Zpqhns=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"State Change Callback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552287806201","endTimeUnixNano":"1735596552287850811","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:932"}},{"key":"state","value":{"stringValue":"Stopping"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"DlBnuxNmVK0=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"State Change Callback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552287824491","endTimeUnixNano":"1735596552287866952","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:932"}},{"key":"state","value":{"stringValue":"Stopping"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"gTtrtr+YbHo=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"State Change Callback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552287796291","endTimeUnixNano":"1735596552399637883","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:932"}},{"key":"state","value":{"stringValue":"Stopping"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"Qkk6itX7wm8=","flags":256,"name":"TestOTLPTracing_TraceCorrelation","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596550907558493","endTimeUnixNano":"1735596552399753564","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:379"}}],"events":[{"timeUnixNano":"1735596550908132151","name":"Add: *scenarios.OTLPTraceReceiver from /home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:133"},{"timeUnixNano":"1735596550908190821","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735596550908242292","name":"AddUpstream: *upstreams.grpcUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:71"},{"timeUnixNano":"1735596550908262792","name":"Add: *upstreams.grpcUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:71"},{"timeUnixNano":"1735596550908269672","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735596550908285502","name":"AddTask: *upstreams.grpcUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:71"},{"timeUnixNano":"1735596550908307833","name":"AddUpstream: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:72"},{"timeUnixNano":"1735596550908326843","name":"Add: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:72"},{"timeUnixNano":"1735596550908336333","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735596550908347513","name":"AddTask: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:72"},{"timeUnixNano":"1735596550908432174","name":"Add: *scenarios.IDP from /home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:139"},{"timeUnixNano":"1735596550908442314","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735596550908521745","name":"AddUpstream: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/mock_idp.go:97"},{"timeUnixNano":"1735596550908543275","name":"Add: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/mock_idp.go:97"},{"timeUnixNano":"1735596550908549705","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735596550908566855","name":"AddTask: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/mock_idp.go:97"},{"timeUnixNano":"1735596550908674197","name":"AddUpstream: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:151"},{"timeUnixNano":"1735596550908689937","name":"Add: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:151"},{"timeUnixNano":"1735596550908697167","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735596550908707587","name":"Add: *testenv.PolicyRoute from /home/ubuntu/pomerium/internal/testenv/types.go:125"},{"timeUnixNano":"1735596550908714717","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735596550908724767","name":"AddTask: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:151"},{"timeUnixNano":"1735596550908739827","name":"Start()"},{"timeUnixNano":"1735596550908745937","name":"state NotRunning -\u003e Starting"},{"timeUnixNano":"1735596550908758458","name":"temp dir: /tmp/TestOTLPTracing_TraceCorrelation2182135484/001"},{"timeUnixNano":"1735596550909053601","name":"AddTask: *testenv.taskFunc from /home/ubuntu/pomerium/internal/testenv/environment.go:608"},{"timeUnixNano":"1735596550909130612","name":"state Starting -\u003e Running"},{"timeUnixNano":"1735596551000151162","name":"envoy profiling not available"},{"timeUnixNano":"1735596552287705270","name":"stop: Stop() called manually"},{"timeUnixNano":"1735596552287763410","name":"state Running -\u003e Stopping"},{"timeUnixNano":"1735596552287772900","name":"notifying 3 listeners of state change"},{"timeUnixNano":"1735596552287811151","name":"invoking state change callback (caller: /home/ubuntu/pomerium/internal/testenv/environment.go:659)"},{"timeUnixNano":"1735596552287828861","name":"invoking state change callback (caller: /home/ubuntu/pomerium/internal/testenv/snippets/shutdown.go:17)"},{"timeUnixNano":"1735596552287859241","name":"invoking state change callback (caller: /home/ubuntu/pomerium/internal/testenv/snippets/shutdown.go:17)"},{"timeUnixNano":"1735596552399607632","name":"pomerium server shut down without error"},{"timeUnixNano":"1735596552399645233","name":"done notifying state change listeners"}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"9BbvwqcIDE4=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552051616251","endTimeUnixNano":"1735596552053279621","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"kKNgDiOw0Jw=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"registry.Registry/Report","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552242718946","endTimeUnixNano":"1735596552242750267","attributes":[{"key":"rpc.service","value":{"stringValue":"registry.Registry"}},{"key":"rpc.method","value":{"stringValue":"Report"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"YjiylRy7lCw=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552053323951","endTimeUnixNano":"1735596552315228660","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"error reading from server: EOF","code":"STATUS_CODE_ERROR"}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"mysN7JNNyTY=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552327146689","endTimeUnixNano":"1735596552389290772","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"error reading server preface: read tcp 127.0.0.1:48314-\u003e127.0.0.1:40165: read: connection reset by peer\"","code":"STATUS_CODE_ERROR"}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"2Sqw7gm3daQ=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552397549129","endTimeUnixNano":"1735596552397602099","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"error reading server preface: read tcp 127.0.0.1:48314-\u003e127.0.0.1:40165: read: connection reset by peer\"","code":"STATUS_CODE_ERROR"}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, + {"timestamp":"2024-12-30T22:09:12.400742295Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/upstreams/http.go:223"}},{"key":"service.name","value":{"stringValue":"Upstream"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp","version":"0.57.0"},"spans":[{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"IKp7lJq2AVg=","parentSpanId":"Nnmvh2NOoyE=","flags":768,"name":"Server: GET /foo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552287072633","endTimeUnixNano":"1735596552287102743","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"127.0.0.1"}},{"key":"net.host.port","value":{"intValue":"44525"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"44066"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/foo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"2"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, + {"timestamp":"2024-12-30T22:09:12.401027609Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/upstreams/http.go:223"}},{"key":"service.name","value":{"stringValue":"IDP"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp","version":"0.57.0"},"spans":[{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"Q1AHuKM97DU=","parentSpanId":"+fYq7JMxmug=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552260527653","endTimeUnixNano":"1735596552260621394","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"43581"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"42722"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"Fd8uhzUZo099GwCYWCNoyQ==","spanId":"rxnESoh2tjw=","flags":256,"name":"Server: GET /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552264061484","endTimeUnixNano":"1735596552264138305","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"43581"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"42726"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"512"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"vtQLJDvERjrZ9z8pGENSYA==","spanId":"hjt1scYLMl8=","flags":256,"name":"Server: POST /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552264425638","endTimeUnixNano":"1735596552264504769","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"43581"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"42726"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"18piKxRJ7tw=","parentSpanId":"uhrN3Ns9n/w=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552266194929","endTimeUnixNano":"1735596552266253250","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"43581"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"42722"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"40s9nB6GeQMtZnV+dJC7AA==","spanId":"oTzLO7CG3g4=","flags":256,"name":"Server: POST /oidc/token","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552266517173","endTimeUnixNano":"1735596552266727745","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"43581"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"42722"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/token"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"193"}},{"key":"http.response_content_length","value":{"intValue":"673"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"dwMPQOqC63eLDuFdJ/DRFg==","spanId":"WeTERDnkqsw=","parentSpanId":"BHBu4awlz/Y=","flags":768,"name":"Server: GET /.well-known/jwks.json","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552267120060","endTimeUnixNano":"1735596552267200661","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"43581"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"42722"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/jwks.json"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"237"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"B5szkLtGsj4UnJtqkubX+g==","spanId":"FudIJLl7VtY=","flags":256,"name":"Server: GET /oidc/userinfo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552267615176","endTimeUnixNano":"1735596552267670417","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"43581"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"42722"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/userinfo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"150"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, + {"timestamp":"2024-12-30T22:09:12.40196375Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/server.go:85"}},{"key":"service.name","value":{"stringValue":"Control Plane"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"x7/JV/qOoQc=","parentSpanId":"AF2VbjcLF8o=","flags":256,"name":"envoyconfig.Builder.BuildClusters","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551016707484","endTimeUnixNano":"1735596551017525873","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"WyZbPrj6kIk=","parentSpanId":"AF2VbjcLF8o=","flags":256,"name":"envoyconfig.Builder.BuildRouteConfigurations","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551016694654","endTimeUnixNano":"1735596551017745616","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"wRoHtbjfaz0=","parentSpanId":"AF2VbjcLF8o=","flags":256,"name":"envoyconfig.Builder.BuildListeners","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551016746665","endTimeUnixNano":"1735596551021080155","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"AF2VbjcLF8o=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"controlplane.Server.buildDiscoveryResources","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551016576533","endTimeUnixNano":"1735596551021622391","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/xds.go:23"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"pbntg72ezVo=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"controlplane.Server.OnConfigChange","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551021643761","endTimeUnixNano":"1735596551021656921","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/server.go:255"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"I5CVUhOnTU4=","parentSpanId":"lzi6B+5Q2nI=","flags":256,"name":"envoyconfig.Builder.BuildClusters","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551249315632","endTimeUnixNano":"1735596551249436133","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"zspRgxWRZco=","parentSpanId":"lzi6B+5Q2nI=","flags":256,"name":"envoyconfig.Builder.BuildRouteConfigurations","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551249235221","endTimeUnixNano":"1735596551249575035","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"lyCOdoUx1+Q=","parentSpanId":"lzi6B+5Q2nI=","flags":256,"name":"envoyconfig.Builder.BuildListeners","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551249249221","endTimeUnixNano":"1735596551250357424","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"lzi6B+5Q2nI=","parentSpanId":"jGekmqeQkJo=","flags":256,"name":"controlplane.Server.buildDiscoveryResources","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551249096239","endTimeUnixNano":"1735596551250474326","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/xds.go:23"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"jGekmqeQkJo=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"controlplane.Server.update","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551244810840","endTimeUnixNano":"1735596551250478946","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/server.go:279"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"gHqxUi5kePg=","parentSpanId":"xSJX1oetTdU=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552258874805","endTimeUnixNano":"1735596552261729827","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"WV8SOlkANhI=","parentSpanId":"3jlv7DJBi34=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552265934866","endTimeUnixNano":"1735596552271656323","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"yhyoDzh9RIM=","parentSpanId":"E6AUkZz0o+g=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552273575835","endTimeUnixNano":"1735596552276684431","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"RXv+G4LTsEk=","parentSpanId":"9BbvwqcIDE4=","flags":768,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552052610012","endTimeUnixNano":"1735596552052848066","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33088"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"+R7poiK6/239h1nEy5hIrA==","spanId":"wtBmI0TId5M=","parentSpanId":"ynwZv1w6sTA=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552061633468","endTimeUnixNano":"1735596552062922194","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"2"}}],"status":{"message":"authorize: error evaluating policy.rego: pomerium.policy:95: eval_builtin_error: get_databroker_record: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"BI0pXOVFSFDjDh4HlytjyA==","spanId":"CxZOOB7mGiQ=","parentSpanId":"7hYMiT5DOHg=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552065072369","endTimeUnixNano":"1735596552065842537","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"2"}}],"status":{"message":"authorize: error evaluating policy.rego: pomerium.policy:95: eval_builtin_error: get_databroker_record: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"BO49aXqcRX/rS1MR1IxpsA==","spanId":"2Z6YH7F78Lc=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552066447434","endTimeUnixNano":"1735596552066962551","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33104"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"nD9dzNOnTKOIJMxmlWZ9bw==","spanId":"lQvkuojiNVU=","parentSpanId":"sIew416JLXc=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552070481851","endTimeUnixNano":"1735596552071222610","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"2"}}],"status":{"message":"authorize: error evaluating policy.rego: pomerium.policy:95: eval_builtin_error: get_databroker_record: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"lCAPf0HR197f/cUT53zYCA==","spanId":"ITRzniYDlzA=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552071717826","endTimeUnixNano":"1735596552071903768","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33104"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"b0XQMNrGpRts3otatJ4VkQ==","spanId":"PNkHn7SfYNs=","parentSpanId":"poak1tCu/Ds=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552077900168","endTimeUnixNano":"1735596552078387493","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"2"}}],"status":{"message":"authorize: error evaluating policy.rego: pomerium.policy:95: eval_builtin_error: get_databroker_record: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"lCsEeGUuNYemOgQ6EAqw/Q==","spanId":"Xrh5E+qCpVc=","parentSpanId":"BI5T6olEsIc=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552087339348","endTimeUnixNano":"1735596552087950615","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"2"}}],"status":{"message":"authorize: error evaluating policy.rego: pomerium.policy:95: eval_builtin_error: get_databroker_record: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"oKHSD4aojA+P/UfWs5LSaQ==","spanId":"2zTy638Eq5k=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552088429270","endTimeUnixNano":"1735596552088637543","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33104"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"69JHBTXAPq7zDqo19nNt3g==","spanId":"nqbiIHn9gV8=","parentSpanId":"rlZX67RjnyY=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552097875770","endTimeUnixNano":"1735596552098442626","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"2"}}],"status":{"message":"authorize: error evaluating policy.rego: pomerium.policy:95: eval_builtin_error: get_databroker_record: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"V9c6M7AQm8lZF4prh3VxSA==","spanId":"yjD3ncvuvJs=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552098899903","endTimeUnixNano":"1735596552099043664","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33104"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"qGCrQnZvKm9AiV3UG+9y4A==","spanId":"7A4ZbdDiN2o=","parentSpanId":"rNEObiMfLz0=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552124101095","endTimeUnixNano":"1735596552124765313","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"2"}}],"status":{"message":"authorize: error evaluating policy.rego: pomerium.policy:95: eval_builtin_error: get_databroker_record: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"oboPZ53IIkU+AOAisyhydQ==","spanId":"13OK1T2dwk8=","parentSpanId":"ws87AD1cVDE=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552185178396","endTimeUnixNano":"1735596552185771603","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"2"}}],"status":{"message":"authorize: error evaluating policy.rego: pomerium.policy:95: eval_builtin_error: get_databroker_record: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"aoeEeDz4Wb1FDja90yAkEQ==","spanId":"W/Z9Zzkiag4=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552186283969","endTimeUnixNano":"1735596552186447471","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33104"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"anG2KFI46uE=","parentSpanId":"9BuTh+PJtJ8=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552250425405","endTimeUnixNano":"1735596552250672338","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33112"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"2ACwBEIpGZ8=","parentSpanId":"yOL5QyxGuD8=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552251440318","endTimeUnixNano":"1735596552251721171","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33112"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"XF9zGG11kRE=","parentSpanId":"xCIoDfKUEmA=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552249204022","endTimeUnixNano":"1735596552252813544","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"cOuoTnMUlEZjYEIVsCPbWg==","spanId":"6P4Ui3Adxa0=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552253489362","endTimeUnixNano":"1735596552253650093","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33104"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"BHkNMsuL5iw=","parentSpanId":"acI32UNM7h4=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552257718511","endTimeUnixNano":"1735596552258121035","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33128"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"HiRbcDL3iIRaCg/blwsIog==","spanId":"1jQJ0R0HrV0=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552262659298","endTimeUnixNano":"1735596552262815790","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33132"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"mDALe+Jrgfg=","parentSpanId":"E9PYtbtULsI=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552265119127","endTimeUnixNano":"1735596552265546902","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33128"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"6usly+cZRsM=","parentSpanId":"aovapvAPfh8=","flags":768,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552268677418","endTimeUnixNano":"1735596552268805120","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33136"}},{"key":"rpc.grpc.status_code","value":{"intValue":"5"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"8fH7m7exgCs=","parentSpanId":"suYSEohRyP8=","flags":768,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552269640899","endTimeUnixNano":"1735596552269823792","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33136"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"+JjftiqLXcg=","parentSpanId":"TwqTw1iB4X0=","flags":768,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552270621410","endTimeUnixNano":"1735596552270745532","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33136"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"McVjdOxONSgdrrlkDUeUQw==","spanId":"lfpJen6rCfQ=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552272272740","endTimeUnixNano":"1735596552272444392","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33132"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"DgnnjNfi9WQ=","parentSpanId":"lWFmfug8YNY=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552272779406","endTimeUnixNano":"1735596552273151110","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33128"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"gNVkkaQRfwg=","parentSpanId":"BCaHfYcFoYI=","flags":768,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552274111891","endTimeUnixNano":"1735596552274255303","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33136"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"SeDH8Vfpadn10kMa8kJvgw==","spanId":"bakD1gohUnw=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552279426554","endTimeUnixNano":"1735596552279568575","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33132"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"fBpfW6Ymuv8=","parentSpanId":"5ThhIloR5KY=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552279993020","endTimeUnixNano":"1735596552280404654","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"LZAG0fuKcRwFnH5cEXT4Ug==","spanId":"sWoD9kavf+E=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552281751630","endTimeUnixNano":"1735596552281986783","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33104"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"c1fn98QaXwo=","parentSpanId":"jF06vgXl+rY=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552282882924","endTimeUnixNano":"1735596552283086246","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33112"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"ImsV4SaMVV8=","parentSpanId":"Z0Z1SUx32F8=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552283799684","endTimeUnixNano":"1735596552283989316","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33112"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"cXrgy5S3krM=","parentSpanId":"N7fb6O62Wao=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552284745205","endTimeUnixNano":"1735596552284901788","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33112"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"pbTnTWBDo/c=","parentSpanId":"lTFYsgKhSFo=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552285570725","endTimeUnixNano":"1735596552285821128","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33112"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"Cn7Zb+Z3DFs=","parentSpanId":"6LX3bJmVP78=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552282117174","endTimeUnixNano":"1735596552286492065","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"L9z/Dey1wDyGsrGbe0xbKw==","spanId":"Qxa3VIfADAY=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552287723980","endTimeUnixNano":"1735596552287857581","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33104"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"UPC6q/yfV2k=","parentSpanId":"YjiylRy7lCw=","flags":768,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552053704325","endTimeUnixNano":"1735596552315825317","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}},{"traceId":"eX1+Nwu+QvGR+nz/98XwWw==","spanId":"jYp+vYBUjRs=","flags":768,"name":"envoy.service.accesslog.v3.AccessLogService/StreamAccessLogs","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552364472913","endTimeUnixNano":"1735596552373285326","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.accesslog.v3.AccessLogService"}},{"key":"rpc.method","value":{"stringValue":"StreamAccessLogs"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}},{"traceId":"QHt9ukn/IblV67PlcrHm/w==","spanId":"iyuOnhdATnw=","flags":768,"name":"envoy.service.discovery.v3.AggregatedDiscoveryService/DeltaAggregatedResources","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596551308452351","endTimeUnixNano":"1735596552381932417","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.discovery.v3.AggregatedDiscoveryService"}},{"key":"rpc.method","value":{"stringValue":"DeltaAggregatedResources"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33086"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp","version":"0.57.0"},"spans":[{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"xSJX1oetTdU=","parentSpanId":"FBPvHWyFn6I=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552258850984","endTimeUnixNano":"1735596552261747438","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"35065"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"59410"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"975"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"3jlv7DJBi34=","parentSpanId":"qZJUemZrh0U=","flags":768,"name":"Server: GET /oauth2/callback","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552265916216","endTimeUnixNano":"1735596552271673263","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"35065"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"59410"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/oauth2/callback"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"443"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"E6AUkZz0o+g=","parentSpanId":"uuvSf1QFkxA=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552273553145","endTimeUnixNano":"1735596552276695981","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"35065"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"59410"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1237"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"FUiwUhbfpO0=","parentSpanId":"LNr+ATKO4qk=","flags":768,"name":"Server: GET /","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552281027002","endTimeUnixNano":"1735596552281141204","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"35065"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"59418"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/callback/"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"66"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, + {"timestamp":"2024-12-30T22:09:12.402575956Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/authenticate.go:54"}},{"key":"service.name","value":{"stringValue":"Authenticate"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp","version":"0.57.0"},"spans":[{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"+fYq7JMxmug=","parentSpanId":"gHqxUi5kePg=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552259041196","endTimeUnixNano":"1735596552260746875","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:43581/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"43581"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"uhrN3Ns9n/w=","parentSpanId":"L6h65KKHBOM=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552266052297","endTimeUnixNano":"1735596552266350501","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:43581/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"43581"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"dwMPQOqC63eLDuFdJ/DRFg==","spanId":"BHBu4awlz/Y=","flags":256,"name":"OAuth2 Client: GET /.well-known/jwks.json","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552266984739","endTimeUnixNano":"1735596552267287902","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:43581/.well-known/jwks.json"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"43581"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"237"}}],"status":{}}]},{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"5JQnjvCb/mM=","parentSpanId":"gHqxUi5kePg=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552258982826","endTimeUnixNano":"1735596552261712807","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"L6h65KKHBOM=","parentSpanId":"WV8SOlkANhI=","flags":256,"name":"authenticate.getOAuthCallback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552265984307","endTimeUnixNano":"1735596552271050666","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:327"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"QVdbJ8Izbi8=","parentSpanId":"uyDfS9VMw2E=","flags":256,"name":"authenticate.SignIn","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552274529026","endTimeUnixNano":"1735596552276674161","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:162"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"uyDfS9VMw2E=","parentSpanId":"yhyoDzh9RIM=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552273718127","endTimeUnixNano":"1735596552276680921","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"aovapvAPfh8=","parentSpanId":"L6h65KKHBOM=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552267935169","endTimeUnixNano":"1735596552269118754","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"events":[{"timeUnixNano":"1735596552269116524","name":"status code: NotFound"}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"suYSEohRyP8=","parentSpanId":"L6h65KKHBOM=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552269284836","endTimeUnixNano":"1735596552270049974","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"TwqTw1iB4X0=","parentSpanId":"L6h65KKHBOM=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552270313067","endTimeUnixNano":"1735596552270952515","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"BCaHfYcFoYI=","parentSpanId":"uyDfS9VMw2E=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552273809458","endTimeUnixNano":"1735596552274482325","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, + {"timestamp":"2024-12-30T22:09:12.403360776Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/authorize/authorize.go:48"}},{"key":"service.name","value":{"stringValue":"Authorize"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"+R7poiK6/239h1nEy5hIrA==","spanId":"I+5+jfwGURc=","parentSpanId":"oPLxRoHsmJg=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552062084883","endTimeUnixNano":"1735596552062226725","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"+R7poiK6/239h1nEy5hIrA==","spanId":"oHv+S87aeOg=","parentSpanId":"Ypo2QQUZDgs=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552062572019","endTimeUnixNano":"1735596552062849073","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"+R7poiK6/239h1nEy5hIrA==","spanId":"Ypo2QQUZDgs=","parentSpanId":"oPLxRoHsmJg=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552062174334","endTimeUnixNano":"1735596552062874583","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"+R7poiK6/239h1nEy5hIrA==","spanId":"oPLxRoHsmJg=","parentSpanId":"702+hX/l24A=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552062022603","endTimeUnixNano":"1735596552062885453","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"+R7poiK6/239h1nEy5hIrA==","spanId":"702+hX/l24A=","parentSpanId":"wtBmI0TId5M=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552061969882","endTimeUnixNano":"1735596552062891643","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"BI0pXOVFSFDjDh4HlytjyA==","spanId":"PR3eGT+sQgk=","parentSpanId":"PXxYHizXWvY=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552065245471","endTimeUnixNano":"1735596552065372002","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"BI0pXOVFSFDjDh4HlytjyA==","spanId":"pKeK3Ol+An8=","parentSpanId":"ImM7s8/+0MA=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552065533264","endTimeUnixNano":"1735596552065780997","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"BI0pXOVFSFDjDh4HlytjyA==","spanId":"ImM7s8/+0MA=","parentSpanId":"PXxYHizXWvY=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552065273211","endTimeUnixNano":"1735596552065795577","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"BI0pXOVFSFDjDh4HlytjyA==","spanId":"PXxYHizXWvY=","parentSpanId":"rIEVxXFXkKg=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552065202010","endTimeUnixNano":"1735596552065806667","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"BI0pXOVFSFDjDh4HlytjyA==","spanId":"rIEVxXFXkKg=","parentSpanId":"CxZOOB7mGiQ=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552065158500","endTimeUnixNano":"1735596552065812947","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"nD9dzNOnTKOIJMxmlWZ9bw==","spanId":"QF0H6wiWXJM=","parentSpanId":"E9u3b8PmIoI=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552070634144","endTimeUnixNano":"1735596552070784215","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"nD9dzNOnTKOIJMxmlWZ9bw==","spanId":"kZ1uVCaZwk4=","parentSpanId":"PROD01WEVLg=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552070938967","endTimeUnixNano":"1735596552071153419","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"nD9dzNOnTKOIJMxmlWZ9bw==","spanId":"PROD01WEVLg=","parentSpanId":"E9u3b8PmIoI=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552070650504","endTimeUnixNano":"1735596552071170890","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"nD9dzNOnTKOIJMxmlWZ9bw==","spanId":"E9u3b8PmIoI=","parentSpanId":"mCghMmDIxcc=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552070580902","endTimeUnixNano":"1735596552071188160","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"nD9dzNOnTKOIJMxmlWZ9bw==","spanId":"mCghMmDIxcc=","parentSpanId":"lQvkuojiNVU=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552070547872","endTimeUnixNano":"1735596552071194050","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"b0XQMNrGpRts3otatJ4VkQ==","spanId":"QKLoYXMX29E=","parentSpanId":"xmjkj3o7dfk=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552078006669","endTimeUnixNano":"1735596552078093980","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"b0XQMNrGpRts3otatJ4VkQ==","spanId":"sryg5n/tnBQ=","parentSpanId":"4XomFRf7+kk=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552078190671","endTimeUnixNano":"1735596552078339492","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"b0XQMNrGpRts3otatJ4VkQ==","spanId":"4XomFRf7+kk=","parentSpanId":"xmjkj3o7dfk=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552078028279","endTimeUnixNano":"1735596552078349673","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"b0XQMNrGpRts3otatJ4VkQ==","spanId":"xmjkj3o7dfk=","parentSpanId":"/g6I0n43z0I=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552077980928","endTimeUnixNano":"1735596552078357263","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"b0XQMNrGpRts3otatJ4VkQ==","spanId":"/g6I0n43z0I=","parentSpanId":"PNkHn7SfYNs=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552077957258","endTimeUnixNano":"1735596552078362603","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"lCsEeGUuNYemOgQ6EAqw/Q==","spanId":"Zr/GT/Zu/ho=","parentSpanId":"tDxPozCLZ/s=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552087494559","endTimeUnixNano":"1735596552087633661","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"lCsEeGUuNYemOgQ6EAqw/Q==","spanId":"bDyRo8kOqOc=","parentSpanId":"Isdgl5r8KSs=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552087744612","endTimeUnixNano":"1735596552087903974","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"lCsEeGUuNYemOgQ6EAqw/Q==","spanId":"Isdgl5r8KSs=","parentSpanId":"tDxPozCLZ/s=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552087526160","endTimeUnixNano":"1735596552087915934","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"lCsEeGUuNYemOgQ6EAqw/Q==","spanId":"tDxPozCLZ/s=","parentSpanId":"whiBKTR+R7g=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552087458459","endTimeUnixNano":"1735596552087922914","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"lCsEeGUuNYemOgQ6EAqw/Q==","spanId":"whiBKTR+R7g=","parentSpanId":"Xrh5E+qCpVc=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552087426029","endTimeUnixNano":"1735596552087927814","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"69JHBTXAPq7zDqo19nNt3g==","spanId":"OHqb77VeJE4=","parentSpanId":"GlBfgb5ewww=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552098031612","endTimeUnixNano":"1735596552098165633","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"69JHBTXAPq7zDqo19nNt3g==","spanId":"eZVpM/28z1o=","parentSpanId":"UE5vNgfZ9Jk=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552098241214","endTimeUnixNano":"1735596552098396226","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"69JHBTXAPq7zDqo19nNt3g==","spanId":"UE5vNgfZ9Jk=","parentSpanId":"GlBfgb5ewww=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552098053802","endTimeUnixNano":"1735596552098407726","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"69JHBTXAPq7zDqo19nNt3g==","spanId":"GlBfgb5ewww=","parentSpanId":"kqNihdBkVcY=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552097995401","endTimeUnixNano":"1735596552098414976","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"69JHBTXAPq7zDqo19nNt3g==","spanId":"kqNihdBkVcY=","parentSpanId":"nqbiIHn9gV8=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552097959031","endTimeUnixNano":"1735596552098419756","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"qGCrQnZvKm9AiV3UG+9y4A==","spanId":"LFUxOjlfg/8=","parentSpanId":"JgeqIDcDVXc=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552124249976","endTimeUnixNano":"1735596552124354848","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"qGCrQnZvKm9AiV3UG+9y4A==","spanId":"VP53hCeHVAc=","parentSpanId":"pW/aWnlh3S8=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552124480579","endTimeUnixNano":"1735596552124709162","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"qGCrQnZvKm9AiV3UG+9y4A==","spanId":"pW/aWnlh3S8=","parentSpanId":"JgeqIDcDVXc=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552124259696","endTimeUnixNano":"1735596552124725643","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"qGCrQnZvKm9AiV3UG+9y4A==","spanId":"JgeqIDcDVXc=","parentSpanId":"3UY1EGRVKNE=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552124215096","endTimeUnixNano":"1735596552124734763","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"qGCrQnZvKm9AiV3UG+9y4A==","spanId":"3UY1EGRVKNE=","parentSpanId":"7A4ZbdDiN2o=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552124173605","endTimeUnixNano":"1735596552124740303","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"oboPZ53IIkU+AOAisyhydQ==","spanId":"HtUTSubmmrE=","parentSpanId":"ovY10LzBQFY=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552185324238","endTimeUnixNano":"1735596552185421339","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"oboPZ53IIkU+AOAisyhydQ==","spanId":"xyxreo92b3g=","parentSpanId":"QF+l8Wz96YQ=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552185523760","endTimeUnixNano":"1735596552185722773","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"oboPZ53IIkU+AOAisyhydQ==","spanId":"QF+l8Wz96YQ=","parentSpanId":"ovY10LzBQFY=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552185350058","endTimeUnixNano":"1735596552185733963","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"oboPZ53IIkU+AOAisyhydQ==","spanId":"ovY10LzBQFY=","parentSpanId":"Fnf7AryD+WI=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552185290468","endTimeUnixNano":"1735596552185741623","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"oboPZ53IIkU+AOAisyhydQ==","spanId":"Fnf7AryD+WI=","parentSpanId":"13OK1T2dwk8=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552185261027","endTimeUnixNano":"1735596552185746723","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"NUZgjI72agI=","parentSpanId":"3V6E9VZ9lFw=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552249348673","endTimeUnixNano":"1735596552249456885","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"cR6yd0zNkYs=","parentSpanId":"9V8ar0k7EFA=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552249548196","endTimeUnixNano":"1735596552251056403","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"GTgvZBQ068Q=","parentSpanId":"9V8ar0k7EFA=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552251076964","endTimeUnixNano":"1735596552251972654","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"wDgeqdCQypw=","parentSpanId":"9V8ar0k7EFA=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552251988344","endTimeUnixNano":"1735596552252010944","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"9V8ar0k7EFA=","parentSpanId":"3V6E9VZ9lFw=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552249386184","endTimeUnixNano":"1735596552252399628","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"3V6E9VZ9lFw=","parentSpanId":"HcyMtZcnRq4=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552249305223","endTimeUnixNano":"1735596552252409978","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"3DSL89VAypc=","parentSpanId":"HcyMtZcnRq4=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552252639201","endTimeUnixNano":"1735596552252687922","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:28"}},{"key":"result.allow","value":{"boolValue":false}},{"key":"result.allow-why-false","value":{"arrayValue":{"values":[{"stringValue":"user-unauthenticated"}]}}},{"key":"result.deny-why-false","value":{"arrayValue":{}}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"HcyMtZcnRq4=","parentSpanId":"XF9zGG11kRE=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552249274383","endTimeUnixNano":"1735596552252697352","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"g2F4uQwPi5I=","parentSpanId":"kVygGFO0Xw0=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552257879962","endTimeUnixNano":"1735596552257960503","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"kVygGFO0Xw0=","parentSpanId":"I0JWe90UcQ8=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552257851872","endTimeUnixNano":"1735596552257973664","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"rlSFOyl7ozs=","parentSpanId":"I0JWe90UcQ8=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552257985584","endTimeUnixNano":"1735596552258019594","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:28"}},{"key":"result.allow","value":{"boolValue":true}},{"key":"result.allow-why-true","value":{"arrayValue":{"values":[{"stringValue":"pomerium-route"}]}}},{"key":"result.deny-why-false","value":{"arrayValue":{}}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"I0JWe90UcQ8=","parentSpanId":"BHkNMsuL5iw=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552257797392","endTimeUnixNano":"1735596552258026874","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"usCxFBdA74o=","parentSpanId":"Oouk7UHsCu4=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552265322909","endTimeUnixNano":"1735596552265450551","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"Oouk7UHsCu4=","parentSpanId":"IAGqRFX/cVs=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552265289779","endTimeUnixNano":"1735596552265464871","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"24K8qH0UQNo=","parentSpanId":"IAGqRFX/cVs=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552265470721","endTimeUnixNano":"1735596552265504361","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:28"}},{"key":"result.allow","value":{"boolValue":true}},{"key":"result.allow-why-true","value":{"arrayValue":{"values":[{"stringValue":"pomerium-route"}]}}},{"key":"result.deny-why-false","value":{"arrayValue":{}}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"IAGqRFX/cVs=","parentSpanId":"mDALe+Jrgfg=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552265228868","endTimeUnixNano":"1735596552265511791","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"TIdzDOPaZa4=","parentSpanId":"JQr00bdtY6g=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552272943178","endTimeUnixNano":"1735596552273045829","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"JQr00bdtY6g=","parentSpanId":"eG//PYnRynA=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552272917778","endTimeUnixNano":"1735596552273059780","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"25vcPv/qqO8=","parentSpanId":"eG//PYnRynA=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552273064050","endTimeUnixNano":"1735596552273110010","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:28"}},{"key":"result.allow","value":{"boolValue":true}},{"key":"result.allow-why-true","value":{"arrayValue":{"values":[{"stringValue":"pomerium-route"}]}}},{"key":"result.deny-why-false","value":{"arrayValue":{}}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"eG//PYnRynA=","parentSpanId":"DgnnjNfi9WQ=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552272856047","endTimeUnixNano":"1735596552273116970","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"2ufRRaRmOpg=","parentSpanId":"yedpQsjuMek=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552280207462","endTimeUnixNano":"1735596552280308813","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"yedpQsjuMek=","parentSpanId":"IDC3HSZb0BE=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552280170692","endTimeUnixNano":"1735596552280322033","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"ze+U8DOBf9o=","parentSpanId":"IDC3HSZb0BE=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552280327954","endTimeUnixNano":"1735596552280359674","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:28"}},{"key":"result.allow","value":{"boolValue":true}},{"key":"result.allow-why-true","value":{"arrayValue":{"values":[{"stringValue":"pomerium-route"}]}}},{"key":"result.deny-why-false","value":{"arrayValue":{}}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"IDC3HSZb0BE=","parentSpanId":"fBpfW6Ymuv8=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552280065711","endTimeUnixNano":"1735596552280366684","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"xgQ5Ms1BPbY=","parentSpanId":"J4x6xb3qh2c=","flags":256,"name":"authorize.getDataBrokerSessionOrServiceAccount","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552282372287","endTimeUnixNano":"1735596552283419570","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/databroker.go:64"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"HbnRXI9BK14=","parentSpanId":"J4x6xb3qh2c=","flags":256,"name":"authorize.getDataBrokerUser","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552283422410","endTimeUnixNano":"1735596552284305360","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/databroker.go:97"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"ZhLEljBrikk=","parentSpanId":"dqbGwdVFBdI=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552284669214","endTimeUnixNano":"1735596552285248751","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"737ae166-e045-403d-bceb-7fdc794b69e7\""}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"2KN++KWW5po=","parentSpanId":"dqbGwdVFBdI=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552285276202","endTimeUnixNano":"1735596552285531025","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"737ae166-e045-403d-bceb-7fdc794b69e7\""}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"NLQFh6R6vYM=","parentSpanId":"dqbGwdVFBdI=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552285631146","endTimeUnixNano":"1735596552285755157","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"737ae166-e045-403d-bceb-7fdc794b69e7\""}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"YzW0uskC6VA=","parentSpanId":"dqbGwdVFBdI=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552285800058","endTimeUnixNano":"1735596552285901849","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.User\""}},{"key":"record_id","value":{"stringValue":"\"a4f74022-cf7e-4ecf-a7f8-d655185fe980\""}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"6G9M65neegc=","parentSpanId":"xW0iCPJJO4s=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552284351620","endTimeUnixNano":"1735596552286220182","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"dqbGwdVFBdI=","parentSpanId":"xW0iCPJJO4s=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552284373821","endTimeUnixNano":"1735596552286388584","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"xW0iCPJJO4s=","parentSpanId":"J4x6xb3qh2c=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552284317990","endTimeUnixNano":"1735596552286402824","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"3HTg2TiSq+E=","parentSpanId":"J4x6xb3qh2c=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552286408794","endTimeUnixNano":"1735596552286448825","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:28"}},{"key":"result.allow","value":{"boolValue":true}},{"key":"result.allow-why-true","value":{"arrayValue":{"values":[{"stringValue":"email-ok"}]}}},{"key":"result.deny-why-false","value":{"arrayValue":{}}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"J4x6xb3qh2c=","parentSpanId":"Cn7Zb+Z3DFs=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552282210506","endTimeUnixNano":"1735596552286456045","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"+R7poiK6/239h1nEy5hIrA==","spanId":"ZitEHOlhYmg=","parentSpanId":"oHv+S87aeOg=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552062721421","endTimeUnixNano":"1735596552062749752","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"BI0pXOVFSFDjDh4HlytjyA==","spanId":"H2Bi92mnl38=","parentSpanId":"pKeK3Ol+An8=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552065647705","endTimeUnixNano":"1735596552065677235","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"nD9dzNOnTKOIJMxmlWZ9bw==","spanId":"Kt4NyLWXBNc=","parentSpanId":"kZ1uVCaZwk4=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552071026898","endTimeUnixNano":"1735596552071051688","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"b0XQMNrGpRts3otatJ4VkQ==","spanId":"U1A1AlTpBik=","parentSpanId":"sryg5n/tnBQ=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552078251041","endTimeUnixNano":"1735596552078268392","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"lCsEeGUuNYemOgQ6EAqw/Q==","spanId":"8TN7lnb8ZZw=","parentSpanId":"bDyRo8kOqOc=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552087812363","endTimeUnixNano":"1735596552087829773","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"69JHBTXAPq7zDqo19nNt3g==","spanId":"MdCipzjfzDA=","parentSpanId":"eZVpM/28z1o=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552098306135","endTimeUnixNano":"1735596552098323105","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"qGCrQnZvKm9AiV3UG+9y4A==","spanId":"bSnSgvrRc9U=","parentSpanId":"VP53hCeHVAc=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552124561060","endTimeUnixNano":"1735596552124582680","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"oboPZ53IIkU+AOAisyhydQ==","spanId":"K0uogfkk0I8=","parentSpanId":"xyxreo92b3g=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552185615441","endTimeUnixNano":"1735596552185636552","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"9BuTh+PJtJ8=","parentSpanId":"cR6yd0zNkYs=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552249646597","endTimeUnixNano":"1735596552251004663","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"yOL5QyxGuD8=","parentSpanId":"GTgvZBQ068Q=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552251131154","endTimeUnixNano":"1735596552251944683","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"jF06vgXl+rY=","parentSpanId":"xgQ5Ms1BPbY=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552282435188","endTimeUnixNano":"1735596552283362919","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"Z0Z1SUx32F8=","parentSpanId":"HbnRXI9BK14=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552283482131","endTimeUnixNano":"1735596552284245699","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"N7fb6O62Wao=","parentSpanId":"6G9M65neegc=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552284432481","endTimeUnixNano":"1735596552285131900","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"lTFYsgKhSFo=","parentSpanId":"6G9M65neegc=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552285250591","endTimeUnixNano":"1735596552286064931","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, + {"timestamp":"2024-12-30T22:09:12.404074234Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/databroker/cache.go:94"}},{"key":"service.name","value":{"stringValue":"Data Broker"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"3Jv9+gNiTBE=","parentSpanId":"WBoo1gQHvyI=","flags":256,"name":"databroker.grpc.AcquireLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551247776624","endTimeUnixNano":"1735596551248251260","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:83"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"FnISXIhjzvI=","parentSpanId":"oDKmK3j0BGo=","flags":256,"name":"databroker.grpc.SyncLatest","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551249094219","endTimeUnixNano":"1735596551249277271","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:393"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"uDaOHkb40p4=","parentSpanId":"MpsYB/n8HZU=","flags":256,"name":"databroker.grpc.SyncLatest","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551249161070","endTimeUnixNano":"1735596551249326972","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:393"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"9co6PIOIWWk=","parentSpanId":"oAf8R1Tzhpo=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551749629896","endTimeUnixNano":"1735596551749642886","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"I2+UHD0llFE=","parentSpanId":"RXv+G4LTsEk=","flags":256,"name":"databroker.grpc.SyncLatest","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552052732244","endTimeUnixNano":"1735596552052829356","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:393"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"mexE3YgRss0=","parentSpanId":"uxgVY6jCPOw=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552249206962","endTimeUnixNano":"1735596552249222362","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"RpB80gUAJnc=","parentSpanId":"anG2KFI46uE=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552250516206","endTimeUnixNano":"1735596552250616108","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"lFfHkwRj0N4=","parentSpanId":"2ACwBEIpGZ8=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552251564179","endTimeUnixNano":"1735596552251685920","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"anVXyj3g0y0=","parentSpanId":"6usly+cZRsM=","flags":256,"name":"databroker.grpc.Get","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552268754989","endTimeUnixNano":"1735596552268775190","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:110"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"XG+fN6jKD2M=","parentSpanId":"8fH7m7exgCs=","flags":256,"name":"databroker.grpc.Put","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552269719780","endTimeUnixNano":"1735596552269761311","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:207"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"I7vcpKXSaL0=","parentSpanId":"+JjftiqLXcg=","flags":256,"name":"databroker.grpc.Put","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552270683971","endTimeUnixNano":"1735596552270718502","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:207"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"WxNK3s1U+kc=","parentSpanId":"gNVkkaQRfwg=","flags":256,"name":"databroker.grpc.Get","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552274169042","endTimeUnixNano":"1735596552274201452","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:110"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"eRRiR8mb/Zk=","parentSpanId":"c1fn98QaXwo=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552282971495","endTimeUnixNano":"1735596552283056846","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"EyiHMzQqpYI=","parentSpanId":"ImsV4SaMVV8=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552283885275","endTimeUnixNano":"1735596552283961596","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"yLyVXa2MkYM=","parentSpanId":"cXrgy5S3krM=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552284821237","endTimeUnixNano":"1735596552284876897","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"E9KYOI4Enow=","parentSpanId":"pbTnTWBDo/c=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552285707277","endTimeUnixNano":"1735596552285789157","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"6ZPe6wSwVqs=","parentSpanId":"UPC6q/yfV2k=","flags":256,"name":"databroker.grpc.Sync","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552053809077","endTimeUnixNano":"1735596552315804667","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:355"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"KIpwWMH3RYw=","parentSpanId":"INZmZYN0zYM=","flags":256,"name":"databroker.grpc.Sync","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551249794687","endTimeUnixNano":"1735596552399213398","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:355"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"5Xc5hfznIok=","parentSpanId":"RK6oHqmnMlg=","flags":256,"name":"databroker.grpc.Sync","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551249850358","endTimeUnixNano":"1735596552399227388","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:355"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"WBoo1gQHvyI=","parentSpanId":"lpAHqgla4lk=","flags":768,"name":"databroker.DataBrokerService/AcquireLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596551247120936","endTimeUnixNano":"1735596551248383661","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"AcquireLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"50506"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"lpAHqgla4lk=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/AcquireLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596551244933821","endTimeUnixNano":"1735596551248572994","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"AcquireLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"35089"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"oDKmK3j0BGo=","parentSpanId":"3NhMqufhs3E=","flags":768,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596551249023349","endTimeUnixNano":"1735596551249292572","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"50506"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"MpsYB/n8HZU=","parentSpanId":"JfRLTLgutnU=","flags":768,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596551249053529","endTimeUnixNano":"1735596551249333762","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"50506"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"3NhMqufhs3E=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596551248774936","endTimeUnixNano":"1735596551249483274","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"35089"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"JfRLTLgutnU=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596551248781656","endTimeUnixNano":"1735596551249500274","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"35089"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"oAf8R1Tzhpo=","parentSpanId":"zZeFpq0lf1U=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596551749484304","endTimeUnixNano":"1735596551749700567","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"50506"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"zZeFpq0lf1U=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596551749149160","endTimeUnixNano":"1735596551749836088","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"35089"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"uxgVY6jCPOw=","parentSpanId":"Nhvnj8qzYVg=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552249128171","endTimeUnixNano":"1735596552249248462","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"50506"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"Nhvnj8qzYVg=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552248935569","endTimeUnixNano":"1735596552249348583","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"35089"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"LAE5i/Gu+f8=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596551249562874","endTimeUnixNano":"1735596552399071366","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"35089"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{"message":"context canceled","code":"STATUS_CODE_ERROR"}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"erfJ2NXx8k8=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596551249581905","endTimeUnixNano":"1735596552399145197","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"35089"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{"message":"context canceled","code":"STATUS_CODE_ERROR"}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"INZmZYN0zYM=","parentSpanId":"LAE5i/Gu+f8=","flags":768,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596551249734266","endTimeUnixNano":"1735596552399244838","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"50506"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"RK6oHqmnMlg=","parentSpanId":"erfJ2NXx8k8=","flags":768,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596551249787427","endTimeUnixNano":"1735596552399254019","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"50506"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"uzNAHYDfDE4=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/ReleaseLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552399284209","endTimeUnixNano":"1735596552399551322","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"ReleaseLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:35089: connect: connection refused\"","code":"STATUS_CODE_ERROR"}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}} +] diff --git a/internal/telemetry/trace/trace.go b/internal/telemetry/trace/trace.go index 95099050f..d3c973c13 100644 --- a/internal/telemetry/trace/trace.go +++ b/internal/telemetry/trace/trace.go @@ -2,87 +2,238 @@ package trace import ( "context" + "errors" "fmt" - "net/url" + "runtime" + "sync" + "sync/atomic" + "time" - octrace "go.opencensus.io/trace" - - "github.com/pomerium/pomerium/internal/log" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" + coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" ) -const ( - // DatadogTracingProviderName is the name of the tracing provider Datadog. - DatadogTracingProviderName = "datadog" - // JaegerTracingProviderName is the name of the tracing provider Jaeger. - JaegerTracingProviderName = "jaeger" - // ZipkinTracingProviderName is the name of the tracing provider Zipkin. - ZipkinTracingProviderName = "zipkin" -) - -// Provider is a trace provider. -type Provider interface { - Register(options *TracingOptions) error - Unregister() error +type Options struct { + DebugFlags DebugFlags + RemoteClient otlptrace.Client } -// TracingOptions contains the configurations settings for a http server. -type TracingOptions struct { - // Shared - Provider string - Service string - Debug bool - - // Datadog - DatadogAddress string - - // Jaeger - - // CollectorEndpoint is the full url to the Jaeger HTTP Thrift collector. - // For example, http://localhost:14268/api/traces - JaegerCollectorEndpoint *url.URL - // AgentEndpoint instructs exporter to send spans to jaeger-agent at this address. - // For example, localhost:6831. - JaegerAgentEndpoint string - - // Zipkin - - // ZipkinEndpoint configures the zipkin collector URI - // Example: http://zipkin:9411/api/v2/spans - ZipkinEndpoint *url.URL - - // SampleRate is percentage of requests which are sampled - SampleRate float64 -} - -// Enabled indicates whether tracing is enabled on a given TracingOptions -func (t *TracingOptions) Enabled() bool { - return t.Provider != "" -} - -// GetProvider creates a new trace provider from TracingOptions. -func GetProvider(opts *TracingOptions) (Provider, error) { - var provider Provider - switch opts.Provider { - case DatadogTracingProviderName: - provider = new(datadogProvider) - case JaegerTracingProviderName: - provider = new(jaegerProvider) - case ZipkinTracingProviderName: - provider = new(zipkinProvider) - default: - return nil, fmt.Errorf("telemetry/trace: provider %s unknown", opts.Provider) +func (op Options) NewContext(parent context.Context) context.Context { + if systemContextFromContext(parent) != nil { + panic("parent already contains trace system context") } - octrace.ApplyConfig(octrace.Config{DefaultSampler: octrace.ProbabilitySampler(opts.SampleRate)}) - - log.Debug().Interface("Opts", opts).Msg("telemetry/trace: provider created") - return provider, nil + if op.RemoteClient == nil { + op.RemoteClient = NewRemoteClientFromEnv() + } + sys := &systemContext{ + options: op, + tpm: &tracerProviderManager{}, + } + ctx := context.WithValue(parent, systemContextKey, sys) + sys.exporterServer = NewServer(ctx, op.RemoteClient) + sys.exporterServer.Start(ctx) + return ctx } -// StartSpan starts a new child span of the current span in the context. If -// there is no span in the context, creates a new trace and span. +// NewContext creates a new top-level background context with tracing machinery +// and configuration that will be used when creating new tracer providers. // -// Returned context contains the newly created span. You can use it to -// propagate the returned span in process. -func StartSpan(ctx context.Context, name string, o ...octrace.StartOption) (context.Context, *octrace.Span) { - return octrace.StartSpan(ctx, name, o...) +// Any context created with NewContext should eventually be shut down by calling +// [ShutdownContext] to ensure all traces are exported. +// +// The parent context should be context.Background(), or a background context +// containing a logger. If any context in the parent's hierarchy was created +// by NewContext, this will panic. +func NewContext(parent context.Context) context.Context { + return Options{}.NewContext(parent) +} + +// NewTracerProvider creates a new [trace.TracerProvider] with the given service +// name and options. +// +// A context returned by [NewContext] must exist somewhere in the hierarchy of +// ctx, otherwise a no-op TracerProvider is returned. The configuration embedded +// within that context will be used to configure its resource attributes and +// exporter automatically. +func NewTracerProvider(ctx context.Context, serviceName string, opts ...sdktrace.TracerProviderOption) trace.TracerProvider { + sys := systemContextFromContext(ctx) + if sys == nil { + return noop.NewTracerProvider() + } + _, file, line, _ := runtime.Caller(1) + exp, err := otlptrace.New(ctx, sys.exporterServer.NewClient()) + if err != nil { + panic(err) + } + r, err := resource.Merge( + resource.Default(), + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceName(serviceName), + attribute.String("provider.created_at", fmt.Sprintf("%s:%d", file, line)), + ), + ) + if err != nil { + panic(err) + } + options := []sdktrace.TracerProviderOption{} + if sys.options.DebugFlags.Check(TrackSpanCallers) { + options = append(options, sdktrace.WithSpanProcessor(&stackTraceProcessor{})) + } + options = append(append(options, + sdktrace.WithBatcher(exp), + sdktrace.WithResource(r), + ), opts...) + for _, proc := range sys.exporterServer.SpanProcessors() { + options = append(options, sdktrace.WithSpanProcessor(proc)) + } + tp := sdktrace.NewTracerProvider(options...) + sys.tpm.Add(tp) + return tp +} + +// Continue starts a new span using the tracer provider of the span in the given +// context. +// +// In most cases, it is better to start spans directly from a specific tracer, +// obtained via dependency injection or some other mechanism. This function is +// useful in shared code where the tracer used to start the span is not +// necessarily the same every time, but can change based on the call site. +func Continue(ctx context.Context, name string, o ...trace.SpanStartOption) (context.Context, trace.Span) { + return trace.SpanFromContext(ctx). + TracerProvider(). + Tracer(PomeriumCoreTracer). + Start(ctx, name, o...) +} + +// ShutdownContext will gracefully shut down all tracing resources created with +// a context returned by [NewContext], including all tracer providers and the +// underlying exporter and remote client. +// +// This should only be called once before exiting, but subsequent calls are +// a no-op. +// +// The provided context does not necessarily need to be the exact context +// returned by [NewContext]; it can be anywhere in its context hierarchy and +// this function will have the same effect. +func ShutdownContext(ctx context.Context) error { + sys := systemContextFromContext(ctx) + if sys == nil { + panic("context was not created with trace.NewContext") + } + + if !sys.shutdown.CompareAndSwap(false, true) { + return nil + } + + var errs []error + if err := sys.tpm.ShutdownAll(context.Background()); err != nil { + errs = append(errs, fmt.Errorf("error shutting down tracer providers: %w", err)) + } + if err := sys.exporterServer.Shutdown(context.Background()); err != nil { + errs = append(errs, fmt.Errorf("error shutting down trace exporter: %w", err)) + } + return errors.Join(errs...) +} + +func ExporterServerFromContext(ctx context.Context) coltracepb.TraceServiceServer { + if sys := systemContextFromContext(ctx); sys != nil { + return sys.exporterServer + } + return nil +} + +func RemoteClientFromContext(ctx context.Context) otlptrace.Client { + if sys := systemContextFromContext(ctx); sys != nil { + return sys.options.RemoteClient + } + return nil +} + +func DebugFlagsFromContext(ctx context.Context) DebugFlags { + if sys := systemContextFromContext(ctx); sys != nil { + return sys.options.DebugFlags + } + return 0 +} + +// WaitForSpans will block up to the given max duration and wait for all +// in-flight spans from tracers created with the given context to end. This +// function can be called more than once, and is safe to call from multiple +// goroutines in parallel. +// +// This requires the [TrackSpanReferences] debug flag to have been set with +// [Options.NewContext]. Otherwise, this function is a no-op and will return +// immediately. +// +// If this function blocks for more than 10 seconds, it will print a warning +// to stderr containing a list of span IDs it is waiting for, and the IDs of +// their parents (if known). Additionally, if the [TrackAllSpans] debug flag +// is set, details about parent spans will be displayed, including call site +// and trace ID. +func WaitForSpans(ctx context.Context, maxDuration time.Duration) error { + if sys := systemContextFromContext(ctx); sys != nil { + return sys.exporterServer.spanExportQueue.WaitForSpans(maxDuration) + } + return nil +} + +// ForceFlush immediately exports all spans that have not yet been exported for +// all tracer providers created using the given context. +func ForceFlush(ctx context.Context) error { + if sys := systemContextFromContext(ctx); sys != nil { + var errs []error + for _, tp := range sys.tpm.tracerProviders { + errs = append(errs, tp.ForceFlush(ctx)) + } + return errors.Join(errs...) + } + return nil +} + +type systemContextKeyType struct{} + +var systemContextKey systemContextKeyType + +type systemContext struct { + options Options + tpm *tracerProviderManager + exporterServer *ExporterServer + shutdown atomic.Bool +} + +func systemContextFromContext(ctx context.Context) *systemContext { + sys, _ := ctx.Value(systemContextKey).(*systemContext) + return sys +} + +type tracerProviderManager struct { + mu sync.Mutex + tracerProviders []*sdktrace.TracerProvider +} + +func (tpm *tracerProviderManager) ShutdownAll(ctx context.Context) error { + tpm.mu.Lock() + defer tpm.mu.Unlock() + var errs []error + for _, tp := range tpm.tracerProviders { + errs = append(errs, tp.ForceFlush(ctx)) + } + for _, tp := range tpm.tracerProviders { + errs = append(errs, tp.Shutdown(ctx)) + } + clear(tpm.tracerProviders) + return errors.Join(errs...) +} + +func (tpm *tracerProviderManager) Add(tp *sdktrace.TracerProvider) { + tpm.mu.Lock() + defer tpm.mu.Unlock() + tpm.tracerProviders = append(tpm.tracerProviders, tp) } diff --git a/internal/telemetry/trace/trace_export_test.go b/internal/telemetry/trace/trace_export_test.go new file mode 100644 index 000000000..9917d6c61 --- /dev/null +++ b/internal/telemetry/trace/trace_export_test.go @@ -0,0 +1,71 @@ +package trace + +import ( + "cmp" + "io" + "slices" + "testing" + "time" + + oteltrace "go.opentelemetry.io/otel/trace" +) + +var ( + NewSpanObserver = newSpanObserver + NewSpanTracker = newSpanTracker +) + +type XStackTraceProcessor = stackTraceProcessor + +func (obs *spanObserver) XWait() { + obs.wait(nil, 5*time.Second) +} + +func (obs *spanObserver) XUnobservedIDs() []oteltrace.SpanID { + obs.cond.L.Lock() + defer obs.cond.L.Unlock() + ids := []oteltrace.SpanID{} + for k, v := range obs.referencedIDs { + if v.IsValid() { + ids = append(ids, k) + } + } + slices.SortFunc(ids, func(a, b oteltrace.SpanID) int { + return cmp.Compare(a.String(), b.String()) + }) + return ids +} + +func (obs *spanObserver) XObservedIDs() []oteltrace.SpanID { + obs.cond.L.Lock() + defer obs.cond.L.Unlock() + ids := []oteltrace.SpanID{} + for k, v := range obs.referencedIDs { + if !v.IsValid() { + ids = append(ids, k) + } + } + slices.SortFunc(ids, func(a, b oteltrace.SpanID) int { + return cmp.Compare(a.String(), b.String()) + }) + return ids +} + +func (t *spanTracker) XInflightSpans() []oteltrace.SpanID { + ids := []oteltrace.SpanID{} + t.inflightSpans.Range(func(key, _ any) bool { + ids = append(ids, key.(oteltrace.SpanID)) + return true + }) + slices.SortFunc(ids, func(a, b oteltrace.SpanID) int { + return cmp.Compare(a.String(), b.String()) + }) + return ids +} + +func SetDebugMessageWriterForTest(t testing.TB, w io.Writer) { + debugMessageWriter = w + t.Cleanup(func() { + debugMessageWriter = nil + }) +} diff --git a/internal/telemetry/trace/trace_test.go b/internal/telemetry/trace/trace_test.go deleted file mode 100644 index b22afb98e..000000000 --- a/internal/telemetry/trace/trace_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package trace - -import ( - "net/url" - "testing" -) - -func TestGetProvider(t *testing.T) { - tests := []struct { - name string - opts *TracingOptions - wantErr bool - }{ - {"jaeger", &TracingOptions{JaegerAgentEndpoint: "localhost:6831", Service: "all", Provider: "jaeger"}, false}, - {"jaeger with debug", &TracingOptions{JaegerAgentEndpoint: "localhost:6831", Service: "all", Provider: "jaeger", Debug: true}, false}, - {"jaeger no endpoint", &TracingOptions{JaegerAgentEndpoint: "", Service: "all", Provider: "jaeger"}, false}, - {"unknown provider", &TracingOptions{JaegerAgentEndpoint: "localhost:0", Service: "all", Provider: "Lucius Cornelius Sulla"}, true}, - {"zipkin with debug", &TracingOptions{ZipkinEndpoint: &url.URL{Host: "localhost"}, Service: "all", Provider: "zipkin", Debug: true}, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if _, err := GetProvider(tt.opts); (err != nil) != tt.wantErr { - t.Errorf("RegisterTracing() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/internal/telemetry/trace/util.go b/internal/telemetry/trace/util.go new file mode 100644 index 000000000..9f62beb85 --- /dev/null +++ b/internal/telemetry/trace/util.go @@ -0,0 +1,136 @@ +package trace + +import ( + "encoding/hex" + "fmt" + "net/url" + "strings" + "unique" + + "go.opentelemetry.io/otel/attribute" + oteltrace "go.opentelemetry.io/otel/trace" + commonv1 "go.opentelemetry.io/proto/otlp/common/v1" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" +) + +func ParseTraceparent(traceparent string) (oteltrace.SpanContext, error) { + parts := strings.Split(traceparent, "-") + if len(parts) != 4 { + return oteltrace.SpanContext{}, fmt.Errorf("malformed traceparent: expected 4 segments, found %d", len(parts)) + } + traceID, err := oteltrace.TraceIDFromHex(parts[1]) + if err != nil { + return oteltrace.SpanContext{}, fmt.Errorf("malformed traceparent: invalid trace ID: %w", err) + } + spanID, err := oteltrace.SpanIDFromHex(parts[2]) + if err != nil { + return oteltrace.SpanContext{}, fmt.Errorf("malformed traceparent: invalid span ID: %w", err) + } + var traceFlags oteltrace.TraceFlags + if flags, err := hex.DecodeString(parts[3]); err != nil { + return oteltrace.SpanContext{}, fmt.Errorf("malformed traceparent: invalid trace flags: %w", err) + } else if len(flags) == 1 { + traceFlags = oteltrace.TraceFlags(flags[0]) + } else { + return oteltrace.SpanContext{}, fmt.Errorf("malformed traceparent: invalid trace flags of size %d", len(flags)) + } + if len(traceID) != 16 { + return oteltrace.SpanContext{}, fmt.Errorf("malformed traceparent: invalid trace ID of size %d", len(traceID)) + } + if len(spanID) != 8 { + return oteltrace.SpanContext{}, fmt.Errorf("malformed traceparent: invalid span ID of size %d", len(spanID)) + } + return oteltrace.NewSpanContext(oteltrace.SpanContextConfig{ + TraceID: traceID, + SpanID: spanID, + TraceFlags: traceFlags, + }), nil +} + +// WithTraceFromSpanContext returns a copy of traceparent with the trace ID +// (2nd segment) and trace flags (4th segment) replaced with the corresponding +// values from spanContext. +func WithTraceFromSpanContext(traceparent string, spanContext oteltrace.SpanContext) string { + parts := strings.Split(traceparent, "-") + if len(parts) != 4 { + return traceparent + } + parts[1] = spanContext.TraceID().String() + parts[3] = spanContext.TraceFlags().String() + return strings.Join(parts, "-") +} + +func FormatSpanName(span *tracev1.Span) { + hasVariables := strings.Contains(span.GetName(), "${") + if hasVariables { + replacements := make([]string, 0, 6) + for _, attr := range span.Attributes { + switch attr.Key { + case "http.url": + u, _ := url.Parse(attr.Value.GetStringValue()) + replacements = append(replacements, + "${path}", u.Path, + "${host}", u.Host, + ) + case "http.method": + replacements = append(replacements, "${method}", attr.Value.GetStringValue()) + } + } + span.Name = strings.NewReplacer(replacements...).Replace(span.Name) + } +} + +var ( + zeroSpanID oteltrace.SpanID + zeroTraceID = unique.Make(oteltrace.TraceID([16]byte{})) +) + +func ToSpanID(bytes []byte) (oteltrace.SpanID, bool) { + switch len(bytes) { + case 0: + return zeroSpanID, true + case 8: + return oteltrace.SpanID(bytes), true + } + return zeroSpanID, false +} + +func ToTraceID(bytes []byte) (unique.Handle[oteltrace.TraceID], bool) { + switch len(bytes) { + case 0: + return zeroTraceID, true + case 16: + return unique.Make(oteltrace.TraceID(bytes)), true + } + return zeroTraceID, false +} + +func NewAttributeSet(kvs ...*commonv1.KeyValue) attribute.Set { + attrs := make([]attribute.KeyValue, len(kvs)) + for i, kv := range kvs { + var value attribute.Value + switch v := kv.Value.Value.(type) { + case *commonv1.AnyValue_BoolValue: + value = attribute.BoolValue(v.BoolValue) + case *commonv1.AnyValue_BytesValue: + value = attribute.StringValue(string(v.BytesValue)) + case *commonv1.AnyValue_DoubleValue: + value = attribute.Float64Value(v.DoubleValue) + case *commonv1.AnyValue_IntValue: + value = attribute.Int64Value(v.IntValue) + case *commonv1.AnyValue_StringValue: + value = attribute.StringValue(v.StringValue) + case *commonv1.AnyValue_ArrayValue: + panic("unimplemented") + case *commonv1.AnyValue_KvlistValue: + panic("unimplemented") + default: + panic(fmt.Sprintf("unexpected v1.isAnyValue_Value: %#v", v)) + } + attrs[i] = attribute.KeyValue{ + Key: attribute.Key(kv.Key), + Value: value, + } + } + return attribute.NewSet(attrs...) +} diff --git a/internal/telemetry/trace/zipkin.go b/internal/telemetry/trace/zipkin.go deleted file mode 100644 index de4188277..000000000 --- a/internal/telemetry/trace/zipkin.go +++ /dev/null @@ -1,49 +0,0 @@ -package trace - -import ( - "fmt" - stdlog "log" - - oczipkin "contrib.go.opencensus.io/exporter/zipkin" - "github.com/openzipkin/zipkin-go" - "github.com/openzipkin/zipkin-go/reporter" - zipkinHTTP "github.com/openzipkin/zipkin-go/reporter/http" - octrace "go.opencensus.io/trace" - - "github.com/pomerium/pomerium/internal/log" -) - -type zipkinProvider struct { - reporter reporter.Reporter - exporter *oczipkin.Exporter -} - -func (provider *zipkinProvider) Register(opts *TracingOptions) error { - localEndpoint, err := zipkin.NewEndpoint(opts.Service, "") - if err != nil { - return fmt.Errorf("telemetry/trace: could not create local endpoint: %w", err) - } - - logger := log.With().Str("service", "zipkin").Logger() - logWriter := &log.StdLogWrapper{Logger: &logger} - stdLogger := stdlog.New(logWriter, "", 0) - - provider.reporter = zipkinHTTP.NewReporter(opts.ZipkinEndpoint.String(), zipkinHTTP.Logger(stdLogger)) - provider.exporter = oczipkin.NewExporter(provider.reporter, localEndpoint) - octrace.RegisterExporter(provider.exporter) - return nil -} - -func (provider *zipkinProvider) Unregister() error { - if provider.exporter != nil { - octrace.UnregisterExporter(provider.exporter) - provider.exporter = nil - } - - var err error - if provider.reporter != nil { - err = provider.reporter.Close() - provider.reporter = nil - } - return err -} diff --git a/internal/testenv/environment.go b/internal/testenv/environment.go index beef11ec2..0c78ea94a 100644 --- a/internal/testenv/environment.go +++ b/internal/testenv/environment.go @@ -17,7 +17,6 @@ import ( "fmt" "io" "math/big" - "math/bits" "net" "net/url" "os" @@ -26,25 +25,34 @@ import ( "path/filepath" "runtime" "strconv" + "strings" "sync" + "sync/atomic" "syscall" "testing" "time" "github.com/pomerium/pomerium/config" "github.com/pomerium/pomerium/config/envoyconfig/filemgr" + databroker_service "github.com/pomerium/pomerium/databroker" "github.com/pomerium/pomerium/internal/log" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/testenv/envutil" "github.com/pomerium/pomerium/internal/testenv/values" "github.com/pomerium/pomerium/pkg/cmd/pomerium" "github.com/pomerium/pomerium/pkg/envoy" "github.com/pomerium/pomerium/pkg/grpc/databroker" "github.com/pomerium/pomerium/pkg/health" + "github.com/pomerium/pomerium/pkg/identity/legacymanager" + "github.com/pomerium/pomerium/pkg/identity/manager" "github.com/pomerium/pomerium/pkg/netutil" "github.com/pomerium/pomerium/pkg/slices" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + oteltrace "go.opentelemetry.io/otel/trace" "golang.org/x/sync/errgroup" "google.golang.org/grpc/grpclog" ) @@ -56,6 +64,7 @@ type Environment interface { // top-level logger scoped to this environment. It will be canceled when // Stop() is called, or during test cleanup. Context() context.Context + Tracer() oteltrace.Tracer Assert() *assert.Assertions Require() *require.Assertions @@ -133,10 +142,29 @@ type Environment interface { // the Pomerium server and Envoy. NewLogRecorder(opts ...LogRecorderOption) *LogRecorder + // GetState returns the current state of the test environment. + GetState() EnvironmentState + // OnStateChanged registers a callback to be invoked when the environment's - // state changes to the given state. The callback is invoked in a separate - // goroutine. - OnStateChanged(state EnvironmentState, callback func()) + // state changes to the given state. Each callback is invoked in a separate + // goroutine, but the test environment will wait for all callbacks to return + // before continuing, after triggering the state change. + // + // Calling the returned stop function will prevent the callback from being + // run. Returns true if it stopped the callback from being run, or false if + // it already ran or is currently running. + // + // If the environment is already in the given state, the callback will be run + // in a separate goroutine immediately and the returned stop function will + // have no effect. A callback run in this way will prevent the state from + // advancing until the callback returns. + // + // State changes are triggered in the following places: + // - NotRunning->Starting: in Start(), as the first operation + // - Starting->Running: in Start(), just before returning + // - Running->Stopping: in Stop(), just before the env context is canceled + // - Stopping->Stopped: in Stop(), after all tasks have completed + OnStateChanged(state EnvironmentState, callback func()) (stop func() bool) } type Certificate tls.Certificate @@ -153,10 +181,9 @@ func (c *Certificate) SPKIHash() string { type EnvironmentState uint32 -const NotRunning EnvironmentState = 0 - const ( - Starting EnvironmentState = 1 << iota + NotRunning EnvironmentState = iota + Starting Running Stopping Stopped @@ -192,10 +219,13 @@ type environment struct { workspaceFolder string silent bool - ctx context.Context - cancel context.CancelCauseFunc - cleanupOnce sync.Once - logWriter *log.MultiWriter + ctx context.Context + cancel context.CancelCauseFunc + cleanupOnce sync.Once + logWriter *log.MultiWriter + tracerProvider oteltrace.TracerProvider + tracer oteltrace.Tracer + rootSpan oteltrace.Span mods []WithCaller[Modifier] tasks []WithCaller[Task] @@ -204,14 +234,17 @@ type environment struct { stateMu sync.Mutex state EnvironmentState stateChangeListeners map[EnvironmentState][]func() + stateChangeBlockers sync.WaitGroup src *configSource } type EnvironmentOptions struct { - debug bool - pauseOnFailure bool - forceSilent bool + debug bool + pauseOnFailure bool + forceSilent bool + traceDebugFlags trace.DebugFlags + traceClient otlptrace.Client } type EnvironmentOption func(*EnvironmentOptions) @@ -249,28 +282,57 @@ func Silent(silent ...bool) EnvironmentOption { } } +const StandardTraceDebugFlags = trace.TrackSpanCallers | + trace.WarnOnIncompleteSpans | + trace.WarnOnIncompleteTraces | + trace.WarnOnUnresolvedReferences | + trace.LogTraceIDMappingsOnWarn | + trace.LogAllSpansOnWarn + +func WithTraceDebugFlags(flags trace.DebugFlags) EnvironmentOption { + return func(o *EnvironmentOptions) { + o.traceDebugFlags = flags + } +} + +func WithTraceClient(traceClient otlptrace.Client) EnvironmentOption { + return func(o *EnvironmentOptions) { + o.traceClient = traceClient + } +} + var setGrpcLoggerOnce sync.Once +const defaultTraceDebugFlags = trace.TrackSpanCallers + var ( - flagDebug = flag.Bool("env.debug", false, "enables test environment debug logging (equivalent to Debug() option)") - flagPauseOnFailure = flag.Bool("env.pause-on-failure", false, "enables pausing the test environment on failure (equivalent to PauseOnFailure() option)") - flagSilent = flag.Bool("env.silent", false, "suppresses all test environment output (equivalent to Silent() option)") + flagDebug = flag.Bool("env.debug", false, "enables test environment debug logging (equivalent to Debug() option)") + flagPauseOnFailure = flag.Bool("env.pause-on-failure", false, "enables pausing the test environment on failure (equivalent to PauseOnFailure() option)") + flagSilent = flag.Bool("env.silent", false, "suppresses all test environment output (equivalent to Silent() option)") + flagTraceDebugFlags = flag.String("env.trace-debug-flags", strconv.Itoa(defaultTraceDebugFlags), "trace debug flags (equivalent to TraceDebugFlags() option)") ) func New(t testing.TB, opts ...EnvironmentOption) Environment { - if runtime.GOOS != "linux" { - t.Skip("test environment only supported on linux") + addTraceDebugFlags := strings.HasPrefix(*flagTraceDebugFlags, "+") + defaultTraceDebugFlags, err := strconv.Atoi(strings.TrimPrefix(*flagTraceDebugFlags, "+")) + if err != nil { + panic("malformed value for --env.trace-debug-flags: " + err.Error()) } options := EnvironmentOptions{ - debug: *flagDebug, - pauseOnFailure: *flagPauseOnFailure, - forceSilent: *flagSilent, + debug: *flagDebug, + pauseOnFailure: *flagPauseOnFailure, + forceSilent: *flagSilent, + traceDebugFlags: trace.DebugFlags(defaultTraceDebugFlags), } options.apply(opts...) if testing.Short() { t.Helper() t.Skip("test environment disabled in short mode") } + if addTraceDebugFlags { + options.traceDebugFlags |= trace.DebugFlags(defaultTraceDebugFlags) + } + trace.UseGlobalPanicTracer() databroker.DebugUseFasterBackoff.Store(true) workspaceFolder, err := os.Getwd() require.NoError(t, err) @@ -305,7 +367,16 @@ func New(t testing.TB, opts ...EnvironmentOption) Environment { }) logger := zerolog.New(writer).With().Timestamp().Logger().Level(zerolog.DebugLevel) - ctx, cancel := context.WithCancelCause(logger.WithContext(context.Background())) + ctx := trace.Options{ + DebugFlags: options.traceDebugFlags, + RemoteClient: options.traceClient, + }.NewContext(logger.WithContext(context.Background())) + tracerProvider := trace.NewTracerProvider(ctx, "Test Environment") + tracer := tracerProvider.Tracer(trace.PomeriumCoreTracer) + ctx, span := tracer.Start(ctx, t.Name(), oteltrace.WithNewRoot()) + require.NoError(t, err) + + ctx, cancel := context.WithCancelCause(ctx) taskErrGroup, ctx := errgroup.WithContext(ctx) e := &environment{ @@ -313,7 +384,7 @@ func New(t testing.TB, opts ...EnvironmentOption) Environment { t: t, assert: assert.New(t), require: require.New(t), - tempDir: t.TempDir(), + tempDir: tempDir(t), ports: Ports{ ProxyHTTP: values.Deferred[int](), ProxyGRPC: values.Deferred[int](), @@ -325,13 +396,18 @@ func New(t testing.TB, opts ...EnvironmentOption) Environment { Debug: values.Deferred[int](), ALPN: values.Deferred[int](), }, - workspaceFolder: workspaceFolder, - silent: silent, - ctx: ctx, - cancel: cancel, - logWriter: writer, - taskErrGroup: taskErrGroup, + workspaceFolder: workspaceFolder, + silent: silent, + ctx: ctx, + cancel: cancel, + tracerProvider: tracerProvider, + tracer: tracer, + logWriter: writer, + taskErrGroup: taskErrGroup, + stateChangeListeners: make(map[EnvironmentState][]func()), + rootSpan: span, } + _, err = rand.Read(e.sharedSecret[:]) require.NoError(t, err) _, err = rand.Read(e.cookieSecret[:]) @@ -362,11 +438,13 @@ func New(t testing.TB, opts ...EnvironmentOption) Environment { func (e *environment) debugf(format string, args ...any) { e.t.Helper() + if e.rootSpan.IsRecording() { + e.rootSpan.AddEvent(fmt.Sprintf(format, args...)) + } if !e.debug { return } - - e.t.Logf("\x1b[34m[debug] "+format+"\x1b[0m", args...) + e.t.Logf("\x1b[34mDEBUG ["+e.t.Name()+"] "+format+"\x1b[0m", args...) } type WithCaller[T any] struct { @@ -394,6 +472,10 @@ func (e *environment) Context() context.Context { return ContextWithEnv(e.ctx, e) } +func (e *environment) Tracer() oteltrace.Tracer { + return e.tracer +} + func (e *environment) Assert() *assert.Assertions { return e.assert } @@ -455,9 +537,11 @@ var ErrCauseTestCleanup = errors.New("test cleanup") var ErrCauseManualStop = errors.New("Stop() called") func (e *environment) Start() { + _, span := e.tracer.Start(e.Context(), "Start") + defer span.End() e.debugf("Start()") e.advanceState(Starting) - e.t.Cleanup(e.cleanup) + e.t.Cleanup(e.onTestCleanup) e.t.Setenv("TMPDIR", e.TempDir()) e.debugf("temp dir: %s", e.TempDir()) @@ -515,6 +599,7 @@ func (e *environment) Start() { log.AccessLogFieldUserAgent, log.AccessLogFieldClientCertificate, } + cfg.Options.TracingSampleRate = 1.0 e.src = &configSource{cfg: cfg} e.AddTask(TaskFunc(func(ctx context.Context) error { @@ -524,8 +609,13 @@ func (e *environment) Start() { require.NoError(e.t, cfg.Options.Validate(), "invoking modifier resulted in an invalid configuration:\nadded by: "+mod.Caller) } - opts := []pomerium.RunOption{ + opts := []pomerium.Option{ pomerium.WithOverrideFileManager(fileMgr), + pomerium.WithEnvoyServerOptions(envoy.WithExitGracePeriod(30 * time.Second)), + pomerium.WithDataBrokerServerOptions( + databroker_service.WithManagerOptions(manager.WithLeaseTTL(1*time.Second)), + databroker_service.WithLegacyManagerOptions(legacymanager.WithLeaseTTL(1*time.Second)), + ), } envoyBinaryPath := filepath.Join(e.workspaceFolder, fmt.Sprintf("pkg/envoy/files/envoy-%s-%s", runtime.GOOS, runtime.GOARCH)) if envutil.EnvoyProfilerAvailable(envoyBinaryPath) { @@ -556,23 +646,29 @@ func (e *environment) Start() { } if len(envVars) > 0 { e.debugf("adding envoy env vars: %v\n", envVars) - opts = append(opts, pomerium.WithEnvoyServerOptions( - envoy.WithExtraEnvVars(envVars...), - envoy.WithExitGracePeriod(10*time.Second), // allow envoy time to flush pprof data to disk - )) + opts = append(opts, pomerium.WithEnvoyServerOptions(envoy.WithExtraEnvVars(envVars...))) } } else { e.debugf("envoy profiling not available") } - return pomerium.Run(ctx, e.src, opts...) + pom := pomerium.New(opts...) + e.OnStateChanged(Stopping, func() { + if err := pom.Shutdown(ctx); err != nil { + log.Ctx(ctx).Err(err).Msg("error shutting down pomerium server") + } else { + e.debugf("pomerium server shut down without error") + } + }) + require.NoError(e.t, pom.Start(ctx, e.tracerProvider, e.src)) + return pom.Wait() })) for i, task := range e.tasks { - log.Ctx(e.ctx).Debug().Str("caller", task.Caller).Msgf("starting task %d", i) + log.Ctx(e.Context()).Debug().Str("caller", task.Caller).Msgf("starting task %d", i) e.taskErrGroup.Go(func() error { - defer log.Ctx(e.ctx).Debug().Str("caller", task.Caller).Msgf("task %d exited", i) - return task.Value.Run(e.ctx) + defer log.Ctx(e.Context()).Debug().Str("caller", task.Caller).Msgf("task %d exited", i) + return task.Value.Run(e.Context()) }) } @@ -695,14 +791,9 @@ func (e *environment) Stop() { b.StopTimer() defer b.StartTimer() } + _, file, line, _ := runtime.Caller(1) e.cleanupOnce.Do(func() { - e.debugf("stop: Stop() called manually") - e.advanceState(Stopping) - e.cancel(ErrCauseManualStop) - err := e.taskErrGroup.Wait() - e.advanceState(Stopped) - e.debugf("stop: done waiting") - assert.ErrorIs(e.t, err, ErrCauseManualStop) + e.cleanup(fmt.Errorf("%w (caller: %s:%d)", ErrCauseManualStop, file, line)) }) } @@ -714,33 +805,51 @@ func (e *environment) Pause() { e.t.Log("\x1b[31mctrl+c received, continuing\x1b[0m") } -func (e *environment) cleanup() { +func (e *environment) onTestCleanup() { e.cleanupOnce.Do(func() { - e.debugf("stop: test cleanup") - if e.t.Failed() { - if e.pauseOnFailure { - e.t.Log("\x1b[31m*** pausing on test failure; continue with ctrl+c ***\x1b[0m") - c := make(chan os.Signal, 1) - signal.Notify(c, syscall.SIGINT) - <-c - e.t.Log("\x1b[31mctrl+c received, continuing\x1b[0m") - signal.Stop(c) - } - } - e.advanceState(Stopping) - e.cancel(ErrCauseTestCleanup) - err := e.taskErrGroup.Wait() - e.advanceState(Stopped) - e.debugf("stop: done waiting") - assert.ErrorIs(e.t, err, ErrCauseTestCleanup) + e.cleanup(ErrCauseTestCleanup) }) } +func (e *environment) cleanup(cancelCause error) { + e.debugf("stop: %s", cancelCause.Error()) + if e.t.Failed() { + if e.pauseOnFailure { + e.t.Log("\x1b[31m*** pausing on test failure; continue with ctrl+c ***\x1b[0m") + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGINT) + <-c + e.t.Log("\x1b[31mctrl+c received, continuing\x1b[0m") + signal.Stop(c) + } + } + e.advanceState(Stopping) + e.cancel(cancelCause) + errs := []error{} + if err := e.taskErrGroup.Wait(); err != nil { + errs = append(errs, fmt.Errorf("error waiting for tasks: %w", err)) + } + e.rootSpan.End() + if err := trace.ShutdownContext(e.Context()); err != nil { + errs = append(errs, fmt.Errorf("error shutting down trace context: %w", err)) + } + e.advanceState(Stopped) + // Wait for any additional callbacks created during stopped callbacks + // (for consistency, we consider the stopped state to "end" here) + e.stateChangeBlockers.Wait() + e.debugf("stop: done") + // err can be nil if e.g. the only task is the internal pomerium task, which + // returns a nil error if it exits cleanly + if err := errors.Join(errs...); err != nil { + assert.ErrorIs(e.t, err, cancelCause) + } +} + func (e *environment) Add(m Modifier) { e.t.Helper() caller := getCaller() e.debugf("Add: %T from %s", m, caller) - switch e.getState() { + switch e.GetState() { case NotRunning: for _, mod := range e.mods { if mod.Value == m { @@ -757,11 +866,11 @@ func (e *environment) Add(m Modifier) { panic("test bug: cannot call Add() before Start() has returned") case Running: e.debugf("Add: state=Running; calling ModifyConfig") - e.src.ModifyConfig(e.ctx, m) + e.src.ModifyConfig(e.Context(), m) case Stopped, Stopping: panic("test bug: cannot call Add() after Stop()") default: - panic(fmt.Sprintf("unexpected environment state: %s", e.getState())) + panic(fmt.Sprintf("unexpected environment state: %s", e.GetState())) } } @@ -803,34 +912,63 @@ func (e *environment) advanceState(newState EnvironmentState) { if newState <= e.state { panic(fmt.Sprintf("internal test environment bug: changed state to <= current: newState=%s, current=%s", newState, e.state)) } + e.stateChangeBlockers.Wait() e.debugf("state %s -> %s", e.state.String(), newState.String()) e.state = newState - e.debugf("notifying %d listeners of state change", len(e.stateChangeListeners[newState])) - for _, listener := range e.stateChangeListeners[newState] { - go listener() + if len(e.stateChangeListeners[newState]) > 0 { + e.debugf("notifying %d listeners of state change", len(e.stateChangeListeners[newState])) + var wg sync.WaitGroup + for _, listener := range e.stateChangeListeners[newState] { + wg.Add(1) + go func() { + _, span := e.tracer.Start(e.Context(), "State Change Callback") + span.SetAttributes(attribute.String("state", newState.String())) + defer span.End() + defer wg.Done() + listener() + }() + } + wg.Wait() + e.debugf("done notifying state change listeners") } } -func (e *environment) getState() EnvironmentState { +func (e *environment) GetState() EnvironmentState { e.stateMu.Lock() defer e.stateMu.Unlock() return e.state } -func (e *environment) OnStateChanged(state EnvironmentState, callback func()) { +func (e *environment) OnStateChanged(state EnvironmentState, callback func()) (cancel func() bool) { e.stateMu.Lock() defer e.stateMu.Unlock() - if e.state&state != 0 { - go callback() - return - } - - // add change listeners for all states, if there are multiple bits set - for state > 0 { - stateBit := EnvironmentState(bits.TrailingZeros32(uint32(state))) - state &= (state - 1) - e.stateChangeListeners[stateBit] = append(e.stateChangeListeners[stateBit], callback) + _, file, line, _ := runtime.Caller(1) + switch { + case state < e.state: + panic(fmt.Sprintf("test bug: OnStateChanged called with state %s which is < current state (%s)", state, e.sharedSecret)) + case state == e.state: + e.stateChangeBlockers.Add(1) + e.debugf("invoking callback for current state (state: %s, caller: %s:%d)", state.String(), file, line) + go func() { + defer func() { + e.stateChangeBlockers.Done() + }() + callback() + }() + return func() bool { return false } + default: + canceled := &atomic.Bool{} + e.stateChangeListeners[state] = append(e.stateChangeListeners[state], func() { + if canceled.CompareAndSwap(false, true) { + e.debugf("invoking state change callback (caller: %s:%d)", file, line) + callback() + } + }) + return func() bool { + e.debugf("stopped state change callback (state: %s, caller: %s:%d)", state.String(), file, line) + return canceled.CompareAndSwap(false, true) + } } } diff --git a/internal/testenv/logs.go b/internal/testenv/logs.go index 73789f10d..e7346902b 100644 --- a/internal/testenv/logs.go +++ b/internal/testenv/logs.go @@ -12,6 +12,7 @@ import ( "io" "os" "reflect" + "regexp" "sync" "testing" "time" @@ -128,7 +129,7 @@ func (e *environment) NewLogRecorder(opts ...LogRecorderOption) *LogRecorder { lr := &LogRecorder{ LogRecorderOptions: options, t: e.t, - canceled: e.ctx.Done(), + canceled: e.Context().Done(), buf: newBuffer(), } e.logWriter.Add(lr.buf) @@ -139,7 +140,7 @@ func (e *environment) NewLogRecorder(opts ...LogRecorderOption) *LogRecorder { } e.logWriter.Remove(lr.buf) }) - context.AfterFunc(e.ctx, lr.removeGlobalWriterOnce) + context.AfterFunc(e.Context(), lr.removeGlobalWriterOnce) return lr } @@ -356,6 +357,11 @@ func match(expected, actual map[string]any, open bool) (matched bool, score int) return false, score } score++ + case *regexp.Regexp: + if !value.MatchString(actualValue) { + return false, score + } + score++ default: return false, score } diff --git a/internal/testenv/scenarios/mock_idp.go b/internal/testenv/scenarios/mock_idp.go index e6d6e539e..85babd674 100644 --- a/internal/testenv/scenarios/mock_idp.go +++ b/internal/testenv/scenarios/mock_idp.go @@ -45,7 +45,7 @@ type IDP struct { func (idp *IDP) Attach(ctx context.Context) { env := testenv.EnvFromContext(ctx) - router := upstreams.HTTP(nil) + router := upstreams.HTTP(nil, upstreams.WithDisplayName("IDP")) idp.url = values.Bind2(env.SubdomainURL("mock-idp"), router.Port(), func(urlStr string, port int) string { u, _ := url.Parse(urlStr) diff --git a/internal/testenv/scenarios/trace_receiver.go b/internal/testenv/scenarios/trace_receiver.go new file mode 100644 index 000000000..dfa7671eb --- /dev/null +++ b/internal/testenv/scenarios/trace_receiver.go @@ -0,0 +1,207 @@ +package scenarios + +import ( + "compress/gzip" + "context" + "fmt" + "io" + "net/http" + "sync" + "time" + + "github.com/pomerium/pomerium/config" + "github.com/pomerium/pomerium/internal/testenv" + "github.com/pomerium/pomerium/internal/testenv/upstreams" + "github.com/pomerium/pomerium/internal/testenv/values" + "github.com/pomerium/pomerium/internal/testutil" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" + "google.golang.org/protobuf/proto" +) + +type OTLPTraceReceiver struct { + coltracepb.UnimplementedTraceServiceServer + + mu sync.Mutex + receivedRequests []*coltracepb.ExportTraceServiceRequest + grpcUpstream values.MutableValue[upstreams.GRPCUpstream] + httpUpstream values.MutableValue[upstreams.HTTPUpstream] +} + +func NewOTLPTraceReceiver() *OTLPTraceReceiver { + return &OTLPTraceReceiver{ + grpcUpstream: values.Deferred[upstreams.GRPCUpstream](), + httpUpstream: values.Deferred[upstreams.HTTPUpstream](), + } +} + +// Export implements v1.TraceServiceServer. +func (rec *OTLPTraceReceiver) Export(_ context.Context, req *coltracepb.ExportTraceServiceRequest) (*coltracepb.ExportTraceServiceResponse, error) { + rec.mu.Lock() + defer rec.mu.Unlock() + rec.receivedRequests = append(rec.receivedRequests, req) + return &coltracepb.ExportTraceServiceResponse{}, nil +} + +// Attach implements testenv.Modifier. +func (rec *OTLPTraceReceiver) Attach(ctx context.Context) { + env := testenv.EnvFromContext(ctx) + + // NB: we cannot install tracing middleware into the receiver server, since + // it will cause a feedback loop of spans created when exporting other spans + + grpcUpstream := upstreams.GRPC(nil, + upstreams.WithDisplayName("OTLP GRPC Receiver"), + upstreams.WithDelayedShutdown(), + upstreams.WithNoClientTracing(), + upstreams.WithNoServerTracing(), + ) + httpUpstream := upstreams.HTTP(nil, + upstreams.WithDisplayName("OTLP HTTP Receiver"), + upstreams.WithDelayedShutdown(), + upstreams.WithNoClientTracing(), + upstreams.WithNoServerTracing(), + ) + + coltracepb.RegisterTraceServiceServer(grpcUpstream, rec) + httpUpstream.Handle("/v1/traces", rec.handleV1Traces) + env.AddUpstream(grpcUpstream) + env.AddUpstream(httpUpstream) + rec.grpcUpstream.Resolve(grpcUpstream) + rec.httpUpstream.Resolve(httpUpstream) +} + +// Modify implements testenv.Modifier. +func (rec *OTLPTraceReceiver) Modify(*config.Config) {} + +func (rec *OTLPTraceReceiver) handleV1Traces(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Content-Type") != "application/x-protobuf" { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte("invalid content type")) + return + } + reader := r.Body + if r.Header.Get("Content-Encoding") == "gzip" { + var err error + reader, err = gzip.NewReader(r.Body) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(err.Error())) + return + } + } + data, err := io.ReadAll(reader) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(err.Error())) + return + } + var req coltracepb.ExportTraceServiceRequest + if err := proto.Unmarshal(data, &req); err != nil { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(err.Error())) + return + } + resp, err := rec.Export(context.TODO(), &req) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + respData, err := proto.Marshal(resp) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + w.Header().Set("Content-Type", "application/x-protobuf") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(respData) +} + +func (rec *OTLPTraceReceiver) ReceivedRequests() []*coltracepb.ExportTraceServiceRequest { + rec.mu.Lock() + defer rec.mu.Unlock() + return rec.receivedRequests +} + +func (rec *OTLPTraceReceiver) PeekResourceSpans() []*tracev1.ResourceSpans { + rec.mu.Lock() + defer rec.mu.Unlock() + + return rec.peekResourceSpansLocked() +} + +func (rec *OTLPTraceReceiver) peekResourceSpansLocked() []*tracev1.ResourceSpans { + return testutil.FlattenExportRequests(rec.receivedRequests) +} + +func (rec *OTLPTraceReceiver) FlushResourceSpans() []*tracev1.ResourceSpans { + rec.mu.Lock() + defer rec.mu.Unlock() + spans := rec.peekResourceSpansLocked() + rec.receivedRequests = nil + return spans +} + +// GRPCEndpointURL returns a url suitable for use with the environment variable +// $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT or with [otlptracegrpc.WithEndpointURL]. +func (rec *OTLPTraceReceiver) GRPCEndpointURL() values.Value[string] { + return values.Chain(rec.grpcUpstream, upstreams.GRPCUpstream.Port, func(port int) string { + return fmt.Sprintf("http://127.0.0.1:%d", port) + }) +} + +// GRPCEndpointURL returns a url suitable for use with the environment variable +// $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT or with [otlptracehttp.WithEndpointURL]. +func (rec *OTLPTraceReceiver) HTTPEndpointURL() values.Value[string] { + return values.Chain(rec.httpUpstream, upstreams.HTTPUpstream.Port, func(port int) string { + return fmt.Sprintf("http://127.0.0.1:%d/v1/traces", port) + }) +} + +func (rec *OTLPTraceReceiver) NewGRPCClient(opts ...otlptracegrpc.Option) otlptrace.Client { + return &deferredClient{ + client: values.Bind(rec.grpcUpstream, func(up upstreams.GRPCUpstream) otlptrace.Client { + return otlptracegrpc.NewClient(append(opts, + otlptracegrpc.WithGRPCConn(up.DirectConnect()), + otlptracegrpc.WithTimeout(1*time.Minute), + )...) + }), + } +} + +func (rec *OTLPTraceReceiver) NewHTTPClient(opts ...otlptracehttp.Option) otlptrace.Client { + return &deferredClient{ + client: values.Chain(rec.httpUpstream, upstreams.HTTPUpstream.Port, func(port int) otlptrace.Client { + return otlptracehttp.NewClient(append(opts, + otlptracehttp.WithEndpointURL(fmt.Sprintf("http://127.0.0.1:%d/v1/traces", port)), + otlptracehttp.WithTimeout(1*time.Minute), + )...) + }), + } +} + +type deferredClient struct { + client values.Value[otlptrace.Client] +} + +// Start implements otlptrace.Client. +func (o *deferredClient) Start(ctx context.Context) error { + return o.client.Value().Start(ctx) +} + +// Stop implements otlptrace.Client. +func (o *deferredClient) Stop(ctx context.Context) error { + return o.client.Value().Stop(ctx) +} + +// UploadTraces implements otlptrace.Client. +func (o *deferredClient) UploadTraces(ctx context.Context, protoSpans []*tracev1.ResourceSpans) error { + return o.client.Value().UploadTraces(ctx, protoSpans) +} + +var _ otlptrace.Client = (*deferredClient)(nil) diff --git a/internal/testenv/selftests/tracing_test.go b/internal/testenv/selftests/tracing_test.go new file mode 100644 index 000000000..324ea5700 --- /dev/null +++ b/internal/testenv/selftests/tracing_test.go @@ -0,0 +1,400 @@ +package selftests_test + +import ( + "context" + "fmt" + "io" + "maps" + "net/http" + "os" + "slices" + "sync/atomic" + "testing" + "time" + + "github.com/pomerium/pomerium/config" + "github.com/pomerium/pomerium/internal/telemetry/trace" + "github.com/pomerium/pomerium/internal/testenv" + "github.com/pomerium/pomerium/internal/testenv/scenarios" + "github.com/pomerium/pomerium/internal/testenv/snippets" + "github.com/pomerium/pomerium/internal/testenv/upstreams" + "github.com/pomerium/pomerium/internal/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.17.0" + oteltrace "go.opentelemetry.io/otel/trace" +) + +func otlpTraceReceiverOrFromEnv(t *testing.T) (modifier testenv.Modifier, newRemoteClient func() otlptrace.Client, getResults func() *testutil.TraceResults) { + t.Setenv("OTEL_TRACES_EXPORTER", "otlp") + tracesEndpoint := os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") + if tracesEndpoint == "" { + tracesEndpoint = os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") + if tracesEndpoint == "" { + srv := scenarios.NewOTLPTraceReceiver() + return srv, + func() otlptrace.Client { + return srv.NewGRPCClient() + }, + func() *testutil.TraceResults { + return testutil.NewTraceResults(srv.FlushResourceSpans()) + } + } + } + return testenv.NoopModifier(), trace.NewRemoteClientFromEnv, nil +} + +var allServices = []string{ + "Test Environment", + "Authorize", + "Authenticate", + "Control Plane", + "Data Broker", + "Upstream", + "IDP", + "HTTP Client", + "Envoy", +} + +func TestOTLPTracing(t *testing.T) { + modifier, newRemoteClient, getResults := otlpTraceReceiverOrFromEnv(t) + env := testenv.New(t, testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags), testenv.WithTraceClient(newRemoteClient())) + env.Add(modifier) + + up := upstreams.HTTP(nil, upstreams.WithDisplayName("Upstream")) + up.Handle("/foo", func(w http.ResponseWriter, _ *http.Request) { + w.Write([]byte("OK")) + }) + env.Add(scenarios.NewIDP([]*scenarios.User{ + { + Email: "foo@example.com", + FirstName: "Firstname", + LastName: "Lastname", + }, + })) + + route := up.Route(). + From(env.SubdomainURL("foo")). + PPL(`{"allow":{"and":["email":{"is":"foo@example.com"}]}}`) + + env.AddUpstream(up) + env.Start() + snippets.WaitStartupComplete(env) + + ctx, span := env.Tracer().Start(env.Context(), "Authenticate", oteltrace.WithNewRoot()) + resp, err := up.Get(route, upstreams.AuthenticateAs("foo@example.com"), upstreams.Path("/foo"), upstreams.Context(ctx)) + span.End() + require.NoError(t, err) + body, err := io.ReadAll(resp.Body) + assert.NoError(t, err) + assert.NoError(t, resp.Body.Close()) + assert.Equal(t, resp.StatusCode, 200) + assert.Equal(t, "OK", string(body)) + + env.Stop() + + if getResults != nil { + results := getResults() + var ( + testEnvironmentLocalTest = fmt.Sprintf("Test Environment: %s", t.Name()) + testEnvironmentAuthenticate = "Test Environment: Authenticate" + authenticateOAuth2Client = "Authenticate: OAuth2 Client: GET /.well-known/jwks.json" + idpServerGetUserinfo = "IDP: Server: GET /oidc/userinfo" + idpServerPostToken = "IDP: Server: POST /oidc/token" + controlPlaneEnvoyAccessLogs = "Control Plane: envoy.service.accesslog.v3.AccessLogService/StreamAccessLogs" + controlPlaneEnvoyDiscovery = "Control Plane: envoy.service.discovery.v3.AggregatedDiscoveryService/DeltaAggregatedResources" + controlPlaneExport = "Control Plane: opentelemetry.proto.collector.trace.v1.TraceService/Export" + ) + + results.MatchTraces(t, + testutil.MatchOptions{ + Exact: true, + CheckDetachedSpans: true, + }, + testutil.Match{Name: testEnvironmentLocalTest, TraceCount: 1, Services: []string{"Test Environment", "Control Plane", "Data Broker"}}, + testutil.Match{Name: testEnvironmentAuthenticate, TraceCount: 1, Services: allServices}, + testutil.Match{Name: authenticateOAuth2Client, TraceCount: testutil.Greater(0)}, + testutil.Match{Name: idpServerGetUserinfo, TraceCount: testutil.EqualToMatch(authenticateOAuth2Client)}, + testutil.Match{Name: idpServerPostToken, TraceCount: testutil.EqualToMatch(authenticateOAuth2Client)}, + testutil.Match{Name: controlPlaneEnvoyDiscovery, TraceCount: 1}, + testutil.Match{Name: controlPlaneExport, TraceCount: testutil.Greater(0)}, + testutil.Match{Name: controlPlaneEnvoyAccessLogs, TraceCount: testutil.Any{}}, + ) + } +} + +func TestOTLPTracing_TraceCorrelation(t *testing.T) { + modifier, newRemoteClient, getResults := otlpTraceReceiverOrFromEnv(t) + env := testenv.New(t, testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags), testenv.WithTraceClient(newRemoteClient())) + env.Add(modifier) + + up := upstreams.HTTP(nil, upstreams.WithDisplayName("Upstream"), upstreams.WithNoClientTracing()) + up.Handle("/foo", func(w http.ResponseWriter, _ *http.Request) { + w.Write([]byte("OK")) + }) + env.Add(scenarios.NewIDP([]*scenarios.User{ + { + Email: "foo@example.com", + FirstName: "Firstname", + LastName: "Lastname", + }, + })) + + route := up.Route(). + From(env.SubdomainURL("foo")). + PPL(`{"allow":{"and":["email":{"is":"foo@example.com"}]}}`) + + env.AddUpstream(up) + env.Start() + snippets.WaitStartupComplete(env) + + resp, err := up.Get(route, upstreams.AuthenticateAs("foo@example.com"), upstreams.Path("/foo"), upstreams.Context(context.Background())) + require.NoError(t, err) + body, err := io.ReadAll(resp.Body) + assert.NoError(t, err) + assert.NoError(t, resp.Body.Close()) + assert.Equal(t, resp.StatusCode, 200) + assert.Equal(t, "OK", string(body)) + + env.Stop() + if getResults != nil { + results := getResults() + traces := results.GetTraces() + downstreamTraces := traces.ByName[fmt.Sprintf("Envoy: ingress: GET foo.localhost.pomerium.io:%d/foo", env.Ports().ProxyHTTP.Value())].WithoutErrors() + upstreamTraces := traces.ByName[fmt.Sprintf("Envoy: ingress: GET 127.0.0.1:%d/foo", up.Port().Value())].WithoutErrors() + assert.Len(t, upstreamTraces, 1) + assert.Len(t, downstreamTraces, 1) + } +} + +type SamplingTestSuite struct { + suite.Suite + env testenv.Environment + getResults func() *testutil.TraceResults + route testenv.Route + upstream upstreams.HTTPUpstream + + sampled atomic.Int32 + notSampled atomic.Int32 +} + +func (s *SamplingTestSuite) SetupTest() { + modifier, newRemoteClient, getResults := otlpTraceReceiverOrFromEnv(s.T()) + s.getResults = getResults + s.env = testenv.New(s.T(), + testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags|trace.EnvoyFlushEverySpan), + testenv.WithTraceClient(newRemoteClient()), + ) + s.env.Add(modifier) + + s.sampled.Store(0) + s.notSampled.Store(0) + + s.env.Add(testenv.ModifierFunc(func(_ context.Context, cfg *config.Config) { + cfg.Options.TracingSampleRate = 0.5 + })) + s.env.Add(scenarios.NewIDP([]*scenarios.User{ + { + Email: "foo@example.com", + FirstName: "Firstname", + LastName: "Lastname", + }, + })) + + s.upstream = upstreams.HTTP(nil, upstreams.WithNoClientTracing(), upstreams.WithDisplayName("Upstream")) + s.upstream.Handle("/", s.handleRequest) + + s.route = s.upstream.Route(). + From(s.env.SubdomainURL("sampling-50pct")). + PPL(`{"allow":{"and":["email":{"is":"foo@example.com"}]}}`) + + s.env.AddUpstream(s.upstream) + s.env.Start() + snippets.WaitStartupComplete(s.env) +} + +func (s *SamplingTestSuite) TearDownTest() { + s.env.Stop() +} + +func (s *SamplingTestSuite) handleRequest(w http.ResponseWriter, req *http.Request) { + span := oteltrace.SpanFromContext(req.Context()) + flags := span.SpanContext().TraceFlags() + if flags.IsSampled() { + s.sampled.Add(1) + } else { + s.notSampled.Add(1) + } + w.Write([]byte("OK")) +} + +func (s *SamplingTestSuite) doRequest(ctx context.Context) { + resp, err := s.upstream.Get(s.route, upstreams.AuthenticateAs("foo@example.com"), upstreams.Path("/"), upstreams.Context(ctx)) + s.Require().NoError(err) + body, err := io.ReadAll(resp.Body) + s.Assert().NoError(err) + s.Assert().NoError(resp.Body.Close()) + s.Assert().Equal(resp.StatusCode, 200) + s.Assert().Equal("OK", string(body)) +} + +func (s *SamplingTestSuite) TestNoExternalTraceparent() { + for { + s.doRequest(context.Background()) + if s.sampled.Load() == 20 { + break + } + } + + s.Assert().NoError(trace.ForceFlush(s.env.Context())) + trace.WaitForSpans(s.env.Context(), 10*time.Second) + + s.Assert().Equal(int32(20), s.sampled.Load()) // 10 sampled + // Ideally we get ~50% sample rate, but CI will always be unlucky. + s.Assert().Greater(s.notSampled.Load(), int32(0)) + + if s.getResults != nil { + results := s.getResults() + traces := results.GetTraces() + s.Assert().Len(traces.ByParticipant["Upstream"], 20) + } +} + +func (s *SamplingTestSuite) TestExternalTraceparentAlwaysSample() { + tracer := trace.NewTracerProvider(s.env.Context(), "Always Sample", + sdktrace.WithSampler(sdktrace.AlwaysSample())).Tracer(trace.PomeriumCoreTracer) + for range 100 { + ctx, span := tracer.Start(context.Background(), "should sample") + s.doRequest(ctx) + span.End() + } + + s.Assert().NoError(trace.ForceFlush(s.env.Context())) + trace.WaitForSpans(s.env.Context(), 10*time.Second) + + // if the request already has a traceparent header, they will always be sampled + // regardless of the random sample rate we configured + s.Assert().Equal(int32(100), s.sampled.Load()) + s.Assert().Equal(int32(0), s.notSampled.Load()) + + if s.getResults != nil { + results := s.getResults() + traces := results.GetTraces() + s.Assert().Len(traces.ByParticipant["Envoy"], 100) + } +} + +func (s *SamplingTestSuite) TestExternalTraceparentNeverSample() { + tracer := trace.NewTracerProvider(s.env.Context(), "Never Sample", sdktrace.WithSampler(sdktrace.NeverSample())).Tracer(trace.PomeriumCoreTracer) + for range 100 { + ctx, span := tracer.Start(context.Background(), "should not sample") + s.doRequest(ctx) + span.End() + } + + s.Assert().NoError(trace.ForceFlush(s.env.Context())) + trace.WaitForSpans(s.env.Context(), 10*time.Second) + + s.Assert().Equal(int32(0), s.sampled.Load()) + s.Assert().Equal(int32(100), s.notSampled.Load()) + + if s.getResults != nil { + results := s.getResults() + traces := results.GetTraces() + if (len(traces.ByParticipant)) != 0 { + // whether or not these show up is timing dependent, but not important + possibleTraces := map[string]struct{}{ + "IDP: Server: POST /oidc/token": {}, + "IDP: Server: GET /oidc/userinfo": {}, + "Authenticate: OAuth2 Client: GET /.well-known/jwks.json": {}, + } + actual := slices.Collect(maps.Keys(traces.ByName)) + for _, name := range actual { + if _, ok := possibleTraces[name]; !ok { + s.Failf("unexpected trace: %s", name) + } + } + } + } +} + +func TestSampling(t *testing.T) { + suite.Run(t, &SamplingTestSuite{}) +} + +func TestExternalSpans(t *testing.T) { + modifier, newRemoteClient, getResults := otlpTraceReceiverOrFromEnv(t) + + // set up external tracer + external := otlptrace.NewUnstarted(newRemoteClient()) + r, err := resource.Merge( + resource.Empty(), + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceName("External"), + ), + ) + require.NoError(t, err) + + externalTracerProvider := sdktrace.NewTracerProvider(sdktrace.WithBatcher(external), sdktrace.WithResource(r)) + + env := testenv.New(t, testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags|trace.EnvoyFlushEverySpan), testenv.WithTraceClient(newRemoteClient())) + env.Add(modifier) + + up := upstreams.HTTP(nil, upstreams.WithNoClientTracing()) + up.Handle("/foo", func(w http.ResponseWriter, _ *http.Request) { + w.Write([]byte("OK")) + }) + env.Add(scenarios.NewIDP([]*scenarios.User{ + { + Email: "foo@example.com", + FirstName: "Firstname", + LastName: "Lastname", + }, + })) + + route := up.Route(). + From(env.SubdomainURL("foo")). + PPL(`{"allow":{"and":["email":{"is":"foo@example.com"}]}}`) + + env.AddUpstream(up) + env.Start() + require.NoError(t, external.Start(env.Context())) + snippets.WaitStartupComplete(env) + + ctx, span := externalTracerProvider.Tracer("external").Start(context.Background(), "External Root", oteltrace.WithNewRoot()) + t.Logf("external span id: %s", span.SpanContext().SpanID().String()) + resp, err := up.Get(route, upstreams.AuthenticateAs("foo@example.com"), upstreams.Path("/foo"), upstreams.Context(ctx)) + span.End() + require.NoError(t, err) + body, err := io.ReadAll(resp.Body) + assert.NoError(t, err) + assert.NoError(t, resp.Body.Close()) + assert.Equal(t, resp.StatusCode, 200) + assert.Equal(t, "OK", string(body)) + + assert.NoError(t, externalTracerProvider.ForceFlush(context.Background())) + assert.NoError(t, externalTracerProvider.Shutdown(context.Background())) + external.Shutdown(ctx) + env.Stop() + + if getResults != nil { + results := getResults() + results.MatchTraces(t, testutil.MatchOptions{CheckDetachedSpans: true}, + testutil.Match{Name: "External: External Root", TraceCount: 1, Services: []string{ + "Authorize", + "Authenticate", + "Control Plane", + "Data Broker", + "IDP", + "Envoy", + "External", + "HTTP Upstream", + }}, + ) + } +} diff --git a/internal/testenv/snippets/shutdown.go b/internal/testenv/snippets/shutdown.go new file mode 100644 index 000000000..879c37725 --- /dev/null +++ b/internal/testenv/snippets/shutdown.go @@ -0,0 +1,40 @@ +package snippets + +import ( + "context" + "errors" + "net/http" + + "github.com/pomerium/pomerium/internal/log" + "github.com/pomerium/pomerium/internal/testenv" +) + +func RunWithDelayedShutdown(ctx context.Context, serve func() error, stop func()) func() error { + env := testenv.EnvFromContext(ctx) + + stopping := make(chan struct{}) + serveExited := make(chan error, 1) + env.OnStateChanged(testenv.Stopping, func() { + close(stopping) + }) + cancel := env.OnStateChanged(testenv.Stopped, func() { + stop() + if err := <-serveExited; err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Ctx(ctx).Err(err).Msg("error stopping server") + } + }) + go func() { + serveExited <- serve() + close(serveExited) + }() + + return func() error { + select { + case <-stopping: + return nil + case err := <-serveExited: + cancel() + return err + } + } +} diff --git a/internal/testenv/snippets/wait.go b/internal/testenv/snippets/wait.go index 4342f9c42..eb44ea36c 100644 --- a/internal/testenv/snippets/wait.go +++ b/internal/testenv/snippets/wait.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/testenv" "github.com/pomerium/pomerium/pkg/grpcutil" "google.golang.org/grpc" @@ -12,6 +13,11 @@ import ( ) func WaitStartupComplete(env testenv.Environment, timeout ...time.Duration) time.Duration { + if env.GetState() == testenv.NotRunning { + panic("test bug: WaitStartupComplete called before starting the test environment") + } + _, span := trace.Continue(env.Context(), "snippets.WaitStartupComplete") + defer span.End() start := time.Now() recorder := env.NewLogRecorder() if len(timeout) == 0 { @@ -31,5 +37,6 @@ func WaitStartupComplete(env testenv.Environment, timeout ...time.Duration) time ) env.Require().NoError(err) env.Require().True(cc.WaitForStateChange(ctx, connectivity.Ready)) + cc.Close() return time.Since(start) } diff --git a/internal/testenv/temp.go b/internal/testenv/temp.go new file mode 100644 index 000000000..b00c38bea --- /dev/null +++ b/internal/testenv/temp.go @@ -0,0 +1,9 @@ +//go:build !darwin + +package testenv + +import "testing" + +func tempDir(t testing.TB) string { + return t.TempDir() +} diff --git a/internal/testenv/temp_darwin.go b/internal/testenv/temp_darwin.go new file mode 100644 index 000000000..ddee7c434 --- /dev/null +++ b/internal/testenv/temp_darwin.go @@ -0,0 +1,21 @@ +//go:build darwin + +package testenv + +import ( + "os" + "testing" +) + +// macos temp directory names are too long +// https://github.com/golang/go/issues/62614 +func tempDir(t testing.TB) string { + dir, err := os.MkdirTemp("", "test") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + os.RemoveAll(dir) + }) + return dir +} diff --git a/internal/testenv/types.go b/internal/testenv/types.go index 014114f22..8e863d3eb 100644 --- a/internal/testenv/types.go +++ b/internal/testenv/types.go @@ -84,7 +84,7 @@ type Aggregate struct { func (d *Aggregate) Add(mod Modifier) { if d.env != nil { - if d.env.(*environment).getState() == NotRunning { + if d.env.(*environment).GetState() == NotRunning { // If the test environment is running, adding to an aggregate is a no-op. // If the test environment has not been started yet, the aggregate is // being used like in the following example, which is incorrect: @@ -161,6 +161,20 @@ func ModifierFunc(fn func(ctx context.Context, cfg *config.Config)) Modifier { return &modifierFunc{fn: fn} } +func NoopModifier() Modifier { + return noopModifier{} +} + +type noopModifier struct{} + +// Attach implements Modifier. +func (n noopModifier) Attach(context.Context) {} + +// Modify implements Modifier. +func (n noopModifier) Modify(*config.Config) {} + +var _ Modifier = (noopModifier{}) + // Task represents a background task that can be added to an [Environment] to // have it run automatically on startup. // @@ -169,10 +183,17 @@ type Task interface { Run(ctx context.Context) error } -type TaskFunc func(ctx context.Context) error +func TaskFunc(fn func(ctx context.Context) error) Task { + return &taskFunc{fn: fn} +} -func (f TaskFunc) Run(ctx context.Context) error { - return f(ctx) +// this is a struct wrapper type instead of a typed func so it can be compared +type taskFunc struct { + fn func(ctx context.Context) error +} + +func (f *taskFunc) Run(ctx context.Context) error { + return f.fn(ctx) } // Upstream represents an upstream server. It is both a [Task] and a [Modifier] diff --git a/internal/testenv/upstreams/grpc.go b/internal/testenv/upstreams/grpc.go index fb22e08a7..b4329c84e 100644 --- a/internal/testenv/upstreams/grpc.go +++ b/internal/testenv/upstreams/grpc.go @@ -6,28 +6,36 @@ import ( "net" "strings" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/testenv" + "github.com/pomerium/pomerium/internal/testenv/snippets" "github.com/pomerium/pomerium/internal/testenv/values" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + oteltrace "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" ) -type Options struct { +type GRPCUpstreamOptions struct { + CommonUpstreamOptions serverOpts []grpc.ServerOption } -type Option func(*Options) - -func (o *Options) apply(opts ...Option) { - for _, op := range opts { - op(o) - } +type GRPCUpstreamOption interface { + applyGRPC(*GRPCUpstreamOptions) } -func ServerOpts(opt ...grpc.ServerOption) Option { - return func(o *Options) { +type GRPCUpstreamOptionFunc func(*GRPCUpstreamOptions) + +func (f GRPCUpstreamOptionFunc) applyGRPC(o *GRPCUpstreamOptions) { + f(o) +} + +func ServerOpts(opt ...grpc.ServerOption) GRPCUpstreamOption { + return GRPCUpstreamOptionFunc(func(o *GRPCUpstreamOptions) { o.serverOpts = append(o.serverOpts, opt...) - } + }) } // GRPCUpstream represents a GRPC server which can be used as the target for @@ -42,13 +50,18 @@ type GRPCUpstream interface { testenv.Upstream grpc.ServiceRegistrar Dial(r testenv.Route, dialOpts ...grpc.DialOption) *grpc.ClientConn + + // Dials the server directly instead of going through a Pomerium route. + DirectConnect(dialOpts ...grpc.DialOption) *grpc.ClientConn } type grpcUpstream struct { - Options + GRPCUpstreamOptions testenv.Aggregate - serverPort values.MutableValue[int] - creds credentials.TransportCredentials + serverPort values.MutableValue[int] + creds credentials.TransportCredentials + serverTracerProvider values.MutableValue[oteltrace.TracerProvider] + clientTracerProvider values.MutableValue[oteltrace.TracerProvider] services []service } @@ -59,13 +72,21 @@ var ( ) // GRPC creates a new GRPC upstream server. -func GRPC(creds credentials.TransportCredentials, opts ...Option) GRPCUpstream { - options := Options{} - options.apply(opts...) +func GRPC(creds credentials.TransportCredentials, opts ...GRPCUpstreamOption) GRPCUpstream { + options := GRPCUpstreamOptions{ + CommonUpstreamOptions: CommonUpstreamOptions{ + displayName: "GRPC Upstream", + }, + } + for _, op := range opts { + op.applyGRPC(&options) + } up := &grpcUpstream{ - Options: options, - creds: creds, - serverPort: values.Deferred[int](), + GRPCUpstreamOptions: options, + creds: creds, + serverPort: values.Deferred[int](), + serverTracerProvider: values.Deferred[oteltrace.TracerProvider](), + clientTracerProvider: values.Deferred[oteltrace.TracerProvider](), } up.RecordCaller() return up @@ -109,30 +130,65 @@ func (g *grpcUpstream) Run(ctx context.Context) error { return err } g.serverPort.Resolve(listener.Addr().(*net.TCPAddr).Port) - server := grpc.NewServer(append(g.serverOpts, grpc.Creds(g.creds))...) + if g.serverTracerProviderOverride != nil { + g.serverTracerProvider.Resolve(g.serverTracerProviderOverride) + } else { + g.serverTracerProvider.Resolve(trace.NewTracerProvider(ctx, g.displayName)) + } + if g.clientTracerProviderOverride != nil { + g.clientTracerProvider.Resolve(g.clientTracerProviderOverride) + } else { + g.clientTracerProvider.Resolve(trace.NewTracerProvider(ctx, "GRPC Client")) + } + server := grpc.NewServer(append(g.serverOpts, + grpc.Creds(g.creds), + grpc.StatsHandler(trace.NewServerStatsHandler(otelgrpc.NewServerHandler( + otelgrpc.WithTracerProvider(g.serverTracerProvider.Value()), + ))), + )...) for _, s := range g.services { server.RegisterService(s.desc, s.impl) } + if g.delayShutdown { + return snippets.RunWithDelayedShutdown(ctx, + func() error { + return server.Serve(listener) + }, + server.GracefulStop, + )() + } errC := make(chan error, 1) go func() { errC <- server.Serve(listener) }() select { case <-ctx.Done(): - server.Stop() + server.GracefulStop() return context.Cause(ctx) case err := <-errC: return err } } -func (g *grpcUpstream) Dial(r testenv.Route, dialOpts ...grpc.DialOption) *grpc.ClientConn { - dialOpts = append(dialOpts, - grpc.WithContextDialer(testenv.GRPCContextDialer), +func (g *grpcUpstream) withDefaultDialOpts(extraDialOpts []grpc.DialOption) []grpc.DialOption { + return append(extraDialOpts, grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(g.Env().ServerCAs(), "")), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), + grpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(g.clientTracerProvider.Value()))), ) - cc, err := grpc.NewClient(strings.TrimPrefix(r.URL().Value(), "https://"), dialOpts...) +} + +func (g *grpcUpstream) Dial(r testenv.Route, dialOpts ...grpc.DialOption) *grpc.ClientConn { + cc, err := grpc.NewClient(strings.TrimPrefix(r.URL().Value(), "https://"), g.withDefaultDialOpts(dialOpts)...) + if err != nil { + panic(err) + } + return cc +} + +func (g *grpcUpstream) DirectConnect(dialOpts ...grpc.DialOption) *grpc.ClientConn { + cc, err := grpc.NewClient(fmt.Sprintf("127.0.0.1:%d", g.Port().Value()), + append(g.withDefaultDialOpts(dialOpts), grpc.WithTransportCredentials(insecure.NewCredentials()))...) if err != nil { panic(err) } diff --git a/internal/testenv/upstreams/http.go b/internal/testenv/upstreams/http.go index 2ec020108..fd8d53ffd 100644 --- a/internal/testenv/upstreams/http.go +++ b/internal/testenv/upstreams/http.go @@ -21,13 +21,21 @@ import ( "github.com/gorilla/mux" "github.com/pomerium/pomerium/integration/forms" "github.com/pomerium/pomerium/internal/retry" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/testenv" + "github.com/pomerium/pomerium/internal/testenv/snippets" "github.com/pomerium/pomerium/internal/testenv/values" - "github.com/pomerium/pomerium/pkg/telemetry/requestid" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" + oteltrace "go.opentelemetry.io/otel/trace" "google.golang.org/protobuf/proto" ) type RequestOptions struct { + requestCtx context.Context path string query url.Values headers map[string]string @@ -80,6 +88,12 @@ func Client(c *http.Client) RequestOption { } } +func Context(ctx context.Context) RequestOption { + return func(o *RequestOptions) { + o.requestCtx = ctx + } +} + func WithClientTrace(ct *httptrace.ClientTrace) RequestOption { return func(o *RequestOptions) { o.trace = ct @@ -111,6 +125,14 @@ func ClientCert[T interface { } } +type HTTPUpstreamOptions struct { + CommonUpstreamOptions +} + +type HTTPUpstreamOption interface { + applyHTTP(*HTTPUpstreamOptions) +} + // HTTPUpstream represents a HTTP server which can be used as the target for // one or more Pomerium routes in a test environment. // @@ -128,13 +150,17 @@ type HTTPUpstream interface { } type httpUpstream struct { + HTTPUpstreamOptions testenv.Aggregate serverPort values.MutableValue[int] tlsConfig values.Value[*tls.Config] clientCache sync.Map // map[testenv.Route]*http.Client - router *mux.Router + router *mux.Router + serverTracerProvider values.MutableValue[oteltrace.TracerProvider] + clientTracerProvider values.MutableValue[oteltrace.TracerProvider] + clientTracer values.Value[oteltrace.Tracer] } var ( @@ -143,12 +169,26 @@ var ( ) // HTTP creates a new HTTP upstream server. -func HTTP(tlsConfig values.Value[*tls.Config]) HTTPUpstream { - up := &httpUpstream{ - serverPort: values.Deferred[int](), - router: mux.NewRouter(), - tlsConfig: tlsConfig, +func HTTP(tlsConfig values.Value[*tls.Config], opts ...HTTPUpstreamOption) HTTPUpstream { + options := HTTPUpstreamOptions{ + CommonUpstreamOptions: CommonUpstreamOptions{ + displayName: "HTTP Upstream", + }, } + for _, op := range opts { + op.applyHTTP(&options) + } + up := &httpUpstream{ + HTTPUpstreamOptions: options, + serverPort: values.Deferred[int](), + router: mux.NewRouter(), + tlsConfig: tlsConfig, + serverTracerProvider: values.Deferred[oteltrace.TracerProvider](), + clientTracerProvider: values.Deferred[oteltrace.TracerProvider](), + } + up.clientTracer = values.Bind(up.clientTracerProvider, func(tp oteltrace.TracerProvider) oteltrace.Tracer { + return tp.Tracer(trace.PomeriumCoreTracer) + }) up.RecordCaller() return up } @@ -185,12 +225,34 @@ func (h *httpUpstream) Run(ctx context.Context) error { if h.tlsConfig != nil { tlsConfig = h.tlsConfig.Value() } + if h.serverTracerProviderOverride != nil { + h.serverTracerProvider.Resolve(h.serverTracerProviderOverride) + } else { + h.serverTracerProvider.Resolve(trace.NewTracerProvider(ctx, h.displayName)) + } + if h.clientTracerProviderOverride != nil { + h.clientTracerProvider.Resolve(h.clientTracerProviderOverride) + } else { + h.clientTracerProvider.Resolve(trace.NewTracerProvider(ctx, "HTTP Client")) + } + h.router.Use(trace.NewHTTPMiddleware(otelhttp.WithTracerProvider(h.serverTracerProvider.Value()))) + server := &http.Server{ Handler: h.router, TLSConfig: tlsConfig, - BaseContext: func(net.Listener) context.Context { - return ctx - }, + // BaseContext: func(net.Listener) context.Context { + // return ctx + // }, + } + if h.delayShutdown { + return snippets.RunWithDelayedShutdown(ctx, + func() error { + return server.Serve(listener) + }, + func() { + _ = server.Shutdown(context.Background()) + }, + )() } errC := make(chan error, 1) go func() { @@ -198,7 +260,7 @@ func (h *httpUpstream) Run(ctx context.Context) error { }() select { case <-ctx.Done(): - server.Close() + _ = server.Shutdown(context.Background()) return context.Cause(ctx) case err := <-errC: return err @@ -217,7 +279,9 @@ func (h *httpUpstream) Post(r testenv.Route, opts ...RequestOption) (*http.Respo // Do implements HTTPUpstream. func (h *httpUpstream) Do(method string, r testenv.Route, opts ...RequestOption) (*http.Response, error) { - options := RequestOptions{} + options := RequestOptions{ + requestCtx: h.Env().Context(), + } options.apply(opts...) u, err := url.Parse(r.URL().Value()) if err != nil { @@ -229,37 +293,15 @@ func (h *httpUpstream) Do(method string, r testenv.Route, opts ...RequestOption) RawQuery: options.query.Encode(), }) } - ctx := h.Env().Context() + ctx, span := h.clientTracer.Value().Start(options.requestCtx, "httpUpstream.Do", oteltrace.WithAttributes( + attribute.String("method", method), + attribute.String("url", u.String()), + )) if options.trace != nil { ctx = httptrace.WithClientTrace(ctx, options.trace) } - req, err := http.NewRequestWithContext(ctx, method, u.String(), nil) - if err != nil { - return nil, err - } - switch body := options.body.(type) { - case string: - req.Body = io.NopCloser(strings.NewReader(body)) - case []byte: - req.Body = io.NopCloser(bytes.NewReader(body)) - case io.Reader: - req.Body = io.NopCloser(body) - case proto.Message: - buf, err := proto.Marshal(body) - if err != nil { - return nil, err - } - req.Body = io.NopCloser(bytes.NewReader(buf)) - req.Header.Set("Content-Type", "application/octet-stream") - default: - buf, err := json.Marshal(body) - if err != nil { - panic(fmt.Sprintf("unsupported body type: %T", body)) - } - req.Body = io.NopCloser(bytes.NewReader(buf)) - req.Header.Set("Content-Type", "application/json") - case nil: - } + options.requestCtx = ctx + defer span.End() newClient := func() *http.Client { transport := http.DefaultTransport.(*http.Transport).Clone() @@ -269,7 +311,12 @@ func (h *httpUpstream) Do(method string, r testenv.Route, opts ...RequestOption) } transport.DialTLSContext = nil c := http.Client{ - Transport: requestid.NewRoundTripper(transport), + Transport: otelhttp.NewTransport(transport, + otelhttp.WithTracerProvider(h.clientTracerProvider.Value()), + otelhttp.WithSpanNameFormatter(func(_ string, r *http.Request) string { + return fmt.Sprintf("Client: %s %s", r.Method, r.URL.Path) + }), + ), } c.Jar, _ = cookiejar.New(&cookiejar.Options{}) return &c @@ -281,14 +328,45 @@ func (h *httpUpstream) Do(method string, r testenv.Route, opts ...RequestOption) var cachedClient any var ok bool if cachedClient, ok = h.clientCache.Load(r); !ok { + span.AddEvent("creating new http client") cachedClient, _ = h.clientCache.LoadOrStore(r, newClient()) + } else { + span.AddEvent("using cached http client") } client = cachedClient.(*http.Client) } var resp *http.Response + resendCount := 0 if err := retry.Retry(ctx, "http", func(ctx context.Context) error { - var err error + req, err := http.NewRequestWithContext(ctx, method, u.String(), nil) + if err != nil { + return retry.NewTerminalError(err) + } + switch body := options.body.(type) { + case string: + req.Body = io.NopCloser(strings.NewReader(body)) + case []byte: + req.Body = io.NopCloser(bytes.NewReader(body)) + case io.Reader: + req.Body = io.NopCloser(body) + case proto.Message: + buf, err := proto.Marshal(body) + if err != nil { + return retry.NewTerminalError(err) + } + req.Body = io.NopCloser(bytes.NewReader(buf)) + req.Header.Set("Content-Type", "application/octet-stream") + default: + buf, err := json.Marshal(body) + if err != nil { + panic(fmt.Sprintf("unsupported body type: %T", body)) + } + req.Body = io.NopCloser(bytes.NewReader(buf)) + req.Header.Set("Content-Type", "application/json") + case nil: + } + if options.authenticateAs != "" { resp, err = authenticateFlow(ctx, client, req, options.authenticateAs) //nolint:bodyclose } else { @@ -296,46 +374,70 @@ func (h *httpUpstream) Do(method string, r testenv.Route, opts ...RequestOption) } // retry on connection refused if err != nil { + span.RecordError(err) var opErr *net.OpError if errors.As(err, &opErr) && opErr.Op == "dial" && opErr.Err.Error() == "connect: connection refused" { + span.AddEvent("Retrying on dial error") return err } return retry.NewTerminalError(err) } - if resp.StatusCode == http.StatusInternalServerError { + if resp.StatusCode/100 == 5 { + resendCount++ + _, _ = io.ReadAll(resp.Body) + _ = resp.Body.Close() + span.SetAttributes(semconv.HTTPRequestResendCount(resendCount)) + span.AddEvent("Retrying on 5xx error", oteltrace.WithAttributes( + attribute.String("status", resp.Status), + )) return errors.New(http.StatusText(resp.StatusCode)) } + span.SetStatus(codes.Ok, "request completed successfully") return nil - }, retry.WithMaxInterval(100*time.Millisecond)); err != nil { + }, + retry.WithInitialInterval(1*time.Millisecond), + retry.WithMaxInterval(100*time.Millisecond), + ); err != nil { return nil, err } return resp, nil } func authenticateFlow(ctx context.Context, client *http.Client, req *http.Request, email string) (*http.Response, error) { + span := oteltrace.SpanFromContext(ctx) var res *http.Response originalHostname := req.URL.Hostname() res, err := client.Do(req) if err != nil { + span.RecordError(err) return nil, err } - location := res.Request.URL if location.Hostname() == originalHostname { // already authenticated - return res, err + span.SetStatus(codes.Ok, "already authenticated") + return res, nil } - defer res.Body.Close() fs := forms.Parse(res.Body) + _, _ = io.ReadAll(res.Body) + _ = res.Body.Close() if len(fs) > 0 { f := fs[0] f.Inputs["email"] = email f.Inputs["token_expiration"] = strconv.Itoa(int((time.Hour * 24).Seconds())) + span.AddEvent("submitting form", oteltrace.WithAttributes(attribute.String("location", location.String()))) formReq, err := f.NewRequestWithContext(ctx, location) if err != nil { + span.RecordError(err) return nil, err } - return client.Do(formReq) + resp, err := client.Do(formReq) + if err != nil { + span.RecordError(err) + return nil, err + } + span.SetStatus(codes.Ok, "form submitted successfully") + return resp, nil } return nil, fmt.Errorf("test bug: expected IDP login form") } diff --git a/internal/testenv/upstreams/options.go b/internal/testenv/upstreams/options.go new file mode 100644 index 000000000..c0bad32b5 --- /dev/null +++ b/internal/testenv/upstreams/options.go @@ -0,0 +1,64 @@ +package upstreams + +import ( + oteltrace "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" +) + +type CommonUpstreamOptions struct { + displayName string + clientTracerProviderOverride oteltrace.TracerProvider + serverTracerProviderOverride oteltrace.TracerProvider + delayShutdown bool +} + +type CommonUpstreamOption interface { + GRPCUpstreamOption + HTTPUpstreamOption +} + +type commonUpstreamOption func(o *CommonUpstreamOptions) + +// applyGRPC implements CommonUpstreamOption. +func (c commonUpstreamOption) applyGRPC(o *GRPCUpstreamOptions) { c(&o.CommonUpstreamOptions) } + +// applyHTTP implements CommonUpstreamOption. +func (c commonUpstreamOption) applyHTTP(o *HTTPUpstreamOptions) { c(&o.CommonUpstreamOptions) } + +func WithDisplayName(displayName string) CommonUpstreamOption { + return commonUpstreamOption(func(o *CommonUpstreamOptions) { + o.displayName = displayName + }) +} + +func WithNoClientTracing() CommonUpstreamOption { + return commonUpstreamOption(func(o *CommonUpstreamOptions) { + o.clientTracerProviderOverride = noop.NewTracerProvider() + }) +} + +func WithNoServerTracing() CommonUpstreamOption { + return commonUpstreamOption(func(o *CommonUpstreamOptions) { + o.serverTracerProviderOverride = noop.NewTracerProvider() + }) +} + +func WithClientTracerProvider(tp oteltrace.TracerProvider) CommonUpstreamOption { + return commonUpstreamOption(func(o *CommonUpstreamOptions) { + o.clientTracerProviderOverride = tp + }) +} + +func WithServerTracerProvider(tp oteltrace.TracerProvider) CommonUpstreamOption { + return commonUpstreamOption(func(o *CommonUpstreamOptions) { + o.serverTracerProviderOverride = tp + }) +} + +// WithDelayedShutdown keeps the server alive until the test environment has +// fully shut down, instead of stopping it during the shutdown sequence. +func WithDelayedShutdown() CommonUpstreamOption { + return commonUpstreamOption(func(o *CommonUpstreamOptions) { + o.delayShutdown = true + }) +} diff --git a/internal/testenv/values/value.go b/internal/testenv/values/value.go index 03d9ac8f2..39c25e782 100644 --- a/internal/testenv/values/value.go +++ b/internal/testenv/values/value.go @@ -69,7 +69,7 @@ func (p *value[T]) Resolve(value T) { p.ResolveFunc(func() T { return value }) } -// Bind creates a new [MutableValue] whose ultimate value depends on the result +// Bind creates a new [Value] whose ultimate value depends on the result // of another [Value] that may not yet be available. When Value() is called on // the result, it will cascade and trigger the full chain of initialization // functions necessary to produce the final value. @@ -118,3 +118,49 @@ func (s List[T]) Value() []T { } return values } + +// Chain is like [Bind], returning a Value[V] whose ultimate value depends on +// the result of a Value[U] which is obtained from the concrete T given in dt. +// It is intended to be used when T has a method returning a Value[U], and +// you want to access U, but don't need T. +// +// Example usage: +// +// type Foo interface { Number() Value[int] } +// var foo Foo +// values.Chain(foo, Foo.Number, func(number int) string { ... }) +func Chain[T any, U any, V any]( + dt Value[T], + callback1 func(value T) Value[U], + callback2 func(value U) V, +) Value[V] { + dv := Deferred[V]() + dv.ResolveFunc(func() V { + du := callback1(dt.Value()) + return callback2(du.Value()) + }) + return dv +} + +// Chain2 is like [Chain], but with two levels of indirection. +// +// Example usage: +// +// type Foo interface { Bar() Value[Bar] } +// type Bar interface { Number() Value[int] } +// var foo Foo +// values.Chain(foo, Foo.Bar, bar.Number, func(number int) string { ... }) +func Chain2[T any, U any, V any, W any]( + dt Value[T], + callback1 func(value T) Value[U], + callback2 func(value U) Value[V], + callback3 func(value V) W, +) Value[W] { + dw := Deferred[W]() + dw.ResolveFunc(func() W { + du := callback1(dt.Value()) + dv := callback2(du.Value()) + return callback3(dv.Value()) + }) + return dw +} diff --git a/internal/testutil/tracing.go b/internal/testutil/tracing.go new file mode 100644 index 000000000..9b1365f18 --- /dev/null +++ b/internal/testutil/tracing.go @@ -0,0 +1,542 @@ +package testutil + +import ( + "cmp" + "context" + "encoding/json" + "fmt" + "maps" + "runtime" + "slices" + "strings" + "sync" + "testing" + "time" + "unique" + + gocmp "github.com/google/go-cmp/cmp" + "github.com/pomerium/pomerium/internal/telemetry/trace" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + oteltrace "go.opentelemetry.io/otel/trace" + coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" + commonv1 "go.opentelemetry.io/proto/otlp/common/v1" + resourcev1 "go.opentelemetry.io/proto/otlp/resource/v1" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/testing/protocmp" +) + +type TraceResults struct { + resourceSpans []*tracev1.ResourceSpans + + GetResources func() []*resourcev1.Resource + GetTraces func() *Traces +} + +type Traces struct { + ByID map[unique.Handle[oteltrace.TraceID]]*TraceDetails + ByName map[string]TraceDetailsList + ByParticipant map[string]TraceDetailsList +} + +func (t *Traces) WithoutErrors() *Traces { + byID := make(map[unique.Handle[oteltrace.TraceID]]*TraceDetails, len(t.ByID)) + for k, v := range t.ByID { + if len(v.Errors) > 0 { + continue + } + byID[k] = v + } + byName := make(map[string]TraceDetailsList) + for k, v := range t.ByName { + filtered := v.WithoutErrors() + if len(filtered) == 0 { + continue + } + byName[k] = filtered + } + byParticipant := make(map[string]TraceDetailsList) + for k, v := range t.ByParticipant { + filtered := v.WithoutErrors() + if len(filtered) == 0 { + continue + } + byParticipant[k] = filtered + } + return &Traces{ + ByID: byID, + ByName: byName, + ByParticipant: byParticipant, + } +} + +type TraceDetails struct { + ID unique.Handle[oteltrace.TraceID] + Name string + Spans []*SpanDetails + Services []string + StartTime time.Time + EndTime time.Time + Duration time.Duration + Errors []int // indexes into Spans +} + +func (td *TraceDetails) Equal(other *TraceDetails) (bool, string) { + diffSpans := func(a, b []*SpanDetails) (bool, string) { + for i := range len(a) { + aRaw := proto.Clone(a[i].Raw).(*tracev1.Span) + trace.FormatSpanName(aRaw) + bRaw := proto.Clone(b[i].Raw).(*tracev1.Span) + trace.FormatSpanName(bRaw) + diff := gocmp.Diff(aRaw, bRaw, protocmp.Transform()) + if diff != "" { + return false, diff + } + } + return true, "" + } + if td.ID != other.ID { + return false, fmt.Sprintf("traces are trivially not equal: ID %s (actual) != %s (expected)", td.ID.Value(), other.ID.Value()) + } + if len(td.Spans) != len(other.Spans) { + return false, fmt.Sprintf("traces are trivially not equal: len(spans) %d (actual) != %d (expected)", len(td.Spans), len(other.Spans)) + } + if !td.StartTime.Equal(other.StartTime) { + return false, fmt.Sprintf("traces are trivially not equal: start time %s (actual) != %s (expected)", td.StartTime, other.StartTime) + } + if !td.EndTime.Equal(other.EndTime) { + return false, fmt.Sprintf("traces are trivially not equal: end time %s (actual) != %s (expected)", td.EndTime, other.EndTime) + } + return diffSpans(td.Spans, other.Spans) +} + +type TraceDetailsList []*TraceDetails + +func (list TraceDetailsList) WithoutExportRPCs() TraceDetailsList { + out := make(TraceDetailsList, 0, len(list)) + for _, td := range list { + if strings.Contains(td.Name, "opentelemetry.proto.collector.trace.v1.TraceService/Export") { + continue + } + out = append(out, td) + } + return out +} + +func (list TraceDetailsList) WithoutErrors() TraceDetailsList { + out := make(TraceDetailsList, 0, len(list)) + for _, td := range list { + if len(td.Errors) > 0 { + continue + } + out = append(out, td) + } + return out +} + +func (td *TraceDetails) SpanTree() *SpanTree { + nodesByID := map[oteltrace.SpanID]*SpanTreeNode{} + nodesByID[oteltrace.SpanID([8]byte{})] = &SpanTreeNode{} // root node + for _, span := range td.Spans { + spanID, _ := trace.ToSpanID(span.Raw.SpanId) + nodesByID[spanID] = &SpanTreeNode{ + Span: span, + } + } + detachedNodesByID := map[oteltrace.SpanID]*SpanTreeNode{} + for _, span := range td.Spans { + spanID, _ := trace.ToSpanID(span.Raw.SpanId) + parentSpanID, _ := trace.ToSpanID(span.Raw.ParentSpanId) + if _, ok := nodesByID[parentSpanID]; !ok { + detachedNodesByID[parentSpanID] = &SpanTreeNode{} + nodesByID[parentSpanID] = detachedNodesByID[parentSpanID] + } + nodesByID[spanID].Parent = nodesByID[parentSpanID] + nodesByID[parentSpanID].Children = append(nodesByID[parentSpanID].Children, nodesByID[spanID]) + } + for _, node := range nodesByID { + slices.SortFunc(node.Children, func(a, b *SpanTreeNode) int { + return cmp.Compare(a.Span.Raw.StartTimeUnixNano, b.Span.Raw.StartTimeUnixNano) + }) + } + return &SpanTree{ + Root: nodesByID[oteltrace.SpanID([8]byte{})], + DetachedParents: detachedNodesByID, + } +} + +type SpanDetails struct { + Raw *tracev1.Span + Resource *resourcev1.Resource + Scope *commonv1.InstrumentationScope + StartTime time.Time + EndTime time.Time + Duration time.Duration + Service string +} + +func NewTraceResults(resourceSpans []*tracev1.ResourceSpans) *TraceResults { + tr := &TraceResults{ + resourceSpans: resourceSpans, + } + tr.GetResources = sync.OnceValue(tr.computeResources) + tr.GetTraces = sync.OnceValue(tr.computeTraces) + return tr +} + +func (tr *TraceResults) computeResources() []*resourcev1.Resource { + resources := []*resourcev1.Resource{} + for _, res := range tr.resourceSpans { + resources = append(resources, res.Resource) + } + return resources +} + +func (tr *TraceResults) computeTraces() *Traces { + tracesByID := map[unique.Handle[oteltrace.TraceID]]*TraceDetails{} + for _, resSpan := range tr.resourceSpans { + resource := resSpan.Resource + for _, scopeSpans := range resSpan.ScopeSpans { + scope := scopeSpans.Scope + for _, span := range scopeSpans.Spans { + traceID, _ := trace.ToTraceID(span.TraceId) + var details *TraceDetails + if d, ok := tracesByID[traceID]; ok { + details = d + } else { + details = &TraceDetails{ + ID: traceID, + } + tracesByID[traceID] = details + } + svc := "" + for _, attr := range resource.Attributes { + if attr.Key == "service.name" { + svc = attr.Value.GetStringValue() + break + } + } + details.Spans = append(details.Spans, &SpanDetails{ + Raw: span, + Resource: resource, + Scope: scope, + StartTime: time.Unix(0, int64(span.StartTimeUnixNano)), + EndTime: time.Unix(0, int64(span.EndTimeUnixNano)), + Duration: time.Duration(span.EndTimeUnixNano - span.StartTimeUnixNano), + Service: svc, + }) + if span.Status != nil { + if span.Status.Code == tracev1.Status_STATUS_CODE_ERROR { + details.Errors = append(details.Errors, len(details.Spans)-1) + } + } + } + } + } + + tracesByName := map[string]TraceDetailsList{} + tracesByParticipant := map[string]TraceDetailsList{} + // sort spans by start time and compute durations + for _, td := range tracesByID { + slices.SortFunc(td.Spans, func(a, b *SpanDetails) int { + return cmp.Compare(a.Raw.StartTimeUnixNano, b.Raw.StartTimeUnixNano) + }) + startTime := td.Spans[0].Raw.StartTimeUnixNano + endTime := td.Spans[0].Raw.EndTimeUnixNano + serviceNames := map[string]struct{}{} + for _, span := range td.Spans { + startTime = min(startTime, span.Raw.StartTimeUnixNano) + endTime = max(endTime, span.Raw.EndTimeUnixNano) + if span.Service != "" { + serviceNames[span.Service] = struct{}{} + } + } + td.StartTime = time.Unix(0, int64(startTime)) + td.EndTime = time.Unix(0, int64(endTime)) + td.Duration = td.EndTime.Sub(td.StartTime) + td.Services = slices.Sorted(maps.Keys(serviceNames)) + td.Name = fmt.Sprintf("%s: %s", td.Spans[0].Service, td.Spans[0].Raw.Name) + tracesByName[td.Name] = append(tracesByName[td.Name], td) + for svc := range serviceNames { + tracesByParticipant[svc] = append(tracesByParticipant[svc], td) + } + } + + return &Traces{ + ByID: tracesByID, + ByName: tracesByName, + ByParticipant: tracesByParticipant, + } +} + +type SpanTree struct { + Root *SpanTreeNode + DetachedParents map[oteltrace.SpanID]*SpanTreeNode +} + +type SpanTreeNode struct { + Span *SpanDetails + Parent *SpanTreeNode + Children []*SpanTreeNode +} + +type Match struct { + Name string + TraceCount any + Services []string +} + +type ( + GreaterOrEqual int + Greater int + + // Any makes no assertions on the trace count. If the trace is not found, it + // doesn't count against the Exact match option. + Any struct{} + + // EqualToMatch asserts that the value is the same as the value of another + // match (by name) + EqualToMatch string + // GreaterThanMatch asserts that the value is greater than the value of + // another match (by name) + GreaterThanMatch string +) + +type MatchOptions struct { + // If true, asserts that there is exactly one [Match] entry per result + Exact bool + // If true, asserts that no traces contain detached spans + CheckDetachedSpans bool +} + +func (tr *TraceResults) MatchTraces(t testing.TB, opts MatchOptions, matches ...Match) { + t.Helper() + traces := tr.GetTraces() + matchArgsByName := map[string]Match{} + for i, m := range matches { + if m.Name != "" { + require.NotContains(t, matchArgsByName, m.Name, "duplicate name") + matchArgsByName[m.Name] = m + if traceDetails, ok := traces.ByName[m.Name]; ok { + switch tc := m.TraceCount.(type) { + case GreaterOrEqual: + assert.GreaterOrEqualf(t, len(traceDetails), int(tc), + "[match %d]: expected %q to have >=%d traces, but found %d", + i+1, m.Name, int(tc), len(traceDetails)) + case Greater: + assert.Greaterf(t, len(traceDetails), int(tc), + "[match %d]: expected %q to have >%d traces, but found %d", + i+1, m.Name, int(tc), len(traceDetails)) + case GreaterThanMatch: + assert.Greaterf(t, len(traceDetails), len(traces.ByName[string(tc)]), + "[match %d]: expected %q to have >%d traces (value of %s), but found %d", + i+1, m.Name, len(traces.ByName[string(tc)]), string(tc), len(traceDetails)) + case EqualToMatch: + assert.Equalf(t, len(traceDetails), len(traces.ByName[string(tc)]), + "[match %d]: expected %q to have %d traces (value of %s), but found %d", + i+1, m.Name, len(traces.ByName[string(tc)]), string(tc), len(traceDetails)) + case Any: + case int: + s := "s" + if tc == 1 { + s = "" + } + assert.Lenf(t, traceDetails, tc, + "[match %d]: expected %q to have %d trace%s, but found %d", + i+1, m.Name, tc, s, len(traceDetails)) + } + + if m.Services != nil { + for _, trace := range traceDetails { + assert.ElementsMatch(t, m.Services, trace.Services) + } + } + } else if _, ok := m.TraceCount.(Any); !ok { + t.Errorf("no traces with name %q found", m.Name) + } + } + } + if opts.CheckDetachedSpans { + for _, trace := range traces.ByID { + tree := trace.SpanTree() + if !assert.Empty(t, tree.DetachedParents) { + for spanID, node := range tree.DetachedParents { + t.Log("------------------------------------") + t.Logf("span id: %s", spanID) + if len(node.Children) != 0 { + t.Log("children:") + } + for _, c := range node.Children { + t.Log(protojson.Format(c.Span.Raw)) + } + t.Log("------------------------------------") + } + } + } + } + if opts.Exact { + expected := slices.Sorted(maps.Keys(matchArgsByName)) + actual := slices.Sorted(maps.Keys(traces.ByName)) + for name, match := range matchArgsByName { + if _, ok := traces.ByName[name]; !ok { + if _, ok := match.TraceCount.(Any); ok { + expected = slices.DeleteFunc(expected, func(s string) bool { return s == name }) + } + } + } + assert.Equal(t, expected, actual) + } +} + +func (tr *TraceResults) AssertEqual(t testing.TB, expectedResults *TraceResults, msgFmtAndArgs ...any) { + t.Helper() + actualTraces := tr.GetTraces() + expectedTraces := expectedResults.GetTraces() + for traceID, expected := range expectedTraces.ByID { + if actual, ok := actualTraces.ByID[traceID]; !ok { + if len(msgFmtAndArgs) > 0 { + t.Errorf("expected trace id %s not found (%s)", traceID.Value().String(), + fmt.Sprintf(msgFmtAndArgs[0].(string), msgFmtAndArgs[1:]...)) + } else { + t.Errorf("expected trace id %s not found", traceID.Value().String()) + } + } else { + if equal, diff := actual.Equal(expected); !equal { + if len(msgFmtAndArgs) > 0 { + t.Errorf("trace %s is not equal (%s):\n%s", traceID.Value().String(), + fmt.Sprintf(msgFmtAndArgs[0].(string), msgFmtAndArgs[1:]...), diff) + } else { + t.Errorf("trace %s is not equal:\n%s", traceID.Value().String(), diff) + } + } + } + } + for traceID := range actualTraces.ByID { + if _, ok := expectedTraces.ByID[traceID]; !ok { + if len(msgFmtAndArgs) > 0 { + t.Errorf("unexpected trace id %s found (%s)", traceID.Value().String(), + fmt.Sprintf(msgFmtAndArgs[0].(string), msgFmtAndArgs[1:]...)) + } else { + t.Errorf("unexpected trace id %s found", traceID.Value().String()) + } + } + } +} + +func FlattenResourceSpans(lists [][]*tracev1.ResourceSpans) []*tracev1.ResourceSpans { + res := trace.NewBuffer() + for _, list := range lists { + for _, resource := range list { + resInfo := trace.NewResourceInfo(resource.Resource, resource.SchemaUrl) + for _, scope := range resource.ScopeSpans { + scopeInfo := trace.NewScopeInfo(scope.Scope, scope.SchemaUrl) + for _, span := range scope.Spans { + res.Insert(resInfo, scopeInfo, span) + } + } + } + } + return res.Flush() +} + +func FlattenExportRequests(reqs []*coltracepb.ExportTraceServiceRequest) []*tracev1.ResourceSpans { + lists := make([][]*tracev1.ResourceSpans, len(reqs)) + for i, req := range reqs { + lists[i] = req.ResourceSpans + } + return FlattenResourceSpans(lists) +} + +type EventRecording struct { + events []trace.DebugEvent + normalizedTo time.Time +} + +func LoadEventRecording(raw []byte) (*EventRecording, error) { + events := []trace.DebugEvent{} + if err := json.Unmarshal(raw, &events); err != nil { + return nil, err + } + for i := 1; i < len(events); i++ { + if events[i].Timestamp.Before(events[i-1].Timestamp) { + return nil, fmt.Errorf("invalid timestamps: event %d occurred before event %d", i, i-1) + } + } + return &EventRecording{ + events: events, + }, nil +} + +func (er *EventRecording) Normalize(startTime time.Time) { + if len(er.events) == 0 { + return + } + er.normalizedTo = startTime + offset := startTime.Sub(er.events[0].Timestamp) + for i, ev := range er.events { + er.events[i].Timestamp = ev.Timestamp.Add(offset) + for _, resSpan := range ev.Request.ResourceSpans { + for _, scopeSpans := range resSpan.ScopeSpans { + for _, span := range scopeSpans.Spans { + span.StartTimeUnixNano += uint64(offset) + span.EndTimeUnixNano += uint64(offset) + for _, event := range span.Events { + event.TimeUnixNano += uint64(offset) + } + } + } + } + } +} + +func (er *EventRecording) NormalizedTo() time.Time { + return er.normalizedTo +} + +type EventCallbackFunc = func(ctx context.Context, req *coltracepb.ExportTraceServiceRequest) (*coltracepb.ExportTraceServiceResponse, error) + +func (er *EventRecording) Events() []trace.DebugEvent { + return er.events +} + +func (er *EventRecording) Clone() *EventRecording { + clonedEvents := make([]trace.DebugEvent, 0, len(er.events)) + for _, ev := range er.events { + clonedEvents = append(clonedEvents, trace.DebugEvent{ + Timestamp: ev.Timestamp, + Request: proto.Clone(ev.Request).(*coltracepb.ExportTraceServiceRequest), + }) + } + c := &EventRecording{ + events: clonedEvents, + normalizedTo: er.normalizedTo, + } + return c +} + +func (er *EventRecording) Replay(callback EventCallbackFunc) error { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + durations := make([]time.Duration, 0, len(er.events)-1) + for i := 1; i < len(er.events); i++ { + durations = append(durations, er.events[i].Timestamp.Sub(er.events[i-1].Timestamp)) + } + + var wg sync.WaitGroup + wg.Add(len(er.events)) + er.Normalize(time.Now()) + for i, ev := range er.events { + go func() { + callback(context.Background(), ev.Request) + wg.Done() + }() + if i < len(er.events)-1 { + time.Sleep(durations[i]) + } + } + wg.Wait() + return nil +} diff --git a/internal/urlutil/proxy.go b/internal/urlutil/proxy.go index 87ef13ebf..7386b8e5d 100644 --- a/internal/urlutil/proxy.go +++ b/internal/urlutil/proxy.go @@ -47,6 +47,14 @@ func GetCallbackURLForRedirectURI(r *http.Request, encodedSessionJWT, rawRedirec if r.FormValue(QueryIsProgrammatic) == "true" { callbackParams.Set(QueryIsProgrammatic, "true") } + // propagate trace context + if tracecontext := r.FormValue(QueryTraceparent); tracecontext != "" { + callbackParams.Set(QueryTraceparent, tracecontext) + } + if tracestate := r.FormValue(QueryTracestate); tracestate != "" { + callbackParams.Set(QueryTracestate, tracestate) + } + // add our encoded and encrypted route-session JWT to a query param callbackParams.Set(QuerySessionEncrypted, encodedSessionJWT) callbackParams.Set(QueryRedirectURI, redirectURI.String()) diff --git a/internal/urlutil/query_params.go b/internal/urlutil/query_params.go index ac5a127bc..3ddee3cc6 100644 --- a/internal/urlutil/query_params.go +++ b/internal/urlutil/query_params.go @@ -20,6 +20,8 @@ const ( QuerySessionState = "pomerium_session_state" QueryVersion = "pomerium_version" QueryRequestUUID = "pomerium_request_uuid" + QueryTraceparent = "pomerium_traceparent" + QueryTracestate = "pomerium_tracestate" ) // URL signature based query params used for verifying the authenticity of a URL. diff --git a/internal/zero/controller/usagereporter/usagereporter_test.go b/internal/zero/controller/usagereporter/usagereporter_test.go index 633227abd..ce25671b2 100644 --- a/internal/zero/controller/usagereporter/usagereporter_test.go +++ b/internal/zero/controller/usagereporter/usagereporter_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "go.opentelemetry.io/otel/trace" "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/protobuf/types/known/timestamppb" @@ -37,7 +38,7 @@ func TestUsageReporter(t *testing.T) { t.Cleanup(cancel) cc := testutil.NewGRPCServer(t, func(srv *grpc.Server) { - databrokerpb.RegisterDataBrokerServiceServer(srv, databroker.New(ctx)) + databrokerpb.RegisterDataBrokerServiceServer(srv, databroker.New(ctx, trace.NewNoopTracerProvider())) }) t.Cleanup(func() { cc.Close() }) diff --git a/pkg/cmd/pomerium/pomerium.go b/pkg/cmd/pomerium/pomerium.go index 1014d247e..00f92767c 100644 --- a/pkg/cmd/pomerium/pomerium.go +++ b/pkg/cmd/pomerium/pomerium.go @@ -3,11 +3,9 @@ package pomerium import ( "context" + "errors" "fmt" "net/http" - "os" - "os/signal" - "syscall" envoy_service_auth_v3 "github.com/envoyproxy/go-control-plane/envoy/service/auth/v3" "go.uber.org/automaxprocs/maxprocs" @@ -24,50 +22,80 @@ import ( "github.com/pomerium/pomerium/internal/events" "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/registry" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/version" derivecert_config "github.com/pomerium/pomerium/pkg/derivecert/config" "github.com/pomerium/pomerium/pkg/envoy" "github.com/pomerium/pomerium/pkg/envoy/files" "github.com/pomerium/pomerium/proxy" + oteltrace "go.opentelemetry.io/otel/trace" ) -type RunOptions struct { +type Options struct { fileMgr *filemgr.Manager envoyServerOptions []envoy.ServerOption databrokerServerOptions []databroker_service.Option } -type RunOption func(*RunOptions) +type Option func(*Options) -func (o *RunOptions) apply(opts ...RunOption) { +func (o *Options) apply(opts ...Option) { for _, op := range opts { op(o) } } -func WithOverrideFileManager(fileMgr *filemgr.Manager) RunOption { - return func(o *RunOptions) { +func WithOverrideFileManager(fileMgr *filemgr.Manager) Option { + return func(o *Options) { o.fileMgr = fileMgr } } -func WithEnvoyServerOptions(opts ...envoy.ServerOption) RunOption { - return func(o *RunOptions) { +func WithEnvoyServerOptions(opts ...envoy.ServerOption) Option { + return func(o *Options) { o.envoyServerOptions = append(o.envoyServerOptions, opts...) } } -func WithDataBrokerServerOptions(opts ...databroker_service.Option) RunOption { - return func(o *RunOptions) { +func WithDataBrokerServerOptions(opts ...databroker_service.Option) Option { + return func(o *Options) { o.databrokerServerOptions = append(o.databrokerServerOptions, opts...) } } // Run runs the main pomerium application. -func Run(ctx context.Context, src config.Source, opts ...RunOption) error { - options := RunOptions{} +func Run(ctx context.Context, src config.Source, opts ...Option) error { + p := New(opts...) + tracerProvider := trace.NewTracerProvider(ctx, "Pomerium") + + if err := p.Start(ctx, tracerProvider, src); err != nil { + return err + } + return p.Wait() +} + +var ErrShutdown = errors.New("Shutdown() called") + +type Pomerium struct { + Options + errGroup *errgroup.Group + + cancel context.CancelCauseFunc + envoyServer *envoy.Server +} + +func New(opts ...Option) *Pomerium { + options := Options{} options.apply(opts...) + return &Pomerium{ + Options: options, + } +} + +func (p *Pomerium) Start(ctx context.Context, tracerProvider oteltrace.TracerProvider, src config.Source) error { + updateTraceClient(ctx, src.GetConfig()) + ctx, p.cancel = context.WithCancelCause(ctx) _, _ = maxprocs.Set(maxprocs.Logger(func(s string, i ...any) { log.Ctx(ctx).Debug().Msgf(s, i...) })) evt := log.Ctx(ctx).Info(). @@ -82,9 +110,8 @@ func Run(ctx context.Context, src config.Source, opts ...RunOption) error { if err != nil { return err } - src = databroker.NewConfigSource(ctx, src, databroker.EnableConfigValidation(true)) - logMgr := config.NewLogManager(ctx, src) - defer logMgr.Close() + src = databroker.NewConfigSource(ctx, tracerProvider, src, databroker.EnableConfigValidation(true)) + _ = config.NewLogManager(ctx, src) // trigger changes when underlying files are changed src = config.NewFileWatcherSource(ctx, src) @@ -98,18 +125,16 @@ func Run(ctx context.Context, src config.Source, opts ...RunOption) error { http.DefaultTransport = config.NewHTTPTransport(src) metricsMgr := config.NewMetricsManager(ctx, src) - defer metricsMgr.Close() - traceMgr := config.NewTraceManager(ctx, src) - defer traceMgr.Close() eventsMgr := events.New() - fileMgr := options.fileMgr + fileMgr := p.fileMgr if fileMgr == nil { fileMgr = filemgr.NewManager() } cfg := src.GetConfig() + src.OnConfigChange(ctx, updateTraceClient) // setup the control plane controlPlane, err := controlplane.NewServer(ctx, cfg, metricsMgr, eventsMgr, fileMgr) @@ -137,11 +162,13 @@ func Run(ctx context.Context, src config.Source, opts ...RunOption) error { Msg("server started") // create envoy server - envoyServer, err := envoy.NewServer(ctx, src, controlPlane.Builder, options.envoyServerOptions...) + p.envoyServer, err = envoy.NewServer(ctx, src, controlPlane.Builder, p.envoyServerOptions...) if err != nil { return fmt.Errorf("error creating envoy server: %w", err) } - defer envoyServer.Close() + context.AfterFunc(ctx, func() { + p.envoyServer.Close() + }) // add services if err := setupAuthenticate(ctx, src, controlPlane); err != nil { @@ -156,50 +183,52 @@ func Run(ctx context.Context, src config.Source, opts ...RunOption) error { } var dataBrokerServer *databroker_service.DataBroker if config.IsDataBroker(src.GetConfig().Options.Services) { - dataBrokerServer, err = setupDataBroker(ctx, src, controlPlane, eventsMgr, options.databrokerServerOptions...) + dataBrokerServer, err = setupDataBroker(ctx, src, controlPlane, eventsMgr, p.databrokerServerOptions...) if err != nil { return fmt.Errorf("setting up databroker: %w", err) } } - if err = setupRegistryReporter(ctx, src); err != nil { + if err = setupRegistryReporter(ctx, tracerProvider, src); err != nil { return fmt.Errorf("setting up registry reporter: %w", err) } if err := setupProxy(ctx, src, controlPlane); err != nil { return err } - ctx, cancel := context.WithCancel(ctx) - go func(ctx context.Context) { - ch := make(chan os.Signal, 2) - defer signal.Stop(ch) - - signal.Notify(ch, os.Interrupt) - signal.Notify(ch, syscall.SIGTERM) - - select { - case <-ch: - case <-ctx.Done(): - } - cancel() - }(ctx) - // run everything - eg, ctx := errgroup.WithContext(ctx) + p.errGroup, ctx = errgroup.WithContext(ctx) if authorizeServer != nil { - eg.Go(func() error { + p.errGroup.Go(func() error { return authorizeServer.Run(ctx) }) } - eg.Go(func() error { + p.errGroup.Go(func() error { return controlPlane.Run(ctx) }) if dataBrokerServer != nil { - eg.Go(func() error { + p.errGroup.Go(func() error { return dataBrokerServer.Run(ctx) }) } - return eg.Wait() + return nil +} + +func (p *Pomerium) Shutdown(ctx context.Context) error { + _ = trace.WaitForSpans(ctx, p.envoyServer.ExitGracePeriod()) + var errs []error + errs = append(errs, p.envoyServer.Close()) // this only errors if signaling envoy fails + p.cancel(ErrShutdown) + errs = append(errs, p.Wait()) + return errors.Join(errs...) +} + +func (p *Pomerium) Wait() error { + err := p.errGroup.Wait() + if errors.Is(err, ErrShutdown) { + return nil + } + return err } func setupAuthenticate(ctx context.Context, src config.Source, controlPlane *controlplane.Server) error { @@ -253,8 +282,8 @@ func setupDataBroker(ctx context.Context, return svc, nil } -func setupRegistryReporter(ctx context.Context, src config.Source) error { - reporter := registry.NewReporter() +func setupRegistryReporter(ctx context.Context, tracerProvider oteltrace.TracerProvider, src config.Source) error { + reporter := registry.NewReporter(tracerProvider) src.OnConfigChange(ctx, reporter.OnConfigChange) reporter.OnConfigChange(ctx, src.GetConfig()) return nil @@ -280,3 +309,31 @@ func setupProxy(ctx context.Context, src config.Source, controlPlane *controlpla return nil } + +func updateTraceClient(ctx context.Context, cfg *config.Config) { + sc, ok := trace.RemoteClientFromContext(ctx).(trace.SyncClient) + if !ok { + return + } + newClient, err := config.NewTraceClientFromOptions(cfg.Options) + if errors.Is(err, config.ErrNoTracingConfig) { + newClient = trace.NewRemoteClientFromEnv() + err = nil + } + if err != nil { + log.Ctx(ctx).Warn().Err(err).Msg("error configuring trace client") + } else { + go func() { + if err := sc.Update(ctx, newClient); err != nil { + log.Ctx(ctx). + Warn(). + Err(err). + Msg("error updating trace client") + } + log.Ctx(ctx). + Info(). + Str("provider", cfg.Options.TracingProvider). + Msg("trace client updated") + }() + } +} diff --git a/pkg/envoy/envoy.go b/pkg/envoy/envoy.go index 78c80a308..8224f3647 100644 --- a/pkg/envoy/envoy.go +++ b/pkg/envoy/envoy.go @@ -16,6 +16,7 @@ import ( "strconv" "strings" "sync" + stdatomic "sync/atomic" "syscall" "time" @@ -42,6 +43,7 @@ type Server struct { wd string cmd *exec.Cmd cmdExited chan struct{} + closing stdatomic.Bool builder *envoyconfig.Builder resourceMonitor ResourceMonitor @@ -59,6 +61,10 @@ type ServerOptions struct { exitGracePeriod time.Duration } +func (o *ServerOptions) ExitGracePeriod() time.Duration { + return o.exitGracePeriod +} + type ServerOption func(*ServerOptions) func (o *ServerOptions) apply(opts ...ServerOption) { @@ -122,8 +128,13 @@ func NewServer(ctx context.Context, src config.Source, builder *envoyconfig.Buil return srv, nil } -// Close kills any underlying envoy process. +// Close attempts to gracefully shut down a running envoy server. If envoy +// does not exit within the defined grace period, it will be killed. Server +// cannot be used again after Close is called. func (srv *Server) Close() error { + if !srv.closing.CompareAndSwap(false, true) { + return nil + } srv.monitorProcessCancel() srv.mu.Lock() @@ -156,6 +167,10 @@ func (srv *Server) Close() error { } func (srv *Server) onConfigChange(ctx context.Context, cfg *config.Config) { + if srv.closing.Load() { + // do not attempt to update the configuration after Close is called + return + } srv.update(ctx, cfg) } @@ -233,10 +248,10 @@ func (srv *Server) run(ctx context.Context, cfg *config.Config) error { go func() { pid := cmd.Process.Pid err := srv.monitorProcess(monitorProcessCtx, int32(pid)) - if err != nil && ctx.Err() == nil { - // If the envoy subprocess exits and ctx is not done, issue a fatal error. - // If ctx is done, the server is already exiting, and envoy is expected - // to be stopped along with it. + if err != nil && ctx.Err() == nil && !srv.closing.Load() { + // If the envoy subprocess exits and ctx is not done (or waiting for envoy + // to gracefully stop), issue a fatal error. If ctx is done, the server is + // already exiting, and envoy is expected to be stopped along with it. log.Ctx(ctx). Fatal(). Int("pid", pid). diff --git a/pkg/envoy/get-envoy/main.go b/pkg/envoy/get-envoy/main.go index 32c453533..a93d5aea0 100644 --- a/pkg/envoy/get-envoy/main.go +++ b/pkg/envoy/get-envoy/main.go @@ -15,14 +15,14 @@ import ( ) var ( - envoyVersion = "1.31.0" + envoyVersion = "1.32.2" targets = []string{ "darwin-amd64", "darwin-arm64", "linux-amd64", "linux-arm64", } - baseURL = "https://github.com/pomerium/envoy-binaries/releases/download/v" + envoyVersion + baseURL = "https://github.com/pomerium/envoy-custom/releases/download/v" + envoyVersion ) func main() { diff --git a/pkg/grpc/client.go b/pkg/grpc/client.go index 0d145769f..4a965b603 100644 --- a/pkg/grpc/client.go +++ b/pkg/grpc/client.go @@ -10,7 +10,6 @@ import ( "google.golang.org/grpc/credentials/insecure" "github.com/pomerium/pomerium/internal/log" - "github.com/pomerium/pomerium/internal/telemetry" "github.com/pomerium/pomerium/pkg/grpcutil" "github.com/pomerium/pomerium/pkg/telemetry/requestid" ) @@ -32,11 +31,8 @@ type Options struct { // NewGRPCClientConn returns a new gRPC pomerium service client connection. func NewGRPCClientConn(ctx context.Context, opts *Options, other ...grpc.DialOption) (*grpc.ClientConn, error) { - clientStatsHandler := telemetry.NewGRPCClientStatsHandler(opts.ServiceName) - unaryClientInterceptors := []grpc.UnaryClientInterceptor{ requestid.UnaryClientInterceptor(), - clientStatsHandler.UnaryInterceptor, } streamClientInterceptors := []grpc.StreamClientInterceptor{ requestid.StreamClientInterceptor(), @@ -49,7 +45,6 @@ func NewGRPCClientConn(ctx context.Context, opts *Options, other ...grpc.DialOpt dialOptions := []grpc.DialOption{ grpc.WithChainUnaryInterceptor(unaryClientInterceptors...), grpc.WithChainStreamInterceptor(streamClientInterceptors...), - grpc.WithStatsHandler(clientStatsHandler.Handler), grpc.WithDisableServiceConfig(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), @@ -75,13 +70,13 @@ type OutboundOptions struct { } // newOutboundGRPCClientConn gets a new outbound gRPC client. -func newOutboundGRPCClientConn(ctx context.Context, opts *OutboundOptions) (*grpc.ClientConn, error) { +func newOutboundGRPCClientConn(ctx context.Context, opts *OutboundOptions, other ...grpc.DialOption) (*grpc.ClientConn, error) { return NewGRPCClientConn(ctx, &Options{ Address: net.JoinHostPort("127.0.0.1", opts.OutboundPort), InstallationID: opts.InstallationID, ServiceName: opts.ServiceName, SignedJWTKey: opts.SignedJWTKey, - }) + }, other...) } // CachedOutboundGRPClientConn keeps a cached outbound gRPC client connection open based on options. @@ -92,7 +87,7 @@ type CachedOutboundGRPClientConn struct { } // Get gets the cached outbound gRPC client, or creates a new one if the options have changed. -func (cache *CachedOutboundGRPClientConn) Get(ctx context.Context, opts *OutboundOptions) (*grpc.ClientConn, error) { +func (cache *CachedOutboundGRPClientConn) Get(ctx context.Context, opts *OutboundOptions, other ...grpc.DialOption) (*grpc.ClientConn, error) { cache.mu.Lock() defer cache.mu.Unlock() @@ -106,7 +101,7 @@ func (cache *CachedOutboundGRPClientConn) Get(ctx context.Context, opts *Outboun } var err error - cache.current, err = newOutboundGRPCClientConn(ctx, opts) + cache.current, err = newOutboundGRPCClientConn(ctx, opts, other...) if err != nil { return nil, err } diff --git a/pkg/grpc/databroker/sync_test.go b/pkg/grpc/databroker/sync_test.go index f047b3058..2fbc67b5c 100644 --- a/pkg/grpc/databroker/sync_test.go +++ b/pkg/grpc/databroker/sync_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace" grpc "google.golang.org/grpc" "github.com/pomerium/pomerium/internal/databroker" @@ -24,7 +25,7 @@ func Test_SyncLatestRecords(t *testing.T) { defer clearTimeout() cc := testutil.NewGRPCServer(t, func(s *grpc.Server) { - databrokerpb.RegisterDataBrokerServiceServer(s, databroker.New(ctx)) + databrokerpb.RegisterDataBrokerServiceServer(s, databroker.New(ctx, trace.NewNoopTracerProvider())) }) c := databrokerpb.NewDataBrokerServiceClient(cc) diff --git a/pkg/identity/providers.go b/pkg/identity/providers.go index 34b569cc1..9ddad4ceb 100644 --- a/pkg/identity/providers.go +++ b/pkg/identity/providers.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "golang.org/x/oauth2" "github.com/pomerium/pomerium/pkg/identity/identity" @@ -22,6 +23,7 @@ import ( "github.com/pomerium/pomerium/pkg/identity/oidc/okta" "github.com/pomerium/pomerium/pkg/identity/oidc/onelogin" "github.com/pomerium/pomerium/pkg/identity/oidc/ping" + oteltrace "go.opentelemetry.io/otel/trace" ) // State is the identity state. @@ -64,13 +66,20 @@ func init() { } // NewAuthenticator returns a new identity provider based on its name. -func NewAuthenticator(o oauth.Options) (a Authenticator, err error) { - ctx := context.Background() - +func NewAuthenticator(ctx context.Context, tracerProvider oteltrace.TracerProvider, o oauth.Options) (a Authenticator, err error) { if o.ProviderName == "" { return nil, fmt.Errorf("identity: provider is not defined") } + ctx = context.WithValue(ctx, oauth2.HTTPClient, &http.Client{ + Transport: otelhttp.NewTransport(nil, + otelhttp.WithTracerProvider(tracerProvider), + otelhttp.WithSpanNameFormatter(func(_ string, r *http.Request) string { + return fmt.Sprintf("OAuth2 Client: %s %s", r.Method, r.URL.Path) + }), + ), + }) + ctor, ok := registry[o.ProviderName] if !ok { return nil, fmt.Errorf("identity: unknown provider: %s", o.ProviderName) diff --git a/proxy/data_test.go b/proxy/data_test.go index 2e74036dc..b15a02539 100644 --- a/proxy/data_test.go +++ b/proxy/data_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" @@ -32,7 +33,7 @@ func Test_getUserInfoData(t *testing.T) { defer clearTimeout() cc := testutil.NewGRPCServer(t, func(srv *grpc.Server) { - databrokerpb.RegisterDataBrokerServiceServer(srv, databroker.New(ctx)) + databrokerpb.RegisterDataBrokerServiceServer(srv, databroker.New(ctx, trace.NewNoopTracerProvider())) }) t.Cleanup(func() { cc.Close() }) diff --git a/proxy/handlers.go b/proxy/handlers.go index 42fa01953..23efb3ecd 100644 --- a/proxy/handlers.go +++ b/proxy/handlers.go @@ -9,11 +9,13 @@ import ( "github.com/go-jose/go-jose/v3/jwt" "github.com/gorilla/mux" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "github.com/pomerium/pomerium/config" "github.com/pomerium/pomerium/internal/handlers" "github.com/pomerium/pomerium/internal/httputil" "github.com/pomerium/pomerium/internal/middleware" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/urlutil" ) @@ -21,6 +23,7 @@ import ( func (p *Proxy) registerDashboardHandlers(r *mux.Router, opts *config.Options) *mux.Router { h := httputil.DashboardSubrouter(r) h.Use(middleware.SetHeaders(httputil.HeadersContentSecurityPolicy)) + h.Use(trace.NewHTTPMiddleware(otelhttp.WithTracerProvider(p.tracerProvider))) // special pomerium endpoints for users to view their session h.Path("/").Handler(httputil.HandlerFunc(p.userInfo)).Methods(http.MethodGet) diff --git a/proxy/handlers_test.go b/proxy/handlers_test.go index 2817b22d0..b86ce1cb0 100644 --- a/proxy/handlers_test.go +++ b/proxy/handlers_test.go @@ -225,9 +225,8 @@ func TestProxy_jsonUserInfo(t *testing.T) { // The /.pomerium/jwt endpoint should be registered only if explicitly enabled. func TestProxy_registerDashboardHandlers_jwtEndpoint(t *testing.T) { - proxy := &Proxy{ - state: atomicutil.NewValue(&proxyState{}), - } + proxy, err := New(context.Background(), &config.Config{Options: config.NewDefaultOptions()}) + require.NoError(t, err) req := httptest.NewRequest(http.MethodGet, "/.pomerium/jwt", nil) rawJWT := "eyJ0eXAiOiJKV1QiLCJhbGciOiJub25lIn0.eyJzdWIiOiIxMjM0NTY3ODkwIn0." req.Header.Set("X-Pomerium-Jwt-Assertion", rawJWT) diff --git a/proxy/proxy.go b/proxy/proxy.go index 58f795860..2f83cdd6b 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -10,6 +10,7 @@ import ( "net/http" "github.com/gorilla/mux" + oteltrace "go.opentelemetry.io/otel/trace" "github.com/pomerium/pomerium/config" "github.com/pomerium/pomerium/internal/atomicutil" @@ -17,6 +18,7 @@ import ( "github.com/pomerium/pomerium/internal/httputil" "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/telemetry/metrics" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/pkg/cryptutil" ) @@ -56,17 +58,20 @@ type Proxy struct { currentOptions *atomicutil.Value[*config.Options] currentRouter *atomicutil.Value[*mux.Router] webauthn *webauthn.Handler + tracerProvider oteltrace.TracerProvider } // New takes a Proxy service from options and a validation function. // Function returns an error if options fail to validate. func New(ctx context.Context, cfg *config.Config) (*Proxy, error) { - state, err := newProxyStateFromConfig(ctx, cfg) + tracerProvider := trace.NewTracerProvider(ctx, "Proxy") + state, err := newProxyStateFromConfig(ctx, tracerProvider, cfg) if err != nil { return nil, err } p := &Proxy{ + tracerProvider: tracerProvider, state: atomicutil.NewValue(state), currentOptions: config.NewAtomicOptions(), currentRouter: atomicutil.NewValue(httputil.NewRouter()), @@ -96,7 +101,7 @@ func (p *Proxy) OnConfigChange(ctx context.Context, cfg *config.Config) { if err := p.setHandlers(ctx, cfg.Options); err != nil { log.Ctx(ctx).Error().Err(err).Msg("proxy: failed to update proxy handlers from configuration settings") } - if state, err := newProxyStateFromConfig(ctx, cfg); err != nil { + if state, err := newProxyStateFromConfig(ctx, p.tracerProvider, cfg); err != nil { log.Ctx(ctx).Error().Err(err).Msg("proxy: failed to update proxy state from configuration settings") } else { p.state.Store(state) diff --git a/proxy/state.go b/proxy/state.go index 5a7727e13..c07e1c51b 100644 --- a/proxy/state.go +++ b/proxy/state.go @@ -9,6 +9,9 @@ import ( "github.com/pomerium/pomerium/internal/authenticateflow" "github.com/pomerium/pomerium/pkg/grpc" "github.com/pomerium/pomerium/pkg/grpc/databroker" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + oteltrace "go.opentelemetry.io/otel/trace" + googlegrpc "google.golang.org/grpc" ) var outboundGRPCConnection = new(grpc.CachedOutboundGRPClientConn) @@ -31,7 +34,7 @@ type proxyState struct { authenticateFlow authenticateFlow } -func newProxyStateFromConfig(ctx context.Context, cfg *config.Config) (*proxyState, error) { +func newProxyStateFromConfig(ctx context.Context, tracerProvider oteltrace.TracerProvider, cfg *config.Config) (*proxyState, error) { err := ValidateOptions(cfg.Options) if err != nil { return nil, err @@ -62,7 +65,7 @@ func newProxyStateFromConfig(ctx context.Context, cfg *config.Config) (*proxySta InstallationID: cfg.Options.InstallationID, ServiceName: cfg.Options.Services, SignedJWTKey: state.sharedKey, - }) + }, googlegrpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(tracerProvider)))) if err != nil { return nil, err } @@ -71,10 +74,10 @@ func newProxyStateFromConfig(ctx context.Context, cfg *config.Config) (*proxySta state.programmaticRedirectDomainWhitelist = cfg.Options.ProgrammaticRedirectDomainWhitelist if cfg.Options.UseStatelessAuthenticateFlow() { - state.authenticateFlow, err = authenticateflow.NewStateless(ctx, + state.authenticateFlow, err = authenticateflow.NewStateless(ctx, tracerProvider, cfg, state.sessionStore, nil, nil, nil) } else { - state.authenticateFlow, err = authenticateflow.NewStateful(ctx, cfg, state.sessionStore) + state.authenticateFlow, err = authenticateflow.NewStateful(ctx, tracerProvider, cfg, state.sessionStore) } if err != nil { return nil, err