wip: new tracing system

authenticate/authenticate.go

@@ -10,7 +10,9 @@ import (
 	"github.com/pomerium/pomerium/config"
 	"github.com/pomerium/pomerium/internal/atomicutil"
 	"github.com/pomerium/pomerium/internal/log"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/pkg/cryptutil"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
 // ValidateOptions checks that configuration are complete and valid.
@@ -38,23 +40,31 @@ func ValidateOptions(o *config.Options) error {
 
 // Authenticate contains data required to run the authenticate service.
 type Authenticate struct {
-	cfg     *authenticateConfig
+	cfg            *authenticateConfig
-	options *atomicutil.Value[*config.Options]
+	options        *atomicutil.Value[*config.Options]
-	state   *atomicutil.Value[*authenticateState]
+	state          *atomicutil.Value[*authenticateState]
+	tracerProvider oteltrace.TracerProvider
+	tracer         oteltrace.Tracer
 }
 
 // New validates and creates a new authenticate service from a set of Options.
 func New(ctx context.Context, cfg *config.Config, options ...Option) (*Authenticate, error) {
 	authenticateConfig := getAuthenticateConfig(options...)
+
+	tracerProvider := trace.NewTracerProvider(ctx, "Authenticate")
+	tracer := tracerProvider.Tracer(trace.PomeriumCoreTracer)
+
 	a := &Authenticate{
-		cfg:     authenticateConfig,
+		cfg:            authenticateConfig,
-		options: config.NewAtomicOptions(),
+		options:        config.NewAtomicOptions(),
-		state:   atomicutil.NewValue(newAuthenticateState()),
+		state:          atomicutil.NewValue(newAuthenticateState()),
+		tracerProvider: tracerProvider,
+		tracer:         tracer,
 	}
 
 	a.options.Store(cfg.Options)
 
-	state, err := newAuthenticateStateFromConfig(ctx, cfg, authenticateConfig)
+	state, err := newAuthenticateStateFromConfig(ctx, tracerProvider, cfg, authenticateConfig)
 	if err != nil {
 		return nil, err
 	}
@@ -70,7 +80,7 @@ func (a *Authenticate) OnConfigChange(ctx context.Context, cfg *config.Config) {
 	}
 
 	a.options.Store(cfg.Options)
-	if state, err := newAuthenticateStateFromConfig(ctx, cfg, a.cfg); err != nil {
+	if state, err := newAuthenticateStateFromConfig(ctx, a.tracerProvider, cfg, a.cfg); err != nil {
 		log.Ctx(ctx).Error().Err(err).Msg("authenticate: failed to update state")
 	} else {
 		a.state.Store(state)

authenticate/config.go

@@ -1,14 +1,17 @@
 package authenticate
 
 import (
+	"context"
+
 	"github.com/pomerium/pomerium/authenticate/events"
 	"github.com/pomerium/pomerium/config"
 	identitypb "github.com/pomerium/pomerium/pkg/grpc/identity"
 	"github.com/pomerium/pomerium/pkg/identity"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
 type authenticateConfig struct {
-	getIdentityProvider func(options *config.Options, idpID string) (identity.Authenticator, error)
+	getIdentityProvider func(ctx context.Context, tracerProvider oteltrace.TracerProvider, options *config.Options, idpID string) (identity.Authenticator, error)
 	profileTrimFn       func(*identitypb.Profile)
 	authEventFn         events.AuthEventFn
 }
@@ -26,7 +29,7 @@ func getAuthenticateConfig(options ...Option) *authenticateConfig {
 }
 
 // WithGetIdentityProvider sets the getIdentityProvider function in the config.
-func WithGetIdentityProvider(getIdentityProvider func(options *config.Options, idpID string) (identity.Authenticator, error)) Option {
+func WithGetIdentityProvider(getIdentityProvider func(ctx context.Context, tracerProvider oteltrace.TracerProvider, options *config.Options, idpID string) (identity.Authenticator, error)) Option {
 	return func(cfg *authenticateConfig) {
 		cfg.getIdentityProvider = getIdentityProvider
 	}

authenticate/handlers.go

@@ -21,7 +21,6 @@ import (
 	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/internal/middleware"
 	"github.com/pomerium/pomerium/internal/sessions"
-	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/urlutil"
 	"github.com/pomerium/pomerium/pkg/cryptutil"
 	"github.com/pomerium/pomerium/pkg/identity"
@@ -114,7 +113,7 @@ func (a *Authenticate) RetrieveSession(next http.Handler) http.Handler {
 // session state is attached to the users's request context.
 func (a *Authenticate) VerifySession(next http.Handler) http.Handler {
 	return httputil.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
-		ctx, span := trace.StartSpan(r.Context(), "authenticate.VerifySession")
+		ctx, span := a.tracer.Start(r.Context(), "authenticate.VerifySession")
 		defer span.End()
 
 		state := a.state.Load()
@@ -160,7 +159,7 @@ func (a *Authenticate) RobotsTxt(w http.ResponseWriter, _ *http.Request) {
 
 // SignIn handles authenticating a user.
 func (a *Authenticate) SignIn(w http.ResponseWriter, r *http.Request) error {
-	ctx, span := trace.StartSpan(r.Context(), "authenticate.SignIn")
+	ctx, span := a.tracer.Start(r.Context(), "authenticate.SignIn")
 	defer span.End()
 
 	state := a.state.Load()
@@ -197,13 +196,13 @@ func (a *Authenticate) SignOut(w http.ResponseWriter, r *http.Request) error {
 }
 
 func (a *Authenticate) signOutRedirect(w http.ResponseWriter, r *http.Request) error {
-	ctx, span := trace.StartSpan(r.Context(), "authenticate.SignOut")
+	ctx, span := a.tracer.Start(r.Context(), "authenticate.SignOut")
 	defer span.End()
 
 	options := a.options.Load()
 	idpID := a.getIdentityProviderIDForRequest(r)
 
-	authenticator, err := a.cfg.getIdentityProvider(options, idpID)
+	authenticator, err := a.cfg.getIdentityProvider(ctx, a.tracerProvider, options, idpID)
 	if err != nil {
 		return err
 	}
@@ -274,7 +273,7 @@ func (a *Authenticate) reauthenticateOrFail(w http.ResponseWriter, r *http.Reque
 	options := a.options.Load()
 	idpID := a.getIdentityProviderIDForRequest(r)
 
-	authenticator, err := a.cfg.getIdentityProvider(options, idpID)
+	authenticator, err := a.cfg.getIdentityProvider(r.Context(), a.tracerProvider, options, idpID)
 	if err != nil {
 		return err
 	}
@@ -307,6 +306,10 @@ func (a *Authenticate) OAuthCallback(w http.ResponseWriter, r *http.Request) err
 	if err != nil {
 		return fmt.Errorf("authenticate.OAuthCallback: %w", err)
 	}
+	q := redirect.Query()
+	if traceparent := q.Get(urlutil.QueryTraceparent); traceparent != "" {
+		w.Header().Set("X-Pomerium-Traceparent", traceparent)
+	}
 	httputil.Redirect(w, r, redirect.String(), http.StatusFound)
 	return nil
 }
@@ -321,7 +324,7 @@ func (a *Authenticate) statusForErrorCode(errorCode string) int {
 }
 
 func (a *Authenticate) getOAuthCallback(w http.ResponseWriter, r *http.Request) (*url.URL, error) {
-	ctx, span := trace.StartSpan(r.Context(), "authenticate.getOAuthCallback")
+	ctx, span := a.tracer.Start(r.Context(), "authenticate.getOAuthCallback")
 	defer span.End()
 
 	state := a.state.Load()
@@ -380,7 +383,7 @@ Or contact your administrator.
 
 	idpID := state.flow.GetIdentityProviderIDForURLValues(redirectURL.Query())
 
-	authenticator, err := a.cfg.getIdentityProvider(options, idpID)
+	authenticator, err := a.cfg.getIdentityProvider(ctx, a.tracerProvider, options, idpID)
 	if err != nil {
 		return nil, err
 	}
@@ -432,7 +435,7 @@ func (a *Authenticate) getSessionFromCtx(ctx context.Context) (*sessions.State,
 }
 
 func (a *Authenticate) userInfo(w http.ResponseWriter, r *http.Request) error {
-	ctx, span := trace.StartSpan(r.Context(), "authenticate.userInfo")
+	ctx, span := a.tracer.Start(r.Context(), "authenticate.userInfo")
 	defer span.End()
 
 	options := a.options.Load()
@@ -484,7 +487,7 @@ func (a *Authenticate) revokeSession(ctx context.Context, w http.ResponseWriter,
 
 	idpID := r.FormValue(urlutil.QueryIdentityProviderID)
 
-	authenticator, err := a.cfg.getIdentityProvider(options, idpID)
+	authenticator, err := a.cfg.getIdentityProvider(ctx, a.tracerProvider, options, idpID)
 	if err != nil {
 		return ""
 	}

authenticate/handlers_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	oteltrace "go.opentelemetry.io/otel/trace"
 	"go.uber.org/mock/gomock"
 	"golang.org/x/crypto/chacha20poly1305"
 	"golang.org/x/oauth2"
@@ -225,7 +226,7 @@ func TestAuthenticate_SignOut(t *testing.T) {
 			ctrl := gomock.NewController(t)
 			defer ctrl.Finish()
 			a := &Authenticate{
-				cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ *config.Options, _ string) (identity.Authenticator, error) {
+				cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ context.Context, _ oteltrace.TracerProvider, _ *config.Options, _ string) (identity.Authenticator, error) {
 					return tt.provider, nil
 				})),
 				state: atomicutil.NewValue(&authenticateState{
@@ -280,7 +281,7 @@ func TestAuthenticate_SignOutDoesNotRequireSession(t *testing.T) {
 
 	sessionStore := &mstore.Store{LoadError: errors.New("no session")}
 	a := &Authenticate{
-		cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ *config.Options, _ string) (identity.Authenticator, error) {
+		cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ context.Context, _ oteltrace.TracerProvider, _ *config.Options, _ string) (identity.Authenticator, error) {
 			return identity.MockProvider{}, nil
 		})),
 		state: atomicutil.NewValue(&authenticateState{
@@ -355,7 +356,7 @@ func TestAuthenticate_OAuthCallback(t *testing.T) {
 			}
 			authURL, _ := url.Parse(tt.authenticateURL)
 			a := &Authenticate{
-				cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ *config.Options, _ string) (identity.Authenticator, error) {
+				cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ context.Context, _ oteltrace.TracerProvider, _ *config.Options, _ string) (identity.Authenticator, error) {
 					return tt.provider, nil
 				})),
 				state: atomicutil.NewValue(&authenticateState{
@@ -467,7 +468,7 @@ func TestAuthenticate_SessionValidatorMiddleware(t *testing.T) {
 				t.Fatal(err)
 			}
 			a := &Authenticate{
-				cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ *config.Options, _ string) (identity.Authenticator, error) {
+				cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ context.Context, _ oteltrace.TracerProvider, _ *config.Options, _ string) (identity.Authenticator, error) {
 					return tt.provider, nil
 				})),
 				state: atomicutil.NewValue(&authenticateState{

authenticate/identity.go

@@ -1,13 +1,16 @@
 package authenticate
 
 import (
+	"context"
+
 	"github.com/pomerium/pomerium/config"
 	"github.com/pomerium/pomerium/internal/urlutil"
 	"github.com/pomerium/pomerium/pkg/identity"
 	"github.com/pomerium/pomerium/pkg/identity/oauth"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
-func defaultGetIdentityProvider(options *config.Options, idpID string) (identity.Authenticator, error) {
+func defaultGetIdentityProvider(ctx context.Context, tracerProvider oteltrace.TracerProvider, options *config.Options, idpID string) (identity.Authenticator, error) {
 	authenticateURL, err := options.GetAuthenticateURL()
 	if err != nil {
 		return nil, err
@@ -23,7 +26,7 @@ func defaultGetIdentityProvider(options *config.Options, idpID string) (identity
 	if err != nil {
 		return nil, err
 	}
-	return identity.NewAuthenticator(oauth.Options{
+	return identity.NewAuthenticator(ctx, tracerProvider, oauth.Options{
 		RedirectURL:     redirectURL,
 		ProviderName:    idp.GetType(),
 		ProviderURL:     idp.GetUrl(),

authenticate/state.go

@@ -8,6 +8,7 @@ import (
 	"net/url"
 
 	"github.com/go-jose/go-jose/v3"
+	oteltrace "go.opentelemetry.io/otel/trace"
 	"golang.org/x/oauth2"
 
 	"github.com/pomerium/pomerium/config"
@@ -65,7 +66,9 @@ func newAuthenticateState() *authenticateState {
 
 func newAuthenticateStateFromConfig(
 	ctx context.Context,
-	cfg *config.Config, authenticateConfig *authenticateConfig,
+	tracerProvider oteltrace.TracerProvider,
+	cfg *config.Config,
+	authenticateConfig *authenticateConfig,
 ) (*authenticateState, error) {
 	err := ValidateOptions(cfg.Options)
 	if err != nil {
@@ -147,6 +150,7 @@ func newAuthenticateStateFromConfig(
 
 	if cfg.Options.UseStatelessAuthenticateFlow() {
 		state.flow, err = authenticateflow.NewStateless(ctx,
+			tracerProvider,
 			cfg,
 			cookieStore,
 			authenticateConfig.getIdentityProvider,
@@ -154,7 +158,7 @@ func newAuthenticateStateFromConfig(
 			authenticateConfig.authEventFn,
 		)
 	} else {
-		state.flow, err = authenticateflow.NewStateful(ctx, cfg, cookieStore)
+		state.flow, err = authenticateflow.NewStateful(ctx, tracerProvider, cfg, cookieStore)
 	}
 	if err != nil {
 		return nil, err

authorize/authorize.go

@@ -23,6 +23,7 @@ import (
 	"github.com/pomerium/pomerium/pkg/grpc"
 	"github.com/pomerium/pomerium/pkg/grpc/databroker"
 	"github.com/pomerium/pomerium/pkg/storage"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
 // Authorize struct holds
@@ -37,18 +38,25 @@ type Authorize struct {
 	// This should provide a consistent view of the data at a given server/record version and
 	// avoid partial updates.
 	stateLock sync.RWMutex
+
+	tracerProvider oteltrace.TracerProvider
+	tracer         oteltrace.Tracer
 }
 
 // New validates and creates a new Authorize service from a set of config options.
 func New(ctx context.Context, cfg *config.Config) (*Authorize, error) {
+	tracerProvider := trace.NewTracerProvider(ctx, "Authorize")
+	tracer := tracerProvider.Tracer(trace.PomeriumCoreTracer)
 	a := &Authorize{
 		currentOptions: config.NewAtomicOptions(),
 		store:          store.New(),
 		globalCache:    storage.NewGlobalCache(time.Minute),
+		tracerProvider: tracerProvider,
+		tracer:         tracer,
 	}
 	a.accessTracker = NewAccessTracker(a, accessTrackerMaxSize, accessTrackerDebouncePeriod)
 
-	state, err := newAuthorizeStateFromConfig(ctx, cfg, a.store, nil)
+	state, err := newAuthorizeStateFromConfig(ctx, tracerProvider, cfg, a.store, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -98,7 +106,7 @@ func newPolicyEvaluator(
 	ctx = log.WithContext(ctx, func(c zerolog.Context) zerolog.Context {
 		return c.Str("service", "authorize")
 	})
-	ctx, span := trace.StartSpan(ctx, "authorize.newPolicyEvaluator")
+	ctx, span := trace.Continue(ctx, "authorize.newPolicyEvaluator")
 	defer span.End()
 
 	clientCA, err := opts.DownstreamMTLS.GetCA()
@@ -151,7 +159,7 @@ func newPolicyEvaluator(
 func (a *Authorize) OnConfigChange(ctx context.Context, cfg *config.Config) {
 	currentState := a.state.Load()
 	a.currentOptions.Store(cfg.Options)
-	if state, err := newAuthorizeStateFromConfig(ctx, cfg, a.store, currentState.evaluator); err != nil {
+	if state, err := newAuthorizeStateFromConfig(ctx, a.tracerProvider, cfg, a.store, currentState.evaluator); err != nil {
 		log.Ctx(ctx).Error().Err(err).Msg("authorize: error updating state")
 	} else {
 		a.state.Store(state)

authorize/check_response.go

@@ -73,7 +73,8 @@ func (a *Authorize) handleResultAllowed(
 	_ *envoy_service_auth_v3.CheckRequest,
 	result *evaluator.Result,
 ) (*envoy_service_auth_v3.CheckResponse, error) {
-	return a.okResponse(result.Headers), nil
+	resp := a.okResponse(result.Headers)
+	return resp, nil
 }
 
 func (a *Authorize) handleResultDenied(
@@ -253,16 +254,24 @@ func (a *Authorize) requireLoginResponse(
 	// always assume https scheme
 	checkRequestURL := getCheckRequestURL(in)
 	checkRequestURL.Scheme = "https"
+	var signInUrlQuery url.Values
 
+	headers := map[string]string{}
+	if id := in.GetAttributes().GetRequest().GetHttp().GetHeaders()["traceparent"]; id != "" {
+		headers["X-Pomerium-Traceparent"] = id
+		headers["X-Pomerium-Tracestate"] = "pomerium.traceparent=" + id // TODO: this might not be necessary anymore
+		signInUrlQuery = url.Values{}
+		signInUrlQuery.Add("pomerium_traceparent", id)
+		signInUrlQuery.Add("pomerium_tracestate", "pomerium.traceparent="+id)
+	}
 	redirectTo, err := state.authenticateFlow.AuthenticateSignInURL(
-		ctx, nil, &checkRequestURL, idp.GetId())
+		ctx, signInUrlQuery, &checkRequestURL, idp.GetId())
 	if err != nil {
 		return nil, err
 	}
+	headers["Location"] = redirectTo
 
-	return a.deniedResponse(ctx, in, http.StatusFound, "Login", map[string]string{
+	return a.deniedResponse(ctx, in, http.StatusFound, "Login", headers)
-		"Location": redirectTo,
-	})
 }
 
 func (a *Authorize) requireWebAuthnResponse(

authorize/databroker.go

@@ -3,7 +3,6 @@ package authorize
 import (
 	"context"
 
-	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/pkg/grpc/databroker"
 	"github.com/pomerium/pomerium/pkg/grpc/session"
 	"github.com/pomerium/pomerium/pkg/grpc/user"
@@ -62,7 +61,7 @@ func (a *Authorize) getDataBrokerSessionOrServiceAccount(
 	sessionID string,
 	dataBrokerRecordVersion uint64,
 ) (s sessionOrServiceAccount, err error) {
-	ctx, span := trace.StartSpan(ctx, "authorize.getDataBrokerSessionOrServiceAccount")
+	ctx, span := a.tracer.Start(ctx, "authorize.getDataBrokerSessionOrServiceAccount")
 	defer span.End()
 
 	record, err := getDataBrokerRecord(ctx, grpcutil.GetTypeURL(new(session.Session)), sessionID, dataBrokerRecordVersion)
@@ -95,7 +94,7 @@ func (a *Authorize) getDataBrokerUser(
 	ctx context.Context,
 	userID string,
 ) (*user.User, error) {
-	ctx, span := trace.StartSpan(ctx, "authorize.getDataBrokerUser")
+	ctx, span := a.tracer.Start(ctx, "authorize.getDataBrokerUser")
 	defer span.End()
 
 	record, err := getDataBrokerRecord(ctx, grpcutil.GetTypeURL(new(user.User)), userID, 0)

authorize/evaluator/evaluator.go

@@ -199,7 +199,7 @@ func getOrCreatePolicyEvaluators(
 
 // Evaluate evaluates the rego for the given policy and generates the identity headers.
 func (e *Evaluator) Evaluate(ctx context.Context, req *Request) (*Result, error) {
-	ctx, span := trace.StartSpan(ctx, "authorize.Evaluator.Evaluate")
+	ctx, span := trace.Continue(ctx, "authorize.Evaluator.Evaluate")
 	defer span.End()
 
 	eg, ctx := errgroup.WithContext(ctx)

authorize/evaluator/headers_evaluator.go

@@ -80,7 +80,7 @@ func NewHeadersEvaluator(store *store.Store) *HeadersEvaluator {
 
 // Evaluate evaluates the headers.rego script.
 func (e *HeadersEvaluator) Evaluate(ctx context.Context, req *HeadersRequest, options ...rego.EvalOption) (*HeadersResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "authorize.HeadersEvaluator.Evaluate")
+	ctx, span := trace.Continue(ctx, "authorize.HeadersEvaluator.Evaluate")
 	defer span.End()
 
 	ectx := new(rego.EvalContext)

authorize/evaluator/policy_evaluator.go

@@ -6,7 +6,7 @@ import (
 	"strings"
 
 	"github.com/open-policy-agent/opa/rego"
-	octrace "go.opencensus.io/trace"
+	"go.opentelemetry.io/otel/attribute"
 
 	"github.com/pomerium/pomerium/authorize/internal/store"
 	"github.com/pomerium/pomerium/config"
@@ -209,9 +209,9 @@ func (e *PolicyEvaluator) Evaluate(ctx context.Context, req *PolicyRequest) (*Po
 }
 
 func (e *PolicyEvaluator) evaluateQuery(ctx context.Context, req *PolicyRequest, query policyQuery) (*PolicyResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "authorize.PolicyEvaluator.evaluateQuery")
+	ctx, span := trace.Continue(ctx, "authorize.PolicyEvaluator.evaluateQuery")
 	defer span.End()
-	span.AddAttributes(octrace.StringAttribute("script_checksum", query.checksum()))
+	span.SetAttributes(attribute.String("script_checksum", query.checksum()))
 
 	rs, err := safeEval(ctx, query.PreparedEvalQuery,
 		rego.EvalInput(req),

authorize/grpc.go

@@ -19,7 +19,6 @@ import (
 	"github.com/pomerium/pomerium/internal/httputil"
 	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/internal/sessions"
-	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/urlutil"
 	"github.com/pomerium/pomerium/pkg/contextutil"
 	"github.com/pomerium/pomerium/pkg/grpc/user"
@@ -29,7 +28,7 @@ import (
 
 // Check implements the envoy auth server gRPC endpoint.
 func (a *Authorize) Check(ctx context.Context, in *envoy_service_auth_v3.CheckRequest) (*envoy_service_auth_v3.CheckResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "authorize.grpc.Check")
+	ctx, span := a.tracer.Start(ctx, "authorize.grpc.Check")
 	defer span.End()
 
 	querier := storage.NewTracingQuerier(

authorize/internal/store/store.go

@@ -14,7 +14,7 @@ import (
 	opastorage "github.com/open-policy-agent/opa/storage"
 	"github.com/open-policy-agent/opa/storage/inmem"
 	"github.com/open-policy-agent/opa/types"
-	octrace "go.opencensus.io/trace"
+	"go.opentelemetry.io/otel/attribute"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
@@ -130,20 +130,20 @@ func (s *Store) GetDataBrokerRecordOption() func(*rego.Rego) {
 			types.NewObject(nil, types.NewDynamicProperty(types.S, types.S)),
 		),
 	}, func(bctx rego.BuiltinContext, op1 *ast.Term, op2 *ast.Term) (*ast.Term, error) {
-		ctx, span := trace.StartSpan(bctx.Context, "rego.get_databroker_record")
+		ctx, span := trace.Continue(bctx.Context, "rego.get_databroker_record")
 		defer span.End()
 
 		recordType, ok := op1.Value.(ast.String)
 		if !ok {
 			return nil, fmt.Errorf("invalid record type: %T", op1)
 		}
-		span.AddAttributes(octrace.StringAttribute("record_type", recordType.String()))
+		span.SetAttributes(attribute.String("record_type", recordType.String()))
 
 		recordIDOrIndex, ok := op2.Value.(ast.String)
 		if !ok {
 			return nil, fmt.Errorf("invalid record id: %T", op2)
 		}
-		span.AddAttributes(octrace.StringAttribute("record_id", recordIDOrIndex.String()))
+		span.SetAttributes(attribute.String("record_id", recordIDOrIndex.String()))
 
 		msg := s.GetDataBrokerRecord(ctx, string(recordType), string(recordIDOrIndex))
 		if msg == nil {

authorize/log.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/pomerium/pomerium/authorize/evaluator"
 	"github.com/pomerium/pomerium/internal/log"
-	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/pkg/grpc/audit"
 	"github.com/pomerium/pomerium/pkg/grpc/databroker"
 	"github.com/pomerium/pomerium/pkg/grpc/session"
@@ -25,7 +24,7 @@ func (a *Authorize) logAuthorizeCheck(
 	in *envoy_service_auth_v3.CheckRequest, out *envoy_service_auth_v3.CheckResponse,
 	res *evaluator.Result, s sessionOrServiceAccount, u *user.User,
 ) {
-	ctx, span := trace.StartSpan(ctx, "authorize.grpc.LogAuthorizeCheck")
+	ctx, span := a.tracer.Start(ctx, "authorize.grpc.LogAuthorizeCheck")
 	defer span.End()
 
 	hdrs := getCheckRequestHeaders(in)
@@ -57,7 +56,7 @@ func (a *Authorize) logAuthorizeCheck(
 	evt.Msg("authorize check")
 
 	if enc := a.state.Load().auditEncryptor; enc != nil {
-		ctx, span := trace.StartSpan(ctx, "authorize.grpc.AuditAuthorizeCheck")
+		ctx, span := a.tracer.Start(ctx, "authorize.grpc.AuditAuthorizeCheck")
 		defer span.End()
 
 		record := &audit.Record{

authorize/state.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/url"
 
+	oteltrace "go.opentelemetry.io/otel/trace"
 	googlegrpc "google.golang.org/grpc"
 
 	"github.com/pomerium/pomerium/authorize/evaluator"
@@ -34,7 +35,10 @@ type authorizeState struct {
 
 func newAuthorizeStateFromConfig(
 	ctx context.Context,
-	cfg *config.Config, store *store.Store, previousPolicyEvaluator *evaluator.Evaluator,
+	tracerProvider oteltrace.TracerProvider,
+	cfg *config.Config,
+	store *store.Store,
+	previousPolicyEvaluator *evaluator.Evaluator,
 ) (*authorizeState, error) {
 	if err := validateOptions(cfg.Options); err != nil {
 		return nil, fmt.Errorf("authorize: bad options: %w", err)
@@ -59,7 +63,7 @@ func newAuthorizeStateFromConfig(
 		return nil, err
 	}
 
-	cc, err := outboundGRPCConnection.Get(ctx, &grpc.OutboundOptions{
+	cc, err := outboundGRPCConnection.Get(ctx, tracerProvider, &grpc.OutboundOptions{
 		OutboundPort:   cfg.OutboundPort,
 		InstallationID: cfg.Options.InstallationID,
 		ServiceName:    cfg.Options.Services,
@@ -85,9 +89,9 @@ func newAuthorizeStateFromConfig(
 	}
 
 	if cfg.Options.UseStatelessAuthenticateFlow() {
-		state.authenticateFlow, err = authenticateflow.NewStateless(ctx, cfg, nil, nil, nil, nil)
+		state.authenticateFlow, err = authenticateflow.NewStateless(ctx, tracerProvider, cfg, nil, nil, nil, nil)
 	} else {
-		state.authenticateFlow, err = authenticateflow.NewStateful(ctx, cfg, nil)
+		state.authenticateFlow, err = authenticateflow.NewStateful(ctx, tracerProvider, cfg, nil)
 	}
 	if err != nil {
 		return nil, err

cmd/pomerium/main.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/pomerium/pomerium/config"
 	"github.com/pomerium/pomerium/internal/log"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/version"
 	_ "github.com/pomerium/pomerium/internal/zero/bootstrap/writers/filesystem"
 	_ "github.com/pomerium/pomerium/internal/zero/bootstrap/writers/k8s"
@@ -30,9 +31,8 @@ func main() {
 	}
 	root.AddCommand(zero_cmd.BuildRootCmd())
 	root.PersistentFlags().StringVar(&configFile, "config", "", "Specify configuration file location")
-
-	ctx := context.Background()
 	log.SetLevel(zerolog.InfoLevel)
+	ctx := trace.NewContext(context.Background())
 	runFn := run
 	if zero_cmd.IsManagedMode(configFile) {
 		runFn = zero_cmd.Run

config/envoyconfig/bootstrap.go

@@ -37,7 +37,7 @@ func (b *Builder) BuildBootstrap(
 	cfg *config.Config,
 	fullyStatic bool,
 ) (bootstrap *envoy_config_bootstrap_v3.Bootstrap, err error) {
-	ctx, span := trace.StartSpan(ctx, "envoyconfig.Builder.BuildBootstrap")
+	ctx, span := trace.Continue(ctx, "envoyconfig.Builder.BuildBootstrap")
 	defer span.End()
 
 	bootstrap = new(envoy_config_bootstrap_v3.Bootstrap)
@@ -180,7 +180,7 @@ func (b *Builder) BuildBootstrapStaticResources(
 	cfg *config.Config,
 	fullyStatic bool,
 ) (staticResources *envoy_config_bootstrap_v3.Bootstrap_StaticResources, err error) {
-	ctx, span := trace.StartSpan(ctx, "envoyconfig.Builder.BuildBootstrapStaticResources")
+	ctx, span := trace.Continue(ctx, "envoyconfig.Builder.BuildBootstrapStaticResources")
 	defer span.End()
 
 	staticResources = new(envoy_config_bootstrap_v3.Bootstrap_StaticResources)

config/envoyconfig/clusters.go

@@ -26,7 +26,7 @@ import (
 
 // BuildClusters builds envoy clusters from the given config.
 func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*envoy_config_cluster_v3.Cluster, error) {
-	ctx, span := trace.StartSpan(ctx, "envoyconfig.Builder.BuildClusters")
+	ctx, span := trace.Continue(ctx, "envoyconfig.Builder.BuildClusters")
 	defer span.End()
 
 	grpcURLs := []*url.URL{{
@@ -104,13 +104,6 @@ func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*env
 		envoyAdminCluster,
 	}
 
-	tracingCluster, err := buildTracingCluster(cfg.Options)
-	if err != nil {
-		return nil, err
-	} else if tracingCluster != nil {
-		clusters = append(clusters, tracingCluster)
-	}
-
 	if config.IsProxy(cfg.Options.Services) {
 		for policy := range cfg.Options.GetAllPolicies() {
 			if len(policy.To) > 0 {

config/envoyconfig/filters.go

@@ -33,6 +33,16 @@ func ExtAuthzFilter(grpcClientTimeout *durationpb.Duration) *envoy_extensions_fi
 								ClusterName: "pomerium-authorize",
 							},
 						},
+						InitialMetadata: []*envoy_config_core_v3.HeaderValue{
+							{
+								Key:   "x-pomerium-traceparent",
+								Value: `%DYNAMIC_METADATA(pomerium.internal:traceparent)%`,
+							},
+							{
+								Key:   "x-pomerium-tracestate",
+								Value: `%DYNAMIC_METADATA(pomerium.internal:tracestate)%`,
+							},
+						},
 					},
 				},
 				MetadataContextNamespaces: []string{"com.pomerium.client-certificate-info"},

config/envoyconfig/http_connection_manager.go

@@ -39,6 +39,22 @@ func (b *Builder) buildVirtualHost(
 		return nil, err
 	}
 	vh.Routes = append(vh.Routes, rs...)
+	vh.RequestHeadersToAdd = []*envoy_config_core_v3.HeaderValueOption{
+		{
+			Header: &envoy_config_core_v3.HeaderValue{
+				Key:   "x-pomerium-traceparent",
+				Value: `%DYNAMIC_METADATA(pomerium.internal:traceparent)%`,
+			},
+			AppendAction: envoy_config_core_v3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD,
+		},
+		{
+			Header: &envoy_config_core_v3.HeaderValue{
+				Key:   "x-pomerium-tracestate",
+				Value: `%DYNAMIC_METADATA(pomerium.internal:tracestate)%`,
+			},
+			AppendAction: envoy_config_core_v3.HeaderValueOption_APPEND_IF_EXISTS_OR_ADD,
+		},
+	}
 
 	return vh, nil
 }

config/envoyconfig/listeners.go

@@ -19,7 +19,7 @@ func (b *Builder) BuildListeners(
 	cfg *config.Config,
 	fullyStatic bool,
 ) ([]*envoy_config_listener_v3.Listener, error) {
-	ctx, span := trace.StartSpan(ctx, "envoyconfig.Builder.BuildListeners")
+	ctx, span := trace.Continue(ctx, "envoyconfig.Builder.BuildListeners")
 	defer span.End()
 
 	var listeners []*envoy_config_listener_v3.Listener

config/envoyconfig/listeners_grpc.go

@@ -67,6 +67,9 @@ func (b *Builder) buildGRPCHTTPConnectionManagerFilter() *envoy_config_listener_
 				PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: fmt.Sprintf("/%s/", svc)},
 				Grpc:          &envoy_config_route_v3.RouteMatch_GrpcRouteMatchOptions{},
 			},
+			Decorator: &envoy_config_route_v3.Decorator{
+				Operation: fmt.Sprintf("pomerium-control-plane-grpc %s", svc),
+			},
 			Action: &envoy_config_route_v3.Route_Route{
 				Route: &envoy_config_route_v3.RouteAction{
 					ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{

config/envoyconfig/listeners_main.go

@@ -7,8 +7,12 @@ import (
 	envoy_config_accesslog_v3 "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3"
 	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	envoy_config_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	tracev3 "github.com/envoyproxy/go-control-plane/envoy/config/trace/v3"
 	envoy_extensions_access_loggers_grpc_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/grpc/v3"
+	envoy_extensions_filters_http_header_to_metadata "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/header_to_metadata/v3"
 	envoy_extensions_filters_network_http_connection_manager "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	metadatav3 "github.com/envoyproxy/go-control-plane/envoy/type/metadata/v3"
+	envoy_tracing_v3 "github.com/envoyproxy/go-control-plane/envoy/type/tracing/v3"
 	envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3"
 	"google.golang.org/protobuf/types/known/durationpb"
 	"google.golang.org/protobuf/types/known/wrapperspb"
@@ -119,6 +123,50 @@ func (b *Builder) buildMainHTTPConnectionManagerFilter(
 	}
 
 	filters := []*envoy_extensions_filters_network_http_connection_manager.HttpFilter{
+		LuaFilter(luascripts.TraceContext),
+		{
+			Name: "envoy.filters.http.header_to_metadata",
+			ConfigType: &envoy_extensions_filters_network_http_connection_manager.HttpFilter_TypedConfig{
+				TypedConfig: marshalAny(&envoy_extensions_filters_http_header_to_metadata.Config{
+					RequestRules: []*envoy_extensions_filters_http_header_to_metadata.Config_Rule{
+						{
+							Header: "x-pomerium-traceparent",
+							OnHeaderPresent: &envoy_extensions_filters_http_header_to_metadata.Config_KeyValuePair{
+								MetadataNamespace: "pomerium.internal",
+								Key:               "traceparent",
+							},
+							Remove: false,
+						},
+						{
+							Header: "x-pomerium-tracestate",
+							OnHeaderPresent: &envoy_extensions_filters_http_header_to_metadata.Config_KeyValuePair{
+								MetadataNamespace: "pomerium.internal",
+								Key:               "tracestate",
+							},
+							Remove: false,
+						},
+					},
+					ResponseRules: []*envoy_extensions_filters_http_header_to_metadata.Config_Rule{
+						{
+							Header: "x-pomerium-traceparent",
+							OnHeaderPresent: &envoy_extensions_filters_http_header_to_metadata.Config_KeyValuePair{
+								MetadataNamespace: "pomerium.internal",
+								Key:               "traceparent",
+							},
+							Remove: false,
+						},
+						{
+							Header: "x-pomerium-tracestate",
+							OnHeaderPresent: &envoy_extensions_filters_http_header_to_metadata.Config_KeyValuePair{
+								MetadataNamespace: "pomerium.internal",
+								Key:               "tracestate",
+							},
+							Remove: false,
+						},
+					},
+				}),
+			},
+		},
 		LuaFilter(luascripts.RemoveImpersonateHeaders),
 		LuaFilter(luascripts.SetClientCertificateMetadata),
 		ExtAuthzFilter(grpcClientTimeout),
@@ -133,11 +181,6 @@ func (b *Builder) buildMainHTTPConnectionManagerFilter(
 		maxStreamDuration = durationpb.New(cfg.Options.WriteTimeout)
 	}
 
-	tracingProvider, err := buildTracingHTTP(cfg.Options)
-	if err != nil {
-		return nil, err
-	}
-
 	localReply, err := b.buildLocalReplyConfig(cfg.Options)
 	if err != nil {
 		return nil, err
@@ -156,8 +199,72 @@ func (b *Builder) buildMainHTTPConnectionManagerFilter(
 		HttpProtocolOptions: http1ProtocolOptions,
 		RequestTimeout:      durationpb.New(cfg.Options.ReadTimeout),
 		Tracing: &envoy_extensions_filters_network_http_connection_manager.HttpConnectionManager_Tracing{
-			RandomSampling: &envoy_type_v3.Percent{Value: cfg.Options.TracingSampleRate * 100},
+			RandomSampling:    &envoy_type_v3.Percent{Value: cfg.Options.TracingSampleRate * 100},
-			Provider:       tracingProvider,
+			ClientSampling:    &envoy_type_v3.Percent{Value: cfg.Options.TracingSampleRate * 100},
+			Verbose:           true,
+			SpawnUpstreamSpan: wrapperspb.Bool(false),
+			Provider: &tracev3.Tracing_Http{
+				Name: "envoy.tracers.opentelemetry",
+				ConfigType: &tracev3.Tracing_Http_TypedConfig{
+					TypedConfig: marshalAny(&tracev3.OpenTelemetryConfig{
+						GrpcService: &envoy_config_core_v3.GrpcService{
+							TargetSpecifier: &envoy_config_core_v3.GrpcService_EnvoyGrpc_{
+								EnvoyGrpc: &envoy_config_core_v3.GrpcService_EnvoyGrpc{
+									ClusterName: "pomerium-control-plane-grpc",
+								},
+							},
+						},
+						ServiceName: "Envoy",
+					}),
+				},
+			},
+			MaxPathTagLength: wrapperspb.UInt32(1024),
+			CustomTags: []*envoy_tracing_v3.CustomTag{
+				{
+					Tag: "pomerium.traceparent",
+					Type: &envoy_tracing_v3.CustomTag_Metadata_{
+						Metadata: &envoy_tracing_v3.CustomTag_Metadata{
+							Kind: &metadatav3.MetadataKind{
+								Kind: &metadatav3.MetadataKind_Request_{
+									Request: &metadatav3.MetadataKind_Request{},
+								},
+							},
+							MetadataKey: &metadatav3.MetadataKey{
+								Key: "pomerium.internal",
+								Path: []*metadatav3.MetadataKey_PathSegment{
+									{
+										Segment: &metadatav3.MetadataKey_PathSegment_Key{
+											Key: "traceparent",
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+				{
+					Tag: "pomerium.tracestate",
+					Type: &envoy_tracing_v3.CustomTag_Metadata_{
+						Metadata: &envoy_tracing_v3.CustomTag_Metadata{
+							Kind: &metadatav3.MetadataKind{
+								Kind: &metadatav3.MetadataKind_Request_{
+									Request: &metadatav3.MetadataKind_Request{},
+								},
+							},
+							MetadataKey: &metadatav3.MetadataKey{
+								Key: "pomerium.internal",
+								Path: []*metadatav3.MetadataKey_PathSegment{
+									{
+										Segment: &metadatav3.MetadataKey_PathSegment_Key{
+											Key: "tracestate",
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
 		},
 		// See https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for
 		UseRemoteAddress:  &wrapperspb.BoolValue{Value: true},

config/envoyconfig/lua.go

@@ -14,6 +14,7 @@ var luascripts struct {
 	RemoveImpersonateHeaders     string
 	RewriteHeaders               string
 	SetClientCertificateMetadata string
+	TraceContext                 string
 }
 
 func init() {
@@ -23,6 +24,7 @@ func init() {
 		"luascripts/remove-impersonate-headers.lua":      &luascripts.RemoveImpersonateHeaders,
 		"luascripts/rewrite-headers.lua":                 &luascripts.RewriteHeaders,
 		"luascripts/set-client-certificate-metadata.lua": &luascripts.SetClientCertificateMetadata,
+		"luascripts/trace-context.lua":                   &luascripts.TraceContext,
 	}
 
 	err := fs.WalkDir(luaFS, "luascripts", func(p string, d fs.DirEntry, err error) error {

config/envoyconfig/luascripts/trace-context.lua

@@ -0,0 +1,33 @@
+function envoy_on_request(request_handle)
+  local headers = request_handle:headers()
+  local path = headers:get(":path")
+
+  if path:find("#") ~= nil then
+    return
+  end
+
+  local function substitute_query_param(query_param_name, header_name)
+    local i, j = path:find(query_param_name .. "=")
+    if i ~= nil and (path:sub(i - 1, i - 1) == "&" or path:sub(i - 1, i - 1) == "?") then
+      local k = path:find("&", j + 1)
+      if k ~= nil then
+        k = k - 1
+      else
+        k = #path
+      end
+      local value = path:sub(j + 1, k)
+      if value ~= nil then
+        headers:replace(header_name, value)
+        return true
+      end
+    end
+    return false
+  end
+
+  if substitute_query_param("pomerium_traceparent", "x-pomerium-traceparent") then
+    substitute_query_param("pomerium_tracestate", "x-pomerium-tracestate")
+  end
+end
+
+function envoy_on_response(response_handle)
+end

config/envoyconfig/outbound.go

@@ -109,6 +109,9 @@ func (b *Builder) buildOutboundRoutes() []*envoy_config_route_v3.Route {
 					PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: prefix},
 					Grpc:          &envoy_config_route_v3.RouteMatch_GrpcRouteMatchOptions{},
 				},
+				Decorator: &envoy_config_route_v3.Decorator{
+					Operation: fmt.Sprintf("Outbound (grpc): %s %s", def.Cluster, prefix),
+				},
 				Action: &envoy_config_route_v3.Route_Route{
 					Route: &envoy_config_route_v3.RouteAction{
 						ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
@@ -131,6 +134,9 @@ func (b *Builder) buildOutboundRoutes() []*envoy_config_route_v3.Route {
 		Match: &envoy_config_route_v3.RouteMatch{
 			PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: "/envoy/stats/prometheus"},
 		},
+		Decorator: &envoy_config_route_v3.Decorator{
+			Operation: "Outbound: envoy-metrics /envoy/stats/prometheus/*",
+		},
 		Action: &envoy_config_route_v3.Route_Route{
 			Route: &envoy_config_route_v3.RouteAction{
 				ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{

config/envoyconfig/route_configurations.go

@@ -20,7 +20,7 @@ func (b *Builder) BuildRouteConfigurations(
 	ctx context.Context,
 	cfg *config.Config,
 ) ([]*envoy_config_route_v3.RouteConfiguration, error) {
-	ctx, span := trace.StartSpan(ctx, "envoyconfig.Builder.BuildRouteConfigurations")
+	ctx, span := trace.Continue(ctx, "envoyconfig.Builder.BuildRouteConfigurations")
 	defer span.End()
 
 	var routeConfigurations []*envoy_config_route_v3.RouteConfiguration

config/envoyconfig/routes.go

@@ -114,6 +114,9 @@ func (b *Builder) buildControlPlanePathRoute(
 		Match: &envoy_config_route_v3.RouteMatch{
 			PathSpecifier: &envoy_config_route_v3.RouteMatch_Path{Path: path},
 		},
+		Decorator: &envoy_config_route_v3.Decorator{
+			Operation: "internal: ${method} ${host}${path}",
+		},
 		Action: &envoy_config_route_v3.Route_Route{
 			Route: &envoy_config_route_v3.RouteAction{
 				ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
@@ -138,6 +141,9 @@ func (b *Builder) buildControlPlanePrefixRoute(
 		Match: &envoy_config_route_v3.RouteMatch{
 			PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: prefix},
 		},
+		Decorator: &envoy_config_route_v3.Decorator{
+			Operation: "internal: ${method} ${host}${path}",
+		},
 		Action: &envoy_config_route_v3.Route_Route{
 			Route: &envoy_config_route_v3.RouteAction{
 				ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
@@ -269,8 +275,11 @@ func (b *Builder) buildRouteForPolicyAndMatch(
 	}
 
 	route := &envoy_config_route_v3.Route{
-		Name:                   name,
+		Name:  name,
-		Match:                  match,
+		Match: match,
+		Decorator: &envoy_config_route_v3.Decorator{
+			Operation: "ingress: ${method} ${host}${path}",
+		},
 		Metadata:               &envoy_config_core_v3.Metadata{},
 		RequestHeadersToRemove: getRequestHeadersToRemove(cfg.Options, policy),
 		ResponseHeadersToAdd:   toEnvoyHeaders(cfg.Options.GetSetResponseHeadersForPolicy(policy)),

config/envoyconfig/tracing.go

@@ -1,134 +0,0 @@
-package envoyconfig
-
-import (
-	"fmt"
-	"net"
-
-	envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
-	envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
-	envoy_config_trace_v3 "github.com/envoyproxy/go-control-plane/envoy/config/trace/v3"
-	"google.golang.org/protobuf/types/known/durationpb"
-
-	"github.com/pomerium/pomerium/config"
-	"github.com/pomerium/pomerium/internal/telemetry/trace"
-	"github.com/pomerium/pomerium/pkg/protoutil"
-)
-
-func buildTracingCluster(options *config.Options) (*envoy_config_cluster_v3.Cluster, error) {
-	tracingOptions, err := config.NewTracingOptions(options)
-	if err != nil {
-		return nil, fmt.Errorf("envoyconfig: invalid tracing config: %w", err)
-	}
-
-	switch tracingOptions.Provider {
-	case trace.DatadogTracingProviderName:
-		addr, _ := parseAddress("127.0.0.1:8126")
-
-		if options.TracingDatadogAddress != "" {
-			addr, err = parseAddress(options.TracingDatadogAddress)
-			if err != nil {
-				return nil, fmt.Errorf("envoyconfig: invalid tracing datadog address: %w", err)
-			}
-		}
-
-		endpoints := []*envoy_config_endpoint_v3.LbEndpoint{{
-			HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{
-				Endpoint: &envoy_config_endpoint_v3.Endpoint{
-					Address: addr,
-				},
-			},
-		}}
-
-		return &envoy_config_cluster_v3.Cluster{
-			Name: "datadog-apm",
-			ConnectTimeout: &durationpb.Duration{
-				Seconds: 5,
-			},
-			ClusterDiscoveryType: getClusterDiscoveryType(endpoints),
-			LbPolicy:             envoy_config_cluster_v3.Cluster_ROUND_ROBIN,
-			LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{
-				ClusterName: "datadog-apm",
-				Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{{
-					LbEndpoints: endpoints,
-				}},
-			},
-		}, nil
-	case trace.ZipkinTracingProviderName:
-		host := tracingOptions.ZipkinEndpoint.Host
-		if _, port, _ := net.SplitHostPort(host); port == "" {
-			if tracingOptions.ZipkinEndpoint.Scheme == "https" {
-				host = net.JoinHostPort(host, "443")
-			} else {
-				host = net.JoinHostPort(host, "80")
-			}
-		}
-
-		addr, err := parseAddress(host)
-		if err != nil {
-			return nil, fmt.Errorf("envoyconfig: invalid tracing zipkin address: %w", err)
-		}
-
-		endpoints := []*envoy_config_endpoint_v3.LbEndpoint{{
-			HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{
-				Endpoint: &envoy_config_endpoint_v3.Endpoint{
-					Address: addr,
-				},
-			},
-		}}
-		return &envoy_config_cluster_v3.Cluster{
-			Name: "zipkin",
-			ConnectTimeout: &durationpb.Duration{
-				Seconds: 5,
-			},
-			ClusterDiscoveryType: getClusterDiscoveryType(endpoints),
-			LbPolicy:             envoy_config_cluster_v3.Cluster_ROUND_ROBIN,
-			LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{
-				ClusterName: "zipkin",
-				Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{{
-					LbEndpoints: endpoints,
-				}},
-			},
-		}, nil
-	default:
-		return nil, nil
-	}
-}
-
-func buildTracingHTTP(options *config.Options) (*envoy_config_trace_v3.Tracing_Http, error) {
-	tracingOptions, err := config.NewTracingOptions(options)
-	if err != nil {
-		return nil, fmt.Errorf("invalid tracing config: %w", err)
-	}
-
-	switch tracingOptions.Provider {
-	case trace.DatadogTracingProviderName:
-		tracingTC := protoutil.NewAny(&envoy_config_trace_v3.DatadogConfig{
-			CollectorCluster: "datadog-apm",
-			ServiceName:      tracingOptions.Service,
-		})
-		return &envoy_config_trace_v3.Tracing_Http{
-			Name: "envoy.tracers.datadog",
-			ConfigType: &envoy_config_trace_v3.Tracing_Http_TypedConfig{
-				TypedConfig: tracingTC,
-			},
-		}, nil
-	case trace.ZipkinTracingProviderName:
-		path := tracingOptions.ZipkinEndpoint.Path
-		if path == "" {
-			path = "/"
-		}
-		tracingTC := protoutil.NewAny(&envoy_config_trace_v3.ZipkinConfig{
-			CollectorCluster:         "zipkin",
-			CollectorEndpoint:        path,
-			CollectorEndpointVersion: envoy_config_trace_v3.ZipkinConfig_HTTP_JSON,
-		})
-		return &envoy_config_trace_v3.Tracing_Http{
-			Name: "envoy.tracers.zipkin",
-			ConfigType: &envoy_config_trace_v3.Tracing_Http_TypedConfig{
-				TypedConfig: tracingTC,
-			},
-		}, nil
-	default:
-		return nil, nil
-	}
-}

config/envoyconfig/tracing_test.go

@@ -1,135 +0,0 @@
-package envoyconfig
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/pomerium/pomerium/config"
-	"github.com/pomerium/pomerium/internal/testutil"
-)
-
-func TestBuildTracingCluster(t *testing.T) {
-	t.Run("datadog", func(t *testing.T) {
-		c, err := buildTracingCluster(&config.Options{
-			TracingProvider: "datadog",
-		})
-		require.NoError(t, err)
-		testutil.AssertProtoJSONEqual(t, `
-			{
-				"name": "datadog-apm",
-				"type": "STATIC",
-				"connectTimeout": "5s",
-				"loadAssignment": {
-					"clusterName": "datadog-apm",
-					"endpoints": [{
-						"lbEndpoints": [{
-							"endpoint": {
-								"address": {
-									"socketAddress": {
-										"address": "127.0.0.1",
-										"portValue": 8126
-									}
-								}
-							}
-						}]
-					}]
-				}
-			}
-		`, c)
-
-		c, err = buildTracingCluster(&config.Options{
-			TracingProvider:       "datadog",
-			TracingDatadogAddress: "example.com:8126",
-		})
-		require.NoError(t, err)
-		testutil.AssertProtoJSONEqual(t, `
-			{
-				"name": "datadog-apm",
-				"type": "STRICT_DNS",
-				"connectTimeout": "5s",
-				"loadAssignment": {
-					"clusterName": "datadog-apm",
-					"endpoints": [{
-						"lbEndpoints": [{
-							"endpoint": {
-								"address": {
-									"socketAddress": {
-										"address": "example.com",
-										"portValue": 8126
-									}
-								}
-							}
-						}]
-					}]
-				}
-			}
-		`, c)
-	})
-	t.Run("zipkin", func(t *testing.T) {
-		c, err := buildTracingCluster(&config.Options{
-			TracingProvider: "zipkin",
-			ZipkinEndpoint:  "https://example.com/api/v2/spans",
-		})
-		require.NoError(t, err)
-		testutil.AssertProtoJSONEqual(t, `
-			{
-				"name": "zipkin",
-				"type": "STRICT_DNS",
-				"connectTimeout": "5s",
-				"loadAssignment": {
-					"clusterName": "zipkin",
-					"endpoints": [{
-						"lbEndpoints": [{
-							"endpoint": {
-								"address": {
-									"socketAddress": {
-										"address": "example.com",
-										"portValue": 443
-									}
-								}
-							}
-						}]
-					}]
-				}
-			}
-		`, c)
-	})
-}
-
-func TestBuildTracingHTTP(t *testing.T) {
-	t.Run("datadog", func(t *testing.T) {
-		h, err := buildTracingHTTP(&config.Options{
-			TracingProvider: "datadog",
-		})
-		require.NoError(t, err)
-		testutil.AssertProtoJSONEqual(t, `
-			{
-				"name": "envoy.tracers.datadog",
-				"typedConfig": {
-					"@type": "type.googleapis.com/envoy.config.trace.v3.DatadogConfig",
-					"collectorCluster": "datadog-apm",
-					"serviceName": "pomerium"
-				}
-			}
-		`, h)
-	})
-	t.Run("zipkin", func(t *testing.T) {
-		h, err := buildTracingHTTP(&config.Options{
-			TracingProvider: "zipkin",
-			ZipkinEndpoint:  "https://example.com/api/v2/spans",
-		})
-		require.NoError(t, err)
-		testutil.AssertProtoJSONEqual(t, `
-			{
-				"name": "envoy.tracers.zipkin",
-				"typedConfig": {
-					"@type": "type.googleapis.com/envoy.config.trace.v3.ZipkinConfig",
-					"collectorCluster": "zipkin",
-					"collectorEndpoint": "/api/v2/spans",
-					"collectorEndpointVersion": "HTTP_JSON"
-				}
-			}
-		`, h)
-	})
-}

config/log.go

@@ -20,11 +20,6 @@ func NewLogManager(ctx context.Context, src Source) *LogManager {
 	return mgr
 }
 
-// Close closes the log manager.
-func (mgr *LogManager) Close() error {
-	return nil
-}
-
 // OnConfigChange is called whenever configuration changes.
 func (mgr *LogManager) OnConfigChange(_ context.Context, cfg *Config) {
 	if cfg == nil || cfg.Options == nil {

config/metrics.go

@@ -45,11 +45,6 @@ func NewMetricsManager(ctx context.Context, src Source) *MetricsManager {
 	return mgr
 }
 
-// Close closes any underlying http server.
-func (mgr *MetricsManager) Close() error {
-	return nil
-}
-
 // OnConfigChange updates the metrics manager when configuration is changed.
 func (mgr *MetricsManager) OnConfigChange(ctx context.Context, cfg *Config) {
 	mgr.mu.Lock()

config/trace.go

@@ -1,125 +0,0 @@
-package config
-
-import (
-	"context"
-	"fmt"
-	"reflect"
-	"sync"
-
-	"github.com/rs/zerolog"
-
-	"github.com/pomerium/pomerium/internal/log"
-	"github.com/pomerium/pomerium/internal/telemetry"
-	"github.com/pomerium/pomerium/internal/telemetry/trace"
-	"github.com/pomerium/pomerium/internal/urlutil"
-)
-
-// TracingOptions are the options for tracing.
-type TracingOptions = trace.TracingOptions
-
-// NewTracingOptions builds a new TracingOptions from core Options
-func NewTracingOptions(o *Options) (*TracingOptions, error) {
-	tracingOpts := TracingOptions{
-		Provider:            o.TracingProvider,
-		Service:             telemetry.ServiceName(o.Services),
-		JaegerAgentEndpoint: o.TracingJaegerAgentEndpoint,
-		SampleRate:          o.TracingSampleRate,
-	}
-
-	switch o.TracingProvider {
-	case trace.DatadogTracingProviderName:
-		tracingOpts.DatadogAddress = o.TracingDatadogAddress
-	case trace.JaegerTracingProviderName:
-		if o.TracingJaegerCollectorEndpoint != "" {
-			jaegerCollectorEndpoint, err := urlutil.ParseAndValidateURL(o.TracingJaegerCollectorEndpoint)
-			if err != nil {
-				return nil, fmt.Errorf("config: invalid jaeger endpoint url: %w", err)
-			}
-			tracingOpts.JaegerCollectorEndpoint = jaegerCollectorEndpoint
-			tracingOpts.JaegerAgentEndpoint = o.TracingJaegerAgentEndpoint
-		}
-	case trace.ZipkinTracingProviderName:
-		zipkinEndpoint, err := urlutil.ParseAndValidateURL(o.ZipkinEndpoint)
-		if err != nil {
-			return nil, fmt.Errorf("config: invalid zipkin endpoint url: %w", err)
-		}
-		tracingOpts.ZipkinEndpoint = zipkinEndpoint
-	case "":
-		return &TracingOptions{}, nil
-	default:
-		return nil, fmt.Errorf("config: provider %s unknown", o.TracingProvider)
-	}
-
-	return &tracingOpts, nil
-}
-
-// A TraceManager manages setting up a trace exporter based on configuration options.
-type TraceManager struct {
-	mu        sync.Mutex
-	traceOpts *TracingOptions
-	provider  trace.Provider
-}
-
-// NewTraceManager creates a new TraceManager.
-func NewTraceManager(ctx context.Context, src Source) *TraceManager {
-	ctx = log.WithContext(ctx, func(c zerolog.Context) zerolog.Context {
-		return c.Str("service", "trace_manager")
-	})
-	mgr := &TraceManager{}
-	src.OnConfigChange(ctx, mgr.OnConfigChange)
-	mgr.OnConfigChange(ctx, src.GetConfig())
-	return mgr
-}
-
-// Close closes any underlying trace exporter.
-func (mgr *TraceManager) Close() error {
-	mgr.mu.Lock()
-	defer mgr.mu.Unlock()
-
-	var err error
-	if mgr.provider != nil {
-		err = mgr.provider.Unregister()
-	}
-	return err
-}
-
-// OnConfigChange updates the manager whenever the configuration is changed.
-func (mgr *TraceManager) OnConfigChange(ctx context.Context, cfg *Config) {
-	mgr.mu.Lock()
-	defer mgr.mu.Unlock()
-
-	traceOpts, err := NewTracingOptions(cfg.Options)
-	if err != nil {
-		log.Ctx(ctx).Error().Err(err).Msg("trace: failed to build tracing options")
-		return
-	}
-
-	if reflect.DeepEqual(traceOpts, mgr.traceOpts) {
-		log.Ctx(ctx).Debug().Msg("no change detected in trace options")
-		return
-	}
-	mgr.traceOpts = traceOpts
-
-	if mgr.provider != nil {
-		_ = mgr.provider.Unregister()
-		mgr.provider = nil
-	}
-
-	if !traceOpts.Enabled() {
-		return
-	}
-
-	log.Ctx(ctx).Info().Interface("options", traceOpts).Msg("trace: starting exporter")
-
-	mgr.provider, err = trace.GetProvider(traceOpts)
-	if err != nil {
-		log.Ctx(ctx).Error().Err(err).Msg("trace: failed to register exporter")
-		return
-	}
-
-	err = mgr.provider.Register(traceOpts)
-	if err != nil {
-		log.Ctx(ctx).Error().Err(err).Msg("trace: failed to register exporter")
-		return
-	}
-}

config/trace_test.go

@@ -1,161 +0,0 @@
-package config
-
-import (
-	"context"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"net/url"
-	"testing"
-	"time"
-
-	"github.com/google/go-cmp/cmp"
-	"github.com/stretchr/testify/assert"
-
-	"github.com/pomerium/pomerium/internal/telemetry/trace"
-)
-
-func Test_NewTracingOptions(t *testing.T) {
-	tests := []struct {
-		name    string
-		opts    *Options
-		want    *TracingOptions
-		wantErr bool
-	}{
-		{
-			"datadog_good",
-			&Options{TracingProvider: "datadog"},
-			&TracingOptions{Provider: "datadog", Service: "pomerium"},
-			false,
-		},
-		{
-			"jaeger_good",
-			&Options{TracingProvider: "jaeger", TracingJaegerAgentEndpoint: "foo", TracingJaegerCollectorEndpoint: "http://foo", Services: ServiceAll},
-			&TracingOptions{Provider: "jaeger", JaegerAgentEndpoint: "foo", JaegerCollectorEndpoint: &url.URL{Scheme: "http", Host: "foo"}, Service: "pomerium"},
-			false,
-		},
-		{
-			"jaeger_bad",
-			&Options{TracingProvider: "jaeger", TracingJaegerAgentEndpoint: "foo", TracingJaegerCollectorEndpoint: "badurl"},
-			nil,
-			true,
-		},
-		{
-			"zipkin_good",
-			&Options{TracingProvider: "zipkin", ZipkinEndpoint: "https://foo/api/v1/spans", Services: ServiceAuthorize},
-			&TracingOptions{Provider: "zipkin", ZipkinEndpoint: &url.URL{Scheme: "https", Host: "foo", Path: "/api/v1/spans"}, Service: "pomerium-authorize"},
-			false,
-		},
-		{
-			"zipkin_bad",
-			&Options{TracingProvider: "zipkin", ZipkinEndpoint: "notaurl"},
-			nil,
-			true,
-		},
-		{
-			"noprovider",
-			&Options{},
-			&TracingOptions{},
-			false,
-		},
-		{
-			"fakeprovider",
-			&Options{TracingProvider: "fake"},
-			nil,
-			true,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := NewTracingOptions(tt.opts)
-			assert.NotEqual(t, err == nil, tt.wantErr, "unexpected error value")
-			assert.Empty(t, cmp.Diff(tt.want, got))
-		})
-	}
-}
-
-func Test_TracingEnabled(t *testing.T) {
-	tests := []struct {
-		name string
-		opts *TracingOptions
-		want bool
-	}{
-		{"enabled", &TracingOptions{Provider: "zipkin"}, true},
-		{"not enabled", &TracingOptions{}, false},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			assert.Equal(t, tt.want, tt.opts.Enabled(), "unexpected tracing state")
-		})
-	}
-}
-
-func TestTraceManager(t *testing.T) {
-	ctx, clearTimeout := context.WithTimeout(context.Background(), time.Second*30)
-	defer clearTimeout()
-
-	type Request struct {
-		URL  string
-		Name string
-	}
-
-	incoming := make(chan Request, 100)
-
-	h := http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
-		var objs []struct {
-			Name string
-		}
-		json.NewDecoder(r.Body).Decode(&objs)
-		for _, obj := range objs {
-			incoming <- Request{Name: obj.Name, URL: r.Host}
-		}
-	})
-
-	srv1 := httptest.NewServer(h)
-	defer srv1.Close()
-	srv2 := httptest.NewServer(h)
-	defer srv2.Close()
-
-	src := NewStaticSource(&Config{Options: &Options{
-		TracingProvider:   "zipkin",
-		ZipkinEndpoint:    srv1.URL,
-		TracingSampleRate: 1,
-	}})
-
-	_ = NewTraceManager(ctx, src)
-
-	_, span := trace.StartSpan(ctx, "Example")
-	span.End()
-
-	src.SetConfig(ctx, &Config{Options: &Options{
-		TracingProvider:   "zipkin",
-		ZipkinEndpoint:    srv2.URL,
-		TracingSampleRate: 1,
-	}})
-
-	_, span = trace.StartSpan(ctx, "Example")
-	span.End()
-
-	expect := map[Request]struct{}{
-		{Name: "example", URL: srv1.Listener.Addr().String()}: {},
-		{Name: "example", URL: srv2.Listener.Addr().String()}: {},
-	}
-
-	for len(expect) > 0 {
-		var req Request
-		select {
-		case <-ctx.Done():
-			t.Error("timeout waiting for requests")
-			return
-		case req = <-incoming:
-		}
-
-		if _, ok := expect[req]; ok {
-			delete(expect, req)
-		} else {
-			t.Error("unexpected request", req)
-			return
-		}
-	}
-}

databroker/cache.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/rs/zerolog"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/metadata"
@@ -18,7 +19,7 @@ import (
 	"github.com/pomerium/pomerium/internal/atomicutil"
 	"github.com/pomerium/pomerium/internal/events"
 	"github.com/pomerium/pomerium/internal/log"
-	"github.com/pomerium/pomerium/internal/telemetry"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/version"
 	"github.com/pomerium/pomerium/pkg/cryptutil"
 	"github.com/pomerium/pomerium/pkg/envoy/files"
@@ -28,6 +29,7 @@ import (
 	"github.com/pomerium/pomerium/pkg/identity"
 	"github.com/pomerium/pomerium/pkg/identity/legacymanager"
 	"github.com/pomerium/pomerium/pkg/identity/manager"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
 // DataBroker represents the databroker service. The databroker service is a simple interface
@@ -42,6 +44,8 @@ type DataBroker struct {
 	localGRPCServer     *grpc.Server
 	localGRPCConnection *grpc.ClientConn
 	sharedKey           *atomicutil.Value[[]byte]
+	tracerProvider      oteltrace.TracerProvider
+	tracer              oteltrace.Tracer
 }
 
 // New creates a new databroker service.
@@ -58,9 +62,12 @@ func New(ctx context.Context, cfg *config.Config, eventsMgr *events.Manager) (*D
 		),
 	)
 
+	tracerProvider := trace.NewTracerProvider(ctx, "Data Broker")
+	tracer := tracerProvider.Tracer(trace.PomeriumCoreTracer)
 	// No metrics handler because we have one in the control plane.  Add one
 	// if we no longer register with that grpc Server
 	localGRPCServer := grpc.NewServer(
+		grpc.StatsHandler(trace.NewStatsHandler(otelgrpc.NewServerHandler(otelgrpc.WithTracerProvider(tracerProvider)))),
 		grpc.ChainStreamInterceptor(log.StreamServerInterceptor(log.Ctx(ctx)), si),
 		grpc.ChainUnaryInterceptor(log.UnaryServerInterceptor(log.Ctx(ctx)), ui),
 	)
@@ -71,12 +78,11 @@ func New(ctx context.Context, cfg *config.Config, eventsMgr *events.Manager) (*D
 	}
 
 	sharedKeyValue := atomicutil.NewValue(sharedKey)
-	clientStatsHandler := telemetry.NewGRPCClientStatsHandler(cfg.Options.Services)
 	clientDialOptions := []grpc.DialOption{
 		grpc.WithInsecure(),
-		grpc.WithChainUnaryInterceptor(clientStatsHandler.UnaryInterceptor, grpcutil.WithUnarySignedJWT(sharedKeyValue.Load)),
+		grpc.WithChainUnaryInterceptor(grpcutil.WithUnarySignedJWT(sharedKeyValue.Load)),
 		grpc.WithChainStreamInterceptor(grpcutil.WithStreamSignedJWT(sharedKeyValue.Load)),
-		grpc.WithStatsHandler(clientStatsHandler.Handler),
+		grpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(tracerProvider))),
 	}
 
 	ctx = log.WithContext(ctx, func(c zerolog.Context) zerolog.Context {
@@ -91,7 +97,7 @@ func New(ctx context.Context, cfg *config.Config, eventsMgr *events.Manager) (*D
 		return nil, err
 	}
 
-	dataBrokerServer, err := newDataBrokerServer(ctx, cfg)
+	dataBrokerServer, err := newDataBrokerServer(ctx, tracerProvider, cfg)
 	if err != nil {
 		return nil, err
 	}
@@ -103,6 +109,8 @@ func New(ctx context.Context, cfg *config.Config, eventsMgr *events.Manager) (*D
 		localGRPCConnection: localGRPCConnection,
 		sharedKey:           sharedKeyValue,
 		eventsMgr:           eventsMgr,
+		tracerProvider:      tracerProvider,
+		tracer:              tracer,
 	}
 	c.Register(c.localGRPCServer)
 
@@ -172,7 +180,7 @@ func (c *DataBroker) update(ctx context.Context, cfg *config.Config) error {
 	}
 
 	if cfg.Options.SupportsUserRefresh() {
-		authenticator, err := identity.NewAuthenticator(oauthOptions)
+		authenticator, err := identity.NewAuthenticator(ctx, c.tracerProvider, oauthOptions)
 		if err != nil {
 			log.Ctx(ctx).Error().Err(err).Msg("databroker: failed to create authenticator")
 		} else {

databroker/databroker.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 
+	oteltrace "go.opentelemetry.io/otel/trace"
 	"google.golang.org/protobuf/types/known/emptypb"
 
 	"github.com/pomerium/pomerium/config"
@@ -23,7 +24,7 @@ type dataBrokerServer struct {
 }
 
 // newDataBrokerServer creates a new databroker service server.
-func newDataBrokerServer(ctx context.Context, cfg *config.Config) (*dataBrokerServer, error) {
+func newDataBrokerServer(ctx context.Context, tracerProvider oteltrace.TracerProvider, cfg *config.Config) (*dataBrokerServer, error) {
 	srv := &dataBrokerServer{
 		sharedKey: atomicutil.NewValue([]byte{}),
 	}
@@ -33,7 +34,7 @@ func newDataBrokerServer(ctx context.Context, cfg *config.Config) (*dataBrokerSe
 		return nil, err
 	}
 
-	srv.server = databroker.New(ctx, opts...)
+	srv.server = databroker.New(ctx, tracerProvider, opts...)
 	srv.setKey(cfg)
 	return srv, nil
 }

databroker/databroker_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -29,7 +30,7 @@ var lis *bufconn.Listener
 func init() {
 	lis = bufconn.Listen(bufSize)
 	s := grpc.NewServer()
-	internalSrv := internal_databroker.New(context.Background())
+	internalSrv := internal_databroker.New(context.Background(), trace.NewNoopTracerProvider())
 	srv := &dataBrokerServer{server: internalSrv, sharedKey: atomicutil.NewValue([]byte{})}
 	databroker.RegisterDataBrokerServiceServer(s, srv)
 

go.mod

@@ -4,11 +4,8 @@ go 1.23.0
 
 require (
 	cloud.google.com/go/storage v1.46.0
-	contrib.go.opencensus.io/exporter/jaeger v0.2.1
 	contrib.go.opencensus.io/exporter/prometheus v0.4.2
-	contrib.go.opencensus.io/exporter/zipkin v0.1.2
 	github.com/CAFxX/httpcompression v0.0.9
-	github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20200406135749-5c268882acf0
 	github.com/VictoriaMetrics/fastcache v1.12.2
 	github.com/aws/aws-sdk-go-v2 v1.32.3
 	github.com/aws/aws-sdk-go-v2/config v1.28.1
@@ -46,7 +43,6 @@ require (
 	github.com/natefinch/atomic v1.0.1
 	github.com/oapi-codegen/runtime v1.1.1
 	github.com/open-policy-agent/opa v0.70.0
-	github.com/openzipkin/zipkin-go v0.4.3
 	github.com/peterbourgon/ff/v3 v3.4.0
 	github.com/pomerium/csrf v1.7.0
 	github.com/pomerium/datasource v0.18.2-0.20221108160055-c6134b5ed524
@@ -67,27 +63,32 @@ require (
 	github.com/volatiletech/null/v9 v9.0.0
 	github.com/yuin/gopher-lua v1.1.1
 	go.opencensus.io v0.24.0
-	go.opentelemetry.io/otel v1.31.0
+	go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.57.0
-	go.opentelemetry.io/otel/bridge/opencensus v1.31.0
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.57.0
-	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.31.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.57.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.31.0
+	go.opentelemetry.io/otel v1.32.0
-	go.opentelemetry.io/otel/metric v1.31.0
+	go.opentelemetry.io/otel/bridge/opencensus v1.32.0
-	go.opentelemetry.io/otel/sdk v1.31.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0
-	go.opentelemetry.io/otel/sdk/metric v1.31.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.32.0
-	go.opentelemetry.io/otel/trace v1.31.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.32.0
+	go.opentelemetry.io/otel/metric v1.32.0
+	go.opentelemetry.io/otel/sdk v1.32.0
+	go.opentelemetry.io/otel/sdk/metric v1.32.0
+	go.opentelemetry.io/otel/trace v1.32.0
+	go.opentelemetry.io/proto/otlp v1.3.1
 	go.uber.org/automaxprocs v1.6.0
 	go.uber.org/mock v0.5.0
 	go.uber.org/zap v1.27.0
 	golang.org/x/crypto v0.28.0
 	golang.org/x/net v0.30.0
 	golang.org/x/oauth2 v0.23.0
-	golang.org/x/sync v0.8.0
+	golang.org/x/sync v0.9.0
-	golang.org/x/sys v0.26.0
+	golang.org/x/sys v0.27.0
 	golang.org/x/time v0.7.0
 	google.golang.org/api v0.203.0
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28
-	google.golang.org/grpc v1.67.1
+	google.golang.org/grpc v1.68.0
 	google.golang.org/protobuf v1.35.1
 	gopkg.in/yaml.v3 v3.0.1
 	namespacelabs.dev/go-filenotify v0.0.0-20220511192020-53ea11be7eaa
@@ -104,7 +105,6 @@ require (
 	cloud.google.com/go/monitoring v1.21.1 // indirect
 	dario.cat/mergo v1.0.0 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
-	github.com/DataDog/datadog-go v3.5.0+incompatible // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect
@@ -141,7 +141,7 @@ require (
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.6.0 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.2 // indirect
@@ -161,7 +161,7 @@ require (
 	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
 	github.com/googleapis/gax-go/v2 v2.13.0 // indirect
 	github.com/gorilla/securecookie v1.1.1 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
@@ -189,7 +189,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
-	github.com/philhofer/fwd v1.1.2 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
@@ -209,10 +209,8 @@ require (
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/tchap/go-patricia/v2 v2.3.1 // indirect
-	github.com/tinylib/msgp v1.1.8 // indirect
 	github.com/tklauser/go-sysconf v0.3.14 // indirect
 	github.com/tklauser/numcpus v0.8.0 // indirect
-	github.com/uber/jaeger-client-go v2.25.0+incompatible // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
@@ -222,19 +220,15 @@ require (
 	github.com/zeebo/blake3 v0.2.4 // indirect
 	github.com/zeebo/xxh3 v1.0.2 // indirect
 	go.opentelemetry.io/contrib/detectors/gcp v1.29.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect
 	golang.org/x/mod v0.20.0 // indirect
-	golang.org/x/text v0.19.0 // indirect
+	golang.org/x/text v0.20.0 // indirect
 	golang.org/x/tools v0.24.0 // indirect
 	google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect
 	google.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a // indirect
-	gopkg.in/DataDog/dd-trace-go.v1 v1.22.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

go.sum

@@ -52,12 +52,8 @@ cloud.google.com/go/storage v1.46.0 h1:OTXISBpFd8KaA2ClT3K3oRk8UGOcTHtrZ1bW88xKi
 cloud.google.com/go/storage v1.46.0/go.mod h1:lM+gMAW91EfXIeMTBmixRsKL/XCxysytoAgduVikjMk=
 cloud.google.com/go/trace v1.11.1 h1:UNqdP+HYYtnm6lb91aNA5JQ0X14GnxkABGlfz2PzPew=
 cloud.google.com/go/trace v1.11.1/go.mod h1:IQKNQuBzH72EGaXEodKlNJrWykGZxet2zgjtS60OtjA=
-contrib.go.opencensus.io/exporter/jaeger v0.2.1 h1:yGBYzYMewVL0yO9qqJv3Z5+IRhPdU7e9o/2oKpX4YvI=
-contrib.go.opencensus.io/exporter/jaeger v0.2.1/go.mod h1:Y8IsLgdxqh1QxYxPC5IgXVmBaeLUeQFfBeBi9PbeZd0=
 contrib.go.opencensus.io/exporter/prometheus v0.4.2 h1:sqfsYl5GIY/L570iT+l93ehxaWJs2/OwXtiWwew3oAg=
 contrib.go.opencensus.io/exporter/prometheus v0.4.2/go.mod h1:dvEHbiKmgvbr5pjaF9fpw1KeYcjrnC1J8B+JKjsZyRQ=
-contrib.go.opencensus.io/exporter/zipkin v0.1.2 h1:YqE293IZrKtqPnpwDPH/lOqTWD/s3Iwabycam74JV3g=
-contrib.go.opencensus.io/exporter/zipkin v0.1.2/go.mod h1:mP5xM3rrgOjpn79MM8fZbj3gsxcuytSqtH0dxSWW1RE=
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
@@ -69,10 +65,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/CAFxX/httpcompression v0.0.9 h1:0ue2X8dOLEpxTm8tt+OdHcgA+gbDge0OqFQWGKSqgrg=
 github.com/CAFxX/httpcompression v0.0.9/go.mod h1:XX8oPZA+4IDcfZ0A71Hz0mZsv/YJOgYygkFhizVPilM=
-github.com/DataDog/datadog-go v3.5.0+incompatible h1:AShr9cqkF+taHjyQgcBcQUt/ZNK+iPq4ROaZwSX5c/U=
-github.com/DataDog/datadog-go v3.5.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
-github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20200406135749-5c268882acf0 h1:Y6HFfo8UuntPOpfmUmLb0o3MNYKfUuH2aNmvypsDbY4=
-github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20200406135749-5c268882acf0/go.mod h1:/VV3EFO/hTNQZHAqaj+CPGy2+ioFrP4EX3iRwozubhQ=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1 h1:pB2F2JKCj1Znmp2rwxxt1J0Fg0wezTMgWYk5Mpbi1kg=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1/go.mod h1:itPGVDKf9cC/ov4MdvJ2QZ0khw4bfoo9jzwTJlaxy2k=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 h1:UQ0AhxogsIRZDkElkblfnwjc3IaltCm2HUMvezQaL7s=
@@ -86,8 +78,6 @@ github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
-github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
-github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI=
 github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI=
 github.com/agnivade/levenshtein v1.2.0 h1:U9L4IOT0Y3i0TIlUIDJ7rVUziKi/zPbrJGaFrtYH3SY=
@@ -208,10 +198,6 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
-github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
-github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
-github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -231,8 +217,8 @@ github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7z
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
-github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
+github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
-github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/fxamacker/cbor/v2 v2.6.0 h1:sU6J2usfADwWlYDAFhZBQ6TnLFBHxgesMrQfQgk1tWA=
 github.com/fxamacker/cbor/v2 v2.6.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
 github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
@@ -267,8 +253,8 @@ github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiU
 github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
 github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
-github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
 github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
@@ -276,9 +262,7 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA
 github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
 github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -314,7 +298,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
-github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/brotli/go/cbrotli v0.0.0-20230829110029-ed738e842d2f h1:jopqB+UTSdJGEJT8tEqYyE29zN91fi2827oLET8tl7k=
@@ -370,8 +353,6 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s=
 github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A=
-github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
 github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
 github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
@@ -382,8 +363,8 @@ github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJr
 github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk=
 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 h1:ad0vkEBuk23VJzZR9nkLVG0YAoN9coASF1GusYX6AlU=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0/go.mod h1:igFoXX2ELCW06bol23DWPB5BEWfZISOzSP5K2sbLea0=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -448,7 +429,6 @@ github.com/libdns/libdns v0.2.2 h1:O6ws7bAfRPaBsgAYt8MDe2HcNBGC29hkZ9MX2eUSX3s=
 github.com/libdns/libdns v0.2.2/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ=
 github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae h1:dIZY4ULFcto4tAFlj1FYZl8ztUZ13bdq+PLY+NOfbyI=
 github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k=
-github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
 github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
 github.com/martinlindhe/base36 v1.1.1 h1:1F1MZ5MGghBXDZ2KJ3QfxmiydlWOGB8HCEtkap5NkVg=
@@ -503,13 +483,11 @@ github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+
 github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro=
 github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/ginkgo/v2 v2.19.1 h1:QXgq3Z8Crl5EL1WBAC98A5sEBHARrAJNzAmMxzLcRF0=
 github.com/onsi/ginkgo/v2 v2.19.1/go.mod h1:O3DtEWQkPa/F7fBMgmZQKKsluAy8pd3rEQdrjkPb9zA=
-github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
@@ -520,18 +498,10 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
 github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
-github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
-github.com/openzipkin/zipkin-go v0.4.3 h1:9EGwpqkgnwdEIJ+Od7QVSEIH+ocmm5nPat0G7sjsSdg=
-github.com/openzipkin/zipkin-go v0.4.3/go.mod h1:M9wCJZFWCo2RiY+o1eBCEMe0Dp2S5LDHcMZmk3RmK7c=
 github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
 github.com/peterbourgon/ff/v3 v3.4.0 h1:QBvM/rizZM1cB0p0lGMdmR7HxZeI/ZrBWB4DqLkMUBc=
 github.com/peterbourgon/ff/v3 v3.4.0/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ=
-github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
-github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw=
-github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
-github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1 h1:VGcrWe3yk6o+t7BdVNy5UDPWa4OZuDWtE1W1ZbS7Kyw=
-github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
 github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
 github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
@@ -539,7 +509,6 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
 github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
 github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -590,7 +559,6 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0=
 github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI=
-github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -636,7 +604,6 @@ github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+
 github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
 github.com/sryoya/protorand v0.0.0-20240429201223-e7440656b2a4 h1:/jKH9ivHOUkahZs3zPfJfOmkXDFB6OdsHZ4W8gyDb/c=
 github.com/sryoya/protorand v0.0.0-20240429201223-e7440656b2a4/go.mod h1:9a23nlv6vzBeVlQq6JQCjljZ6sfzsB6aha1m5Ly1W2Y=
-github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
@@ -662,17 +629,12 @@ github.com/tchap/go-patricia/v2 v2.3.1 h1:6rQp39lgIYZ+MHmdEq4xzuk1t7OdC35z/xm0BG
 github.com/tchap/go-patricia/v2 v2.3.1/go.mod h1:VZRHKAb53DLaG+nA9EaYYiaEx6YztwDlLElMsnSHD4k=
 github.com/testcontainers/testcontainers-go v0.34.0 h1:5fbgF0vIN5u+nD3IWabQwRybuB4GY8G2HHgCkbMzMHo=
 github.com/testcontainers/testcontainers-go v0.34.0/go.mod h1:6P/kMkQe8yqPHfPWNulFGdFHTD8HB2vLq/231xY2iPQ=
-github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
-github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0=
-github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw=
 github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU=
 github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY=
 github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY=
 github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE=
 github.com/tniswong/go.rfcx v0.0.0-20181019234604-07783c52761f h1:C43EMGXFtvYf/zunHR6ivZV7Z6ytg73t0GXwYyicXMQ=
 github.com/tniswong/go.rfcx v0.0.0-20181019234604-07783c52761f/go.mod h1:N+sR0vLSCTtI6o06PMWsjMB4TVqqDttKNq4iC9wvxVY=
-github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U=
-github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
 github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/valyala/gozstd v1.20.1 h1:xPnnnvjmaDDitMFfDxmQ4vpx0+3CdTg2o3lALvXTU/g=
 github.com/valyala/gozstd v1.20.1/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
@@ -713,30 +675,34 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.opentelemetry.io/contrib/detectors/gcp v1.29.0 h1:TiaiXB4DpGD3sdzNlYQxruQngn5Apwzi1X0DRhuGvDQ=
 go.opentelemetry.io/contrib/detectors/gcp v1.29.0/go.mod h1:GW2aWZNwR2ZxDLdv8OyC2G8zkRoQBuURgV7RPQgcPoU=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc=
+go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.57.0 h1:ydMxn2B3ZKzDXmjgE/tBtq7RsArxmikZUlRWComOPFs=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI=
+go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.57.0/go.mod h1:rD9Z+09JseOeFdSJUrtnA2hO4XBY3lf1Tj0tPqf+LEM=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.57.0 h1:qtFISDHKolvIxzSs0gIaiPUPR0Cucb0F2coHC7ZLdps=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.57.0/go.mod h1:Y+Pop1Q6hCOnETWTW4NROK/q1hv50hM7yDaUTjG8lp8=
-go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY=
+go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.57.0 h1:7F3XCD6WYzDkwbi8I8N+oYJWquPVScnRosKGgqjsR8c=
-go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE=
+go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.57.0/go.mod h1:Dk3C0BfIlZDZ5c6eVS7TYiH2vssuyUU3vUsgbrR+5V4=
-go.opentelemetry.io/otel/bridge/opencensus v1.31.0 h1:YrCZ8NpdMTunNIzRnNoG3KjSLu0PNmRtgtQVJuCxkAQ=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 h1:DheMAlT6POBP+gh8RUH19EOTnQIor5QE0uSRPtzCpSw=
-go.opentelemetry.io/otel/bridge/opencensus v1.31.0/go.mod h1:2yEkg7WRb15imAr0jfS4XDNd8LNe/hRES+kFezyO6LI=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0/go.mod h1:wZcGmeVO9nzP67aYSLDqXNWK87EZWhi7JWj1v7ZXf94=
-go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.31.0 h1:FZ6ei8GFW7kyPYdxJaV2rgI6M+4tvZzhYsQ2wgyVC08=
+go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U=
-go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.31.0/go.mod h1:MdEu/mC6j3D+tTEfvI15b5Ci2Fn7NneJ71YMoiS3tpI=
+go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 h1:K0XaT3DwHAcV4nKLzcQvwAgSyisUghWoY20I7huthMk=
+go.opentelemetry.io/otel/bridge/opencensus v1.32.0 h1:OVbbFgPG60UolI8ZUs+Z75NnKiO0C9QltXBrqUDImS0=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0/go.mod h1:B5Ki776z/MBnVha1Nzwp5arlzBbE3+1jk+pGmaP5HME=
+go.opentelemetry.io/otel/bridge/opencensus v1.32.0/go.mod h1:J5SEiJNu6zzqpcA6+AVpxUKzxNocUMsefgHRpS8zdW8=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.31.0 h1:FFeLy03iVTXP6ffeN2iXrxfGsZGCjVx0/4KlizjyBwU=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0 h1:j7ZSD+5yn+lo3sGV69nW04rRR0jhYnBwjuX3r0HvnK0=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.31.0/go.mod h1:TMu73/k1CP8nBUpDLc71Wj/Kf7ZS9FK5b53VapRsP9o=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0/go.mod h1:WXbYJTUaZXAbYd8lbgGuvih0yuCfOFC5RJoYnoLcGz8=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 h1:j9+03ymgYhPKmeXGk5Zu+cIZOlVzd9Zv7QIiyItjFBU=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.32.0 h1:IJFEoHiytixx8cMiVAO+GmHR6Frwu+u5Ur8njpFO6Ac=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0/go.mod h1:Y5+XiUG4Emn1hTfciPzGPJaSI+RpDts6BnCIir0SLqk=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.32.0/go.mod h1:3rHrKNtLIoS0oZwkY2vxi+oJcwFRWdtUyRII+so45p8=
-go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.32.0 h1:9kV11HXBHZAvuPUZxmMWrH8hZn/6UnHX4K0mu36vNsU=
-go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.32.0/go.mod h1:JyA0FHXe22E1NeNiHmVp7kFHglnexDQ7uRWDiiJ1hKQ=
-go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0 h1:cMyu9O88joYEaI47CnQkxO1XZdpoTF9fEnW2duIddhw=
-go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0/go.mod h1:6Am3rn7P9TVVeXYG+wtcGE7IE1tsQ+bP3AuWcKt/gOI=
-go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc=
+go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M=
-go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8=
+go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8=
-go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys=
+go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4=
-go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A=
+go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU=
+go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU=
+go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ=
+go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM=
+go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8=
 go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0=
 go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8=
 go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
@@ -792,7 +758,6 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
 golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
@@ -833,7 +798,6 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
@@ -860,8 +824,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
-golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -916,18 +880,16 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
-golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
@@ -940,12 +902,11 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
+golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
-golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -995,7 +956,6 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
 golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=
@@ -1058,12 +1018,11 @@ google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 h1:Df6WuGvthPzc+JiQ/G+m+sNX24kc0aTBqoDN/0yyykE=
 google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53/go.mod h1:fheguH3Am2dGp1LfXkrvwqC/KlFq8F0nLq3LryOMrrE=
-google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 h1:T6rh4haD3GVYsgEfWExoCZA2o2FmbNyKpTuAxbEFPTg=
+google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 h1:M0KvPgPmDZHPlbRbaNU1APr28TvwvvdUPlSv7PUvy8g=
-google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:wp2WsuBYj6j8wUdo3ToZsdxxixbvQNAHqVJrTgi5E5M=
+google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28/go.mod h1:dguCy7UOdZhTvLzDyt15+rOrawrpM4q7DD9dQ1P11P4=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 h1:XVhgTWWV3kGQlwJHR3upFWZeTsei6Oks1apkZSeonIE=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
@@ -1076,8 +1035,8 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji
 google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E=
+google.golang.org/grpc v1.68.0 h1:aHQeeJbo8zAkAa3pRzrVjZlbz6uSfeOXlJNQM0RAbz0=
-google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA=
+google.golang.org/grpc v1.68.0/go.mod h1:fmSPC5AsjSBCK54MyHRx48kpOti1/jRfOlwEWywNjWA=
 google.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a h1:UIpYSuWdWHSzjwcAFRLjKcPXFZVVLXGEM23W+NWqipw=
 google.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a/go.mod h1:9i1T9n4ZinTUZGgzENMi8MDDgbGC5mqTS75JAv6xN3A=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
@@ -1096,8 +1055,6 @@ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=
 google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
-gopkg.in/DataDog/dd-trace-go.v1 v1.22.0 h1:gpWsqqkwUldNZXGJqT69NU9MdEDhLboK1C4nMgR0MWw=
-gopkg.in/DataDog/dd-trace-go.v1 v1.22.0/go.mod h1:DVp8HmDh8PuTu2Z0fVVlBsyWaC++fzwVCaGWylTe3tg=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internal/authenticateflow/stateful.go

@@ -9,6 +9,8 @@ import (
 	"net/url"
 	"time"
 
+	"go.opentelemetry.io/otel"
+	oteltrace "go.opentelemetry.io/otel/trace"
 	"golang.org/x/oauth2"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
@@ -19,6 +21,7 @@ import (
 	"github.com/pomerium/pomerium/internal/httputil"
 	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/internal/sessions"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/urlutil"
 	"github.com/pomerium/pomerium/pkg/cryptutil"
 	"github.com/pomerium/pomerium/pkg/grpc"
@@ -56,7 +59,7 @@ type Stateful struct {
 
 // NewStateful initializes the authentication flow for the given configuration
 // and session store.
-func NewStateful(ctx context.Context, cfg *config.Config, sessionStore sessions.SessionStore) (*Stateful, error) {
+func NewStateful(ctx context.Context, tracerProvider oteltrace.TracerProvider, cfg *config.Config, sessionStore sessions.SessionStore) (*Stateful, error) {
 	s := &Stateful{
 		sessionDuration: cfg.Options.CookieExpire,
 		sessionStore:    sessionStore,
@@ -89,6 +92,7 @@ func NewStateful(ctx context.Context, cfg *config.Config, sessionStore sessions.
 	}
 
 	dataBrokerConn, err := outboundGRPCConnection.Get(ctx,
+		tracerProvider,
 		&grpc.OutboundOptions{
 			OutboundPort:   cfg.OutboundPort,
 			InstallationID: cfg.Options.InstallationID,
@@ -316,7 +320,7 @@ func (s *Stateful) LogAuthenticateEvent(*http.Request) {}
 // AuthenticateSignInURL returns a URL to redirect the user to the authenticate
 // domain.
 func (s *Stateful) AuthenticateSignInURL(
-	_ context.Context, queryParams url.Values, redirectURL *url.URL, idpID string,
+	ctx context.Context, queryParams url.Values, redirectURL *url.URL, idpID string,
 ) (string, error) {
 	signinURL := s.authenticateURL.ResolveReference(&url.URL{
 		Path: "/.pomerium/sign_in",
@@ -327,6 +331,7 @@ func (s *Stateful) AuthenticateSignInURL(
 	}
 	queryParams.Set(urlutil.QueryRedirectURI, redirectURL.String())
 	queryParams.Set(urlutil.QueryIdentityProviderID, idpID)
+	otel.GetTextMapPropagator().Inject(ctx, trace.PomeriumURLQueryCarrier(queryParams))
 	signinURL.RawQuery = queryParams.Encode()
 	redirectTo := urlutil.NewSignedURL(s.sharedKey, signinURL).String()
 

internal/authenticateflow/stateful_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/go-jose/go-jose/v3/jwt"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace"
 	"go.uber.org/mock/gomock"
 	"golang.org/x/oauth2"
 	"google.golang.org/grpc"
@@ -69,7 +70,7 @@ func TestStatefulSignIn(t *testing.T) {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
 			sessionStore := &mstore.Store{SaveError: tt.saveError}
-			flow, err := NewStateful(context.Background(), &config.Config{Options: opts}, sessionStore)
+			flow, err := NewStateful(context.Background(), trace.NewNoopTracerProvider(), &config.Config{Options: opts}, sessionStore)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -123,7 +124,7 @@ func TestStatefulAuthenticateSignInURL(t *testing.T) {
 	opts.AuthenticateURLString = "https://authenticate.example.com"
 	key := cryptutil.NewKey()
 	opts.SharedKey = base64.StdEncoding.EncodeToString(key)
-	flow, err := NewStateful(context.Background(), &config.Config{Options: opts}, nil)
+	flow, err := NewStateful(context.Background(), trace.NewNoopTracerProvider(), &config.Config{Options: opts}, nil)
 	require.NoError(t, err)
 
 	t.Run("NilQueryParams", func(t *testing.T) {
@@ -238,7 +239,7 @@ func TestStatefulCallback(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			flow, err := NewStateful(context.Background(), &config.Config{Options: opts}, tt.sessionStore)
+			flow, err := NewStateful(context.Background(), trace.NewNoopTracerProvider(), &config.Config{Options: opts}, tt.sessionStore)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -289,7 +290,7 @@ func TestStatefulCallback(t *testing.T) {
 
 func TestStatefulRevokeSession(t *testing.T) {
 	opts := config.NewDefaultOptions()
-	flow, err := NewStateful(context.Background(), &config.Config{Options: opts}, nil)
+	flow, err := NewStateful(context.Background(), trace.NewNoopTracerProvider(), &config.Config{Options: opts}, nil)
 	require.NoError(t, err)
 
 	ctrl := gomock.NewController(t)
@@ -367,7 +368,7 @@ func TestPersistSession(t *testing.T) {
 
 	opts := config.NewDefaultOptions()
 	opts.CookieExpire = 4 * time.Hour
-	flow, err := NewStateful(context.Background(), &config.Config{Options: opts}, nil)
+	flow, err := NewStateful(context.Background(), trace.NewNoopTracerProvider(), &config.Config{Options: opts}, nil)
 	require.NoError(t, err)
 
 	ctrl := gomock.NewController(t)

internal/authenticateflow/stateless.go

@@ -29,6 +29,7 @@ import (
 	"github.com/pomerium/pomerium/pkg/grpc/user"
 	"github.com/pomerium/pomerium/pkg/hpke"
 	"github.com/pomerium/pomerium/pkg/identity"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
 // Stateless implements the stateless authentication flow. In this flow, the
@@ -56,18 +57,21 @@ type Stateless struct {
 
 	dataBrokerClient databroker.DataBrokerServiceClient
 
-	getIdentityProvider func(options *config.Options, idpID string) (identity.Authenticator, error)
+	getIdentityProvider func(ctx context.Context, tracerProvider oteltrace.TracerProvider, options *config.Options, idpID string) (identity.Authenticator, error)
 	profileTrimFn       func(*identitypb.Profile)
 	authEventFn         events.AuthEventFn
+
+	tracerProvider oteltrace.TracerProvider
 }
 
 // NewStateless initializes the authentication flow for the given
 // configuration, session store, and additional options.
 func NewStateless(
 	ctx context.Context,
+	tracerProvider oteltrace.TracerProvider,
 	cfg *config.Config,
 	sessionStore sessions.SessionStore,
-	getIdentityProvider func(options *config.Options, idpID string) (identity.Authenticator, error),
+	getIdentityProvider func(ctx context.Context, tracerProvider oteltrace.TracerProvider, options *config.Options, idpID string) (identity.Authenticator, error),
 	profileTrimFn func(*identitypb.Profile),
 	authEventFn events.AuthEventFn,
 ) (*Stateless, error) {
@@ -77,6 +81,7 @@ func NewStateless(
 		getIdentityProvider: getIdentityProvider,
 		profileTrimFn:       profileTrimFn,
 		authEventFn:         authEventFn,
+		tracerProvider:      tracerProvider,
 	}
 
 	var err error
@@ -132,7 +137,7 @@ func NewStateless(
 		return nil, fmt.Errorf("authorize: get authenticate JWKS key fetcher: %w", err)
 	}
 
-	dataBrokerConn, err := outboundGRPCConnection.Get(ctx, &grpc.OutboundOptions{
+	dataBrokerConn, err := outboundGRPCConnection.Get(ctx, tracerProvider, &grpc.OutboundOptions{
 		OutboundPort:   cfg.OutboundPort,
 		InstallationID: cfg.Options.InstallationID,
 		ServiceName:    cfg.Options.Services,
@@ -154,7 +159,7 @@ func (s *Stateless) VerifySession(ctx context.Context, r *http.Request, _ *sessi
 		return fmt.Errorf("identity profile load error: %w", err)
 	}
 
-	authenticator, err := s.getIdentityProvider(s.options, profile.GetProviderId())
+	authenticator, err := s.getIdentityProvider(ctx, s.tracerProvider, s.options, profile.GetProviderId())
 	if err != nil {
 		return fmt.Errorf("couldn't get identity provider: %w", err)
 	}

internal/benchmarks/latency_bench_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/pomerium/pomerium/config"
 	"github.com/pomerium/pomerium/internal/testenv"
 	"github.com/pomerium/pomerium/internal/testenv/envutil"
 	"github.com/pomerium/pomerium/internal/testenv/scenarios"
@@ -48,7 +49,8 @@ func TestRequestLatency(t *testing.T) {
 	for i := range numRoutes {
 		routes[i] = up.Route().
 			From(env.SubdomainURL(fmt.Sprintf("from-%d", i))).
-			PPL(fmt.Sprintf(`{"allow":{"and":["email":{"is":"user%d@example.com"}]}}`, i))
+			Policy(func(p *config.Policy) { p.AllowPublicUnauthenticatedAccess = true })
+		// PPL(fmt.Sprintf(`{"allow":{"and":["email":{"is":"user%d@example.com"}]}}`, i))
 	}
 	env.AddUpstream(up)
 

internal/controlplane/events.go

@@ -71,7 +71,7 @@ func (srv *Server) getDataBrokerClient(ctx context.Context) (databrokerpb.DataBr
 		return nil, err
 	}
 
-	cc, err := outboundGRPCConnection.Get(ctx, &grpc.OutboundOptions{
+	cc, err := outboundGRPCConnection.Get(ctx, srv.tracerProvider, &grpc.OutboundOptions{
 		OutboundPort:   cfg.OutboundPort,
 		InstallationID: cfg.Options.InstallationID,
 		ServiceName:    cfg.Options.Services,

internal/controlplane/http.go

@@ -2,6 +2,7 @@
 package controlplane
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"time"
@@ -9,18 +10,21 @@ import (
 	"github.com/CAFxX/httpcompression"
 	"github.com/gorilla/mux"
 	"github.com/rs/zerolog"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 
 	"github.com/pomerium/pomerium/config"
 	"github.com/pomerium/pomerium/internal/handlers"
 	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/internal/middleware"
 	"github.com/pomerium/pomerium/internal/telemetry"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/urlutil"
 	hpke_handlers "github.com/pomerium/pomerium/pkg/hpke/handlers"
 	"github.com/pomerium/pomerium/pkg/telemetry/requestid"
 )
 
-func (srv *Server) addHTTPMiddleware(root *mux.Router, logger *zerolog.Logger, _ *config.Config) {
+func (srv *Server) addHTTPMiddleware(ctx context.Context, root *mux.Router, _ *config.Config) {
+	logger := log.Ctx(ctx)
 	compressor, err := httpcompression.DefaultAdapter()
 	if err != nil {
 		panic(err)
@@ -48,6 +52,7 @@ func (srv *Server) addHTTPMiddleware(root *mux.Router, logger *zerolog.Logger, _
 	root.Use(telemetry.HTTPStatsHandler(func() string {
 		return srv.currentConfig.Load().Options.InstallationID
 	}, srv.name))
+	root.Use(trace.NewHTTPMiddleware(otelhttp.WithTracerProvider(srv.tracerProvider)))
 }
 
 func (srv *Server) mountCommonEndpoints(root *mux.Router, cfg *config.Config) error {

internal/controlplane/server.go

@@ -11,6 +11,8 @@ import (
 	envoy_service_discovery_v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
 	"github.com/gorilla/mux"
 	"github.com/rs/zerolog"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
+	coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/health/grpc_health_v1"
@@ -25,7 +27,6 @@ import (
 	"github.com/pomerium/pomerium/internal/events"
 	"github.com/pomerium/pomerium/internal/httputil/reproxy"
 	"github.com/pomerium/pomerium/internal/log"
-	"github.com/pomerium/pomerium/internal/telemetry"
 	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/urlutil"
 	"github.com/pomerium/pomerium/internal/version"
@@ -34,6 +35,7 @@ import (
 	"github.com/pomerium/pomerium/pkg/grpcutil"
 	"github.com/pomerium/pomerium/pkg/httputil"
 	"github.com/pomerium/pomerium/pkg/telemetry/requestid"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
 // A Service can be mounted on the control plane.
@@ -43,6 +45,7 @@ type Service interface {
 
 // A Server is the control-plane gRPC and HTTP servers.
 type Server struct {
+	coltracepb.UnimplementedTraceServiceServer
 	GRPCListener    net.Listener
 	GRPCServer      *grpc.Server
 	HTTPListener    net.Listener
@@ -66,6 +69,9 @@ type Server struct {
 	proxySvc        Service
 
 	haveSetCapacity map[string]bool
+
+	tracerProvider oteltrace.TracerProvider
+	tracer         oteltrace.Tracer
 }
 
 // NewServer creates a new Server. Listener ports are chosen by the OS.
@@ -76,7 +82,10 @@ func NewServer(
 	eventsMgr *events.Manager,
 	fileMgr *filemgr.Manager,
 ) (*Server, error) {
+	tracerProvider := trace.NewTracerProvider(ctx, "Control Plane")
 	srv := &Server{
+		tracerProvider:  tracerProvider,
+		tracer:          tracerProvider.Tracer(trace.PomeriumCoreTracer),
 		metricsMgr:      metricsMgr,
 		EventsMgr:       eventsMgr,
 		filemgr:         fileMgr,
@@ -105,7 +114,7 @@ func NewServer(
 		),
 	)
 	srv.GRPCServer = grpc.NewServer(
-		grpc.StatsHandler(telemetry.NewGRPCServerStatsHandler(cfg.Options.Services)),
+		grpc.StatsHandler(trace.NewStatsHandler(otelgrpc.NewServerHandler(otelgrpc.WithTracerProvider(tracerProvider)))),
 		grpc.ChainUnaryInterceptor(
 			log.UnaryServerInterceptor(log.Ctx(ctx)),
 			requestid.UnaryServerInterceptor(),
@@ -177,7 +186,7 @@ func NewServer(
 
 	srv.xdsmgr = xdsmgr.NewManager(res)
 	envoy_service_discovery_v3.RegisterAggregatedDiscoveryServiceServer(srv.GRPCServer, srv.xdsmgr)
-
+	coltracepb.RegisterTraceServiceServer(srv.GRPCServer, trace.ExporterServerFromContext(ctx))
 	return srv, nil
 }
 
@@ -241,7 +250,7 @@ func (srv *Server) Run(ctx context.Context) error {
 
 // OnConfigChange updates the pomerium config options.
 func (srv *Server) OnConfigChange(ctx context.Context, cfg *config.Config) error {
-	ctx, span := trace.StartSpan(ctx, "controlplane.Server.OnConfigChange")
+	ctx, span := srv.tracer.Start(ctx, "controlplane.Server.OnConfigChange")
 	defer span.End()
 
 	select {
@@ -265,7 +274,7 @@ func (srv *Server) EnableProxy(ctx context.Context, svc Service) error {
 }
 
 func (srv *Server) update(ctx context.Context, cfg *config.Config) error {
-	ctx, span := trace.StartSpan(ctx, "controlplane.Server.update")
+	ctx, span := srv.tracer.Start(ctx, "controlplane.Server.update")
 	defer span.End()
 
 	if err := srv.updateRouter(ctx, cfg); err != nil {
@@ -283,7 +292,7 @@ func (srv *Server) update(ctx context.Context, cfg *config.Config) error {
 
 func (srv *Server) updateRouter(ctx context.Context, cfg *config.Config) error {
 	httpRouter := mux.NewRouter()
-	srv.addHTTPMiddleware(httpRouter, log.Ctx(ctx), cfg)
+	srv.addHTTPMiddleware(ctx, httpRouter, cfg)
 	if err := srv.mountCommonEndpoints(httpRouter, cfg); err != nil {
 		return err
 	}

internal/controlplane/xds.go

@@ -9,7 +9,6 @@ import (
 	"golang.org/x/sync/errgroup"
 
 	"github.com/pomerium/pomerium/internal/log"
-	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/pkg/cryptutil"
 	"github.com/pomerium/pomerium/pkg/protoutil"
 )
@@ -21,7 +20,7 @@ const (
 )
 
 func (srv *Server) buildDiscoveryResources(ctx context.Context) (map[string][]*envoy_service_discovery_v3.Resource, error) {
-	ctx, span := trace.StartSpan(ctx, "controlplane.Server.buildDiscoveryResources")
+	ctx, span := srv.tracer.Start(ctx, "controlplane.Server.buildDiscoveryResources")
 	defer span.End()
 
 	cfg := srv.currentConfig.Load()

internal/databroker/config_source.go

@@ -22,6 +22,7 @@ import (
 	"github.com/pomerium/pomerium/pkg/grpc/databroker"
 	"github.com/pomerium/pomerium/pkg/grpcutil"
 	"github.com/pomerium/pomerium/pkg/health"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
 // ConfigSource provides a new Config source that decorates an underlying config with
@@ -35,6 +36,7 @@ type ConfigSource struct {
 	updaterHash            uint64
 	cancel                 func()
 	enableValidation       bool
+	tracerProvider         oteltrace.TracerProvider
 
 	config.ChangeDispatcher
 }
@@ -50,11 +52,13 @@ type EnableConfigValidation bool
 // NewConfigSource creates a new ConfigSource.
 func NewConfigSource(
 	ctx context.Context,
+	tracerProvider oteltrace.TracerProvider,
 	underlying config.Source,
 	enableValidation EnableConfigValidation,
 	listeners ...config.ChangeListener,
 ) *ConfigSource {
 	src := &ConfigSource{
+		tracerProvider:         tracerProvider,
 		enableValidation:       bool(enableValidation),
 		dbConfigs:              map[string]dbConfig{},
 		outboundGRPCConnection: new(grpc.CachedOutboundGRPClientConn),
@@ -85,7 +89,7 @@ func (src *ConfigSource) GetConfig() *config.Config {
 type firstTime bool
 
 func (src *ConfigSource) rebuild(ctx context.Context, firstTime firstTime) {
-	_, span := trace.StartSpan(ctx, "databroker.config_source.rebuild")
+	_, span := trace.Continue(ctx, "databroker.config_source.rebuild")
 	defer span.End()
 
 	now := time.Now()
@@ -259,7 +263,7 @@ func (src *ConfigSource) runUpdater(ctx context.Context, cfg *config.Config) {
 
 	ctx, src.cancel = context.WithCancel(ctx)
 
-	cc, err := src.outboundGRPCConnection.Get(ctx, connectionOptions)
+	cc, err := src.outboundGRPCConnection.Get(ctx, src.tracerProvider, connectionOptions)
 	if err != nil {
 		log.Ctx(ctx).Error().Err(err).Msg("databroker: failed to create gRPC connection to data broker")
 		return

internal/databroker/config_source_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace"
 	"google.golang.org/grpc"
 	"google.golang.org/protobuf/proto"
 
@@ -41,7 +42,7 @@ func TestConfigSource(t *testing.T) {
 	defer func() { _ = li.Close() }()
 	_, outboundPort, _ := net.SplitHostPort(li.Addr().String())
 
-	dataBrokerServer := New(ctx)
+	dataBrokerServer := New(ctx, trace.NewNoopTracerProvider())
 	srv := grpc.NewServer()
 	databroker.RegisterDataBrokerServiceServer(srv, dataBrokerServer)
 	go func() { _ = srv.Serve(li) }()
@@ -65,7 +66,7 @@ func TestConfigSource(t *testing.T) {
 		OutboundPort: outboundPort,
 		Options:      base,
 	})
-	src := NewConfigSource(ctx, baseSource, EnableConfigValidation(true), func(_ context.Context, cfg *config.Config) {
+	src := NewConfigSource(ctx, trace.NewNoopTracerProvider(), baseSource, EnableConfigValidation(true), func(_ context.Context, cfg *config.Config) {
 		cfgs <- cfg
 	})
 	cfgs <- src.GetConfig()

internal/databroker/registry.go

@@ -9,7 +9,6 @@ import (
 	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/internal/registry"
 	"github.com/pomerium/pomerium/internal/registry/inmemory"
-	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	registrypb "github.com/pomerium/pomerium/pkg/grpc/registry"
 	"github.com/pomerium/pomerium/pkg/storage"
 )
@@ -25,7 +24,7 @@ func (stream registryWatchServer) Context() context.Context {
 
 // Report calls the registry Report method.
 func (srv *Server) Report(ctx context.Context, req *registrypb.RegisterRequest) (*registrypb.RegisterResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.Report")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Report")
 	defer span.End()
 
 	r, err := srv.getRegistry(ctx)
@@ -38,7 +37,7 @@ func (srv *Server) Report(ctx context.Context, req *registrypb.RegisterRequest)
 
 // List calls the registry List method.
 func (srv *Server) List(ctx context.Context, req *registrypb.ListRequest) (*registrypb.ServiceList, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.List")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.List")
 	defer span.End()
 
 	r, err := srv.getRegistry(ctx)
@@ -52,7 +51,7 @@ func (srv *Server) List(ctx context.Context, req *registrypb.ListRequest) (*regi
 // Watch calls the registry Watch method.
 func (srv *Server) Watch(req *registrypb.ListRequest, stream registrypb.Registry_WatchServer) error {
 	ctx := stream.Context()
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.Watch")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Watch")
 	defer span.End()
 
 	r, err := srv.getRegistry(ctx)

internal/databroker/server.go

@@ -22,22 +22,28 @@ import (
 	"github.com/pomerium/pomerium/pkg/storage"
 	"github.com/pomerium/pomerium/pkg/storage/inmemory"
 	"github.com/pomerium/pomerium/pkg/storage/postgres"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
 // Server implements the databroker service using an in memory database.
 type Server struct {
 	cfg *serverConfig
 
-	mu         sync.RWMutex
+	mu             sync.RWMutex
-	backend    storage.Backend
+	backend        storage.Backend
-	backendCtx context.Context
+	backendCtx     context.Context
-	registry   registry.Interface
+	registry       registry.Interface
+	tracerProvider oteltrace.TracerProvider
+	tracer         oteltrace.Tracer
 }
 
 // New creates a new server.
-func New(ctx context.Context, options ...ServerOption) *Server {
+func New(ctx context.Context, tracerProvider oteltrace.TracerProvider, options ...ServerOption) *Server {
+	tracer := tracerProvider.Tracer(trace.PomeriumCoreTracer)
 	srv := &Server{
-		backendCtx: ctx,
+		backendCtx:     ctx,
+		tracerProvider: tracerProvider,
+		tracer:         tracer,
 	}
 	srv.UpdateConfig(ctx, options...)
 	return srv
@@ -74,7 +80,7 @@ func (srv *Server) UpdateConfig(ctx context.Context, options ...ServerOption) {
 
 // AcquireLease acquires a lease.
 func (srv *Server) AcquireLease(ctx context.Context, req *databroker.AcquireLeaseRequest) (*databroker.AcquireLeaseResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.AcquireLease")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.AcquireLease")
 	defer span.End()
 	log.Ctx(ctx).Debug().
 		Str("name", req.GetName()).
@@ -101,7 +107,7 @@ func (srv *Server) AcquireLease(ctx context.Context, req *databroker.AcquireLeas
 
 // Get gets a record from the in-memory list.
 func (srv *Server) Get(ctx context.Context, req *databroker.GetRequest) (*databroker.GetResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.Get")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Get")
 	defer span.End()
 	log.Ctx(ctx).Debug().
 		Str("type", req.GetType()).
@@ -128,7 +134,7 @@ func (srv *Server) Get(ctx context.Context, req *databroker.GetRequest) (*databr
 
 // ListTypes lists all the record types.
 func (srv *Server) ListTypes(ctx context.Context, _ *emptypb.Empty) (*databroker.ListTypesResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.ListTypes")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.ListTypes")
 	defer span.End()
 	log.Ctx(ctx).Debug().Msg("list types")
 
@@ -145,7 +151,7 @@ func (srv *Server) ListTypes(ctx context.Context, _ *emptypb.Empty) (*databroker
 
 // Query queries for records.
 func (srv *Server) Query(ctx context.Context, req *databroker.QueryRequest) (*databroker.QueryResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.Query")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Query")
 	defer span.End()
 	log.Ctx(ctx).Debug().
 		Str("type", req.GetType()).
@@ -198,7 +204,7 @@ func (srv *Server) Query(ctx context.Context, req *databroker.QueryRequest) (*da
 
 // Put updates an existing record or adds a new one.
 func (srv *Server) Put(ctx context.Context, req *databroker.PutRequest) (*databroker.PutResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.Put")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Put")
 	defer span.End()
 
 	records := req.GetRecords()
@@ -237,7 +243,7 @@ func (srv *Server) Put(ctx context.Context, req *databroker.PutRequest) (*databr
 
 // Patch updates specific fields of an existing record.
 func (srv *Server) Patch(ctx context.Context, req *databroker.PatchRequest) (*databroker.PatchResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.Patch")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Patch")
 	defer span.End()
 
 	records := req.GetRecords()
@@ -276,7 +282,7 @@ func (srv *Server) Patch(ctx context.Context, req *databroker.PatchRequest) (*da
 
 // ReleaseLease releases a lease.
 func (srv *Server) ReleaseLease(ctx context.Context, req *databroker.ReleaseLeaseRequest) (*emptypb.Empty, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.ReleaseLease")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.ReleaseLease")
 	defer span.End()
 	log.Ctx(ctx).Debug().
 		Str("name", req.GetName()).
@@ -298,7 +304,7 @@ func (srv *Server) ReleaseLease(ctx context.Context, req *databroker.ReleaseLeas
 
 // RenewLease releases a lease.
 func (srv *Server) RenewLease(ctx context.Context, req *databroker.RenewLeaseRequest) (*emptypb.Empty, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.RenewLease")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.RenewLease")
 	defer span.End()
 	log.Ctx(ctx).Debug().
 		Str("name", req.GetName()).
@@ -323,7 +329,7 @@ func (srv *Server) RenewLease(ctx context.Context, req *databroker.RenewLeaseReq
 
 // SetOptions sets options for a type in the databroker.
 func (srv *Server) SetOptions(ctx context.Context, req *databroker.SetOptionsRequest) (*databroker.SetOptionsResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.SetOptions")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.SetOptions")
 	defer span.End()
 
 	backend, err := srv.getBackend(ctx)
@@ -346,7 +352,7 @@ func (srv *Server) SetOptions(ctx context.Context, req *databroker.SetOptionsReq
 // Sync streams updates for the given record type.
 func (srv *Server) Sync(req *databroker.SyncRequest, stream databroker.DataBrokerService_SyncServer) error {
 	ctx := stream.Context()
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.Sync")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Sync")
 	defer span.End()
 
 	ctx, cancel := context.WithCancel(ctx)
@@ -384,7 +390,7 @@ func (srv *Server) Sync(req *databroker.SyncRequest, stream databroker.DataBroke
 // SyncLatest returns the latest value of every record in the databroker as a stream of records.
 func (srv *Server) SyncLatest(req *databroker.SyncLatestRequest, stream databroker.DataBrokerService_SyncLatestServer) error {
 	ctx := stream.Context()
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.SyncLatest")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.SyncLatest")
 	defer span.End()
 
 	ctx, cancel := context.WithCancel(ctx)

internal/middleware/middleware.go

@@ -14,7 +14,7 @@ import (
 func SetHeaders(headers map[string]string) func(next http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			ctx, span := trace.StartSpan(r.Context(), "middleware.SetHeaders")
+			ctx, span := trace.Continue(r.Context(), "middleware.SetHeaders")
 			defer span.End()
 			for key, val := range headers {
 				w.Header().Set(key, val)
@@ -29,7 +29,7 @@ func SetHeaders(headers map[string]string) func(next http.Handler) http.Handler
 func ValidateSignature(sharedKey []byte) func(next http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return httputil.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
-			ctx, span := trace.StartSpan(r.Context(), "middleware.ValidateSignature")
+			ctx, span := trace.Continue(r.Context(), "middleware.ValidateSignature")
 			defer span.End()
 			if err := ValidateRequestURL(r, sharedKey); err != nil {
 				return httputil.NewError(http.StatusBadRequest, err)

internal/registry/reporter.go

@@ -15,18 +15,21 @@ import (
 	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/pkg/grpc"
 	pb "github.com/pomerium/pomerium/pkg/grpc/registry"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
 // Reporter periodically submits a list of services available on this instance to the service registry
 type Reporter struct {
 	cancel                 func()
 	outboundGRPCConnection *grpc.CachedOutboundGRPClientConn
+	tracerProvider         oteltrace.TracerProvider
 }
 
 // NewReporter creates a new Reporter.
-func NewReporter() *Reporter {
+func NewReporter(tracerProvider oteltrace.TracerProvider) *Reporter {
 	return &Reporter{
 		outboundGRPCConnection: new(grpc.CachedOutboundGRPClientConn),
+		tracerProvider:         tracerProvider,
 	}
 }
 
@@ -47,7 +50,7 @@ func (r *Reporter) OnConfigChange(ctx context.Context, cfg *config.Config) {
 		return
 	}
 
-	registryConn, err := r.outboundGRPCConnection.Get(ctx, &grpc.OutboundOptions{
+	registryConn, err := r.outboundGRPCConnection.Get(ctx, r.tracerProvider, &grpc.OutboundOptions{
 		OutboundPort:   cfg.OutboundPort,
 		InstallationID: cfg.Options.InstallationID,
 		ServiceName:    cfg.Options.Services,

internal/sessions/state.go

@@ -36,6 +36,9 @@ type State struct {
 
 	// IdentityProviderID is the identity provider for the session.
 	IdentityProviderID string `json:"idp_id,omitempty"`
+
+	Traceparent string `json:"traceparent,omitempty"`
+	Tracestate  string `json:"tracestate,omitempty"`
 }
 
 // NewState creates a new State.

internal/sessions/tracing.go

@@ -0,0 +1,35 @@
+package sessions
+
+import "go.opentelemetry.io/otel/propagation"
+
+type SessionStateCarrier struct {
+	*State
+}
+
+// Get implements propagation.TextMapCarrier.
+func (s SessionStateCarrier) Get(key string) string {
+	switch key {
+	case "pomerium_traceparent":
+		return s.Traceparent
+	case "pomerium_tracestate":
+		return s.Tracestate
+	}
+	return ""
+}
+
+// Set implements propagation.TextMapCarrier.
+func (s SessionStateCarrier) Set(key string, value string) {
+	switch key {
+	case "pomerium_traceparent":
+		s.Traceparent = value
+	case "pomerium_tracestate":
+		s.Tracestate = value
+	}
+}
+
+// Keys implements propagation.TextMapCarrier.
+func (s SessionStateCarrier) Keys() []string {
+	return nil
+}
+
+var _ propagation.TextMapCarrier = SessionStateCarrier{}

internal/telemetry/grpc.go

@@ -1,100 +0,0 @@
-package telemetry
-
-import (
-	"context"
-	"strings"
-
-	"go.opencensus.io/plugin/ocgrpc"
-	"go.opencensus.io/plugin/ochttp/propagation/b3"
-	"go.opencensus.io/trace"
-	"go.opencensus.io/trace/propagation"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/metadata"
-	grpcstats "google.golang.org/grpc/stats"
-
-	"github.com/pomerium/pomerium/internal/telemetry/metrics"
-)
-
-const (
-	grpcTraceBinHeader = "grpc-trace-bin"
-	b3TraceIDHeader    = "x-b3-traceid"
-	b3SpanIDHeader     = "x-b3-spanid"
-)
-
-type tagRPCHandler interface {
-	TagRPC(context.Context, *grpcstats.RPCTagInfo) context.Context
-}
-
-// GRPCServerStatsHandler provides a grpc stats.Handler for metrics and tracing for a pomerium service
-type GRPCServerStatsHandler struct {
-	service        string
-	metricsHandler tagRPCHandler
-	grpcstats.Handler
-}
-
-// TagRPC implements grpc.stats.Handler and adds metrics and tracing metadata to the context of a given RPC
-func (h *GRPCServerStatsHandler) TagRPC(ctx context.Context, tagInfo *grpcstats.RPCTagInfo) context.Context {
-	// the opencensus trace handler only supports grpc-trace-bin, so we use that code and support b3 too
-
-	md, _ := metadata.FromIncomingContext(ctx)
-	name := strings.TrimPrefix(tagInfo.FullMethodName, "/")
-	name = strings.Replace(name, "/", ".", -1)
-
-	var parent trace.SpanContext
-	hasParent := false
-	if traceBin := md[grpcTraceBinHeader]; len(traceBin) > 0 {
-		parent, hasParent = propagation.FromBinary([]byte(traceBin[0]))
-	}
-
-	if hdr := md[b3TraceIDHeader]; len(hdr) > 0 {
-		if tid, ok := b3.ParseTraceID(hdr[0]); ok {
-			parent.TraceID = tid
-			hasParent = true
-		}
-	}
-	if hdr := md[b3SpanIDHeader]; len(hdr) > 0 {
-		if sid, ok := b3.ParseSpanID(hdr[0]); ok {
-			parent.SpanID = sid
-			hasParent = true
-		}
-	}
-
-	if hasParent {
-		ctx, _ = trace.StartSpanWithRemoteParent(ctx, name, parent,
-			trace.WithSpanKind(trace.SpanKindServer))
-	} else {
-		ctx, _ = trace.StartSpan(ctx, name,
-			trace.WithSpanKind(trace.SpanKindServer))
-	}
-
-	// ocgrpc's TagRPC must be called to attach the context rpcDataKey correctly
-	// https://github.com/census-instrumentation/opencensus-go/blob/bf52d9df8bb2d44cad934587ab946794456cf3c8/plugin/ocgrpc/server_stats_handler.go#L45
-	metricCtx := h.metricsHandler.TagRPC(h.Handler.TagRPC(ctx, tagInfo), tagInfo)
-	return metricCtx
-}
-
-// NewGRPCServerStatsHandler creates a new GRPCServerStatsHandler for a pomerium service
-func NewGRPCServerStatsHandler(service string) grpcstats.Handler {
-	return &GRPCServerStatsHandler{
-		service:        ServiceName(service),
-		Handler:        &ocgrpc.ServerHandler{},
-		metricsHandler: metrics.NewGRPCServerMetricsHandler(ServiceName(service)),
-	}
-}
-
-// GRPCClientStatsHandler provides DialOptions for grpc clients to instrument network calls with
-// both metrics and tracing
-type GRPCClientStatsHandler struct {
-	UnaryInterceptor grpc.UnaryClientInterceptor
-	// TODO: we should have a streaming interceptor too
-	grpcstats.Handler
-}
-
-// NewGRPCClientStatsHandler returns a new GRPCClientStatsHandler used to create
-// telemetry related client DialOptions
-func NewGRPCClientStatsHandler(service string) *GRPCClientStatsHandler {
-	return &GRPCClientStatsHandler{
-		Handler:          &ocgrpc.ClientHandler{},
-		UnaryInterceptor: metrics.GRPCClientInterceptor(ServiceName(service)),
-	}
-}

internal/telemetry/grpc_test.go

@@ -1,47 +0,0 @@
-package telemetry
-
-import (
-	"context"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"go.opencensus.io/plugin/ocgrpc"
-	"go.opencensus.io/plugin/ochttp/propagation/b3"
-	"go.opencensus.io/trace"
-	"google.golang.org/grpc/metadata"
-	grpcstats "google.golang.org/grpc/stats"
-)
-
-type mockTagHandler struct {
-	called bool
-}
-
-type mockCtxTag string
-
-func (m *mockTagHandler) TagRPC(ctx context.Context, _ *grpcstats.RPCTagInfo) context.Context {
-	m.called = true
-	return context.WithValue(ctx, mockCtxTag("added"), "true")
-}
-
-func Test_GRPCServerStatsHandler(t *testing.T) {
-	metricsHandler := &mockTagHandler{}
-	h := &GRPCServerStatsHandler{
-		metricsHandler: metricsHandler,
-		Handler:        &ocgrpc.ServerHandler{},
-	}
-
-	ctx := context.WithValue(context.Background(), mockCtxTag("original"), "true")
-	ctx = metadata.NewIncomingContext(ctx, metadata.MD{
-		b3TraceIDHeader: {"9de3f6756f315fef"},
-		b3SpanIDHeader:  {"b4f83d3096b6bf9c"},
-	})
-	ctx = h.TagRPC(ctx, &grpcstats.RPCTagInfo{})
-
-	assert.True(t, metricsHandler.called)
-	assert.Equal(t, ctx.Value(mockCtxTag("added")), "true")
-	assert.Equal(t, ctx.Value(mockCtxTag("original")), "true")
-
-	span := trace.FromContext(ctx)
-	expectedTraceID, _ := b3.ParseTraceID("9de3f6756f315fef")
-	assert.Equal(t, expectedTraceID, span.SpanContext().TraceID)
-}

internal/telemetry/trace/carriers.go

@@ -0,0 +1,27 @@
+package trace
+
+import (
+	"net/url"
+
+	"go.opentelemetry.io/otel/propagation"
+)
+
+type PomeriumURLQueryCarrier url.Values
+
+// Get implements propagation.TextMapCarrier.
+func (q PomeriumURLQueryCarrier) Get(key string) string {
+	return url.Values(q).Get("pomerium_" + key)
+}
+
+// Set implements propagation.TextMapCarrier.
+func (q PomeriumURLQueryCarrier) Set(key string, value string) {
+	url.Values(q).Set("pomerium_"+key, value)
+}
+
+// Keys implements propagation.TextMapCarrier.
+func (q PomeriumURLQueryCarrier) Keys() []string {
+	// this function is never called in otel
+	return nil
+}
+
+var _ propagation.TextMapCarrier = PomeriumURLQueryCarrier{}

internal/telemetry/trace/datadog.go

@@ -1,34 +0,0 @@
-package trace
-
-import (
-	datadog "github.com/DataDog/opencensus-go-exporter-datadog"
-	octrace "go.opencensus.io/trace"
-)
-
-type datadogProvider struct {
-	exporter *datadog.Exporter
-}
-
-func (provider *datadogProvider) Register(opts *TracingOptions) error {
-	dOpts := datadog.Options{
-		Service:   opts.Service,
-		TraceAddr: opts.DatadogAddress,
-	}
-	dex, err := datadog.NewExporter(dOpts)
-	if err != nil {
-		return err
-	}
-	octrace.RegisterExporter(dex)
-	provider.exporter = dex
-	return nil
-}
-
-func (provider *datadogProvider) Unregister() error {
-	if provider.exporter == nil {
-		return nil
-	}
-	octrace.UnregisterExporter(provider.exporter)
-	provider.exporter.Stop()
-	provider.exporter = nil
-	return nil
-}

internal/telemetry/trace/jaeger.go

@@ -1,37 +0,0 @@
-package trace
-
-import (
-	"contrib.go.opencensus.io/exporter/jaeger"
-	octrace "go.opencensus.io/trace"
-)
-
-type jaegerProvider struct {
-	exporter *jaeger.Exporter
-}
-
-func (provider *jaegerProvider) Register(opts *TracingOptions) error {
-	jOpts := jaeger.Options{
-		ServiceName:   opts.Service,
-		AgentEndpoint: opts.JaegerAgentEndpoint,
-	}
-	if opts.JaegerCollectorEndpoint != nil {
-		jOpts.CollectorEndpoint = opts.JaegerCollectorEndpoint.String()
-	}
-	jex, err := jaeger.NewExporter(jOpts)
-	if err != nil {
-		return err
-	}
-	octrace.RegisterExporter(jex)
-	provider.exporter = jex
-	return nil
-}
-
-func (provider *jaegerProvider) Unregister() error {
-	if provider.exporter == nil {
-		return nil
-	}
-	octrace.UnregisterExporter(provider.exporter)
-	provider.exporter.Flush()
-	provider.exporter = nil
-	return nil
-}

internal/telemetry/trace/middleware.go

@@ -0,0 +1,43 @@
+package trace
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/propagation"
+)
+
+func NewHTTPMiddleware(opts ...otelhttp.Option) func(http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			routeStr := ""
+			route := mux.CurrentRoute(r)
+			if route != nil {
+				var err error
+				routeStr, err = route.GetPathTemplate()
+				if err != nil {
+					routeStr, err = route.GetPathRegexp()
+					if err != nil {
+						routeStr = ""
+					}
+				}
+			}
+			traceparent := r.Header.Get("Traceparent")
+			if traceparent != "" {
+				xPomeriumTraceparent := r.Header.Get("X-Pomerium-Traceparent")
+				if xPomeriumTraceparent != "" {
+					sc, err := ParseTraceparent(xPomeriumTraceparent)
+					if err == nil {
+						r.Header.Set("Traceparent", ReplaceTraceID(traceparent, sc.TraceID()))
+						ctx := otel.GetTextMapPropagator().Extract(r.Context(), propagation.HeaderCarrier(r.Header))
+						r = r.WithContext(ctx)
+					}
+				}
+			}
+			otelhttp.NewHandler(next, fmt.Sprintf("Server: %s %s", r.Method, routeStr), opts...).ServeHTTP(w, r)
+		})
+	}
+}

internal/telemetry/trace/server.go

@@ -0,0 +1,341 @@
+package trace
+
+import (
+	"context"
+	"encoding/base64"
+	"net"
+	"net/url"
+	"strings"
+	"sync"
+
+	coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
+	commonv1 "go.opentelemetry.io/proto/otlp/common/v1"
+	resourcev1 "go.opentelemetry.io/proto/otlp/resource/v1"
+	tracev1 "go.opentelemetry.io/proto/otlp/trace/v1"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/test/bufconn"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/pomerium/pomerium/internal/hashutil"
+	"github.com/pomerium/pomerium/internal/log"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	oteltrace "go.opentelemetry.io/otel/trace"
+)
+
+type PendingSpans struct {
+	scope       *commonv1.InstrumentationScope
+	scopeSchema string
+	spans       []*tracev1.Span
+}
+
+func (ps *PendingSpans) Insert(span *tracev1.Span) {
+	ps.spans = append(ps.spans, span)
+}
+
+func NewPendingSpans(scope *commonv1.InstrumentationScope, scopeSchema string) *PendingSpans {
+	return &PendingSpans{
+		scope:       scope,
+		scopeSchema: scopeSchema,
+	}
+}
+
+type PendingScopes struct {
+	resource     *ResourceInfo
+	spansByScope map[string]*PendingSpans
+}
+
+func (ptr *PendingScopes) Insert(scope *commonv1.InstrumentationScope, scopeSchema string, span *tracev1.Span) {
+	var spans *PendingSpans
+	if sp, ok := ptr.spansByScope[scope.GetName()]; ok {
+		spans = sp
+	} else {
+		spans = NewPendingSpans(scope, scopeSchema)
+		ptr.spansByScope[scope.GetName()] = spans
+	}
+	spans.Insert(span)
+}
+
+func (ptr *PendingScopes) Delete(scope *commonv1.InstrumentationScope) (cascade bool) {
+	delete(ptr.spansByScope, scope.GetName())
+	return len(ptr.spansByScope) == 0
+}
+
+func (ptr *PendingScopes) AsScopeSpansList(rewriteTraceId oteltrace.TraceID) []*tracev1.ScopeSpans {
+	out := make([]*tracev1.ScopeSpans, 0, len(ptr.spansByScope))
+	for _, spans := range ptr.spansByScope {
+		for _, span := range spans.spans {
+			span.TraceId = rewriteTraceId[:]
+		}
+		scopeSpans := &tracev1.ScopeSpans{
+			Scope:     spans.scope,
+			SchemaUrl: spans.scopeSchema,
+			Spans:     spans.spans,
+		}
+		out = append(out, scopeSpans)
+	}
+	return out
+}
+
+func NewPendingScopes(resource *ResourceInfo) *PendingScopes {
+	return &PendingScopes{
+		resource:     resource,
+		spansByScope: make(map[string]*PendingSpans),
+	}
+}
+
+type PendingResources struct {
+	scopesByResourceID map[string]*PendingScopes
+}
+
+func (ptr *PendingResources) Insert(resource *ResourceInfo, scope *commonv1.InstrumentationScope, scopeSchema string, span *tracev1.Span) {
+	resourceEq := resource.ID()
+	var scopes *PendingScopes
+	if sc, ok := ptr.scopesByResourceID[resourceEq]; ok {
+		scopes = sc
+	} else {
+		scopes = NewPendingScopes(resource)
+		ptr.scopesByResourceID[resourceEq] = scopes
+	}
+	scopes.Insert(scope, scopeSchema, span)
+}
+
+func (ptr *PendingResources) Delete(resource *ResourceInfo, scope *commonv1.InstrumentationScope) (cascade bool) {
+	resourceEq := resource.ID()
+	if ptr.scopesByResourceID[resourceEq].Delete(scope) {
+		delete(ptr.scopesByResourceID, resourceEq)
+	}
+	return len(ptr.scopesByResourceID) == 0
+}
+
+func (ptr *PendingResources) AsResourceSpans(rewriteTraceId oteltrace.TraceID) []*tracev1.ResourceSpans {
+	out := make([]*tracev1.ResourceSpans, 0, len(ptr.scopesByResourceID))
+	for _, scopes := range ptr.scopesByResourceID {
+		resourceSpans := &tracev1.ResourceSpans{
+			Resource:   scopes.resource.Resource,
+			ScopeSpans: scopes.AsScopeSpansList(rewriteTraceId),
+			SchemaUrl:  scopes.resource.Schema,
+		}
+		out = append(out, resourceSpans)
+	}
+	return out
+}
+
+func NewPendingResources() *PendingResources {
+	return &PendingResources{scopesByResourceID: make(map[string]*PendingScopes)}
+}
+
+type ResourceInfo struct {
+	Resource *resourcev1.Resource
+	Schema   string
+	ID       func() string
+}
+
+func newResourceInfo(resource *resourcev1.Resource, resourceSchema string) *ResourceInfo {
+	r := &ResourceInfo{
+		Resource: resource,
+		Schema:   resourceSchema,
+	}
+	r.ID = sync.OnceValue(r.computeID)
+	return r
+}
+
+func (r *ResourceInfo) computeID() string {
+	hash := hashutil.NewDigest()
+	tmp := resourcev1.Resource{
+		Attributes: r.Resource.Attributes,
+	}
+	bytes, _ := proto.Marshal(&tmp)
+	hash.WriteStringWithLen(r.Schema)
+	hash.WriteWithLen(bytes)
+	return base64.StdEncoding.EncodeToString(hash.Sum(nil))
+}
+
+type SpanExportQueue struct {
+	mu                        sync.Mutex
+	pendingResourcesByTraceId map[string]*PendingResources
+	knownTraceIdMappings      map[string]oteltrace.TraceID
+	uploadC                   chan []*tracev1.ResourceSpans
+}
+
+func NewSpanExportQueue(ctx context.Context, client otlptrace.Client) *SpanExportQueue {
+	q := &SpanExportQueue{
+		pendingResourcesByTraceId: make(map[string]*PendingResources),
+		knownTraceIdMappings:      make(map[string]oteltrace.TraceID),
+		uploadC:                   make(chan []*tracev1.ResourceSpans, 8),
+	}
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case resourceSpans := <-q.uploadC:
+				if err := client.UploadTraces(ctx, resourceSpans); err != nil {
+					log.Ctx(ctx).Err(err).Msg("error uploading traces")
+				}
+			}
+		}
+	}()
+	return q
+}
+
+type WithSchema[T any] struct {
+	Value  T
+	Schema string
+}
+
+func (q *SpanExportQueue) insertPendingSpanLocked(resource *ResourceInfo, scope *commonv1.InstrumentationScope, scopeSchema string, span *tracev1.Span) {
+	spanTraceIdHex := oteltrace.TraceID(span.TraceId).String()
+	var pendingTraceResources *PendingResources
+	if ptr, ok := q.pendingResourcesByTraceId[spanTraceIdHex]; ok {
+		pendingTraceResources = ptr
+	} else {
+		pendingTraceResources = NewPendingResources()
+		q.pendingResourcesByTraceId[spanTraceIdHex] = pendingTraceResources
+	}
+	pendingTraceResources.Insert(resource, scope, scopeSchema, span)
+}
+
+func (q *SpanExportQueue) resolveTraceIdMappingLocked(resource *ResourceInfo, scope *commonv1.InstrumentationScope, scopeSchema string, span *tracev1.Span, mapping oteltrace.TraceID) {
+	originalTraceIdHex := oteltrace.TraceID(span.TraceId).String()
+	q.insertPendingSpanLocked(resource, scope, scopeSchema, span)
+	q.knownTraceIdMappings[originalTraceIdHex] = mapping
+	toUpload := q.pendingResourcesByTraceId[originalTraceIdHex].AsResourceSpans(mapping)
+	if q.pendingResourcesByTraceId[originalTraceIdHex].Delete(resource, scope) {
+		delete(q.pendingResourcesByTraceId, originalTraceIdHex)
+	}
+	q.uploadC <- toUpload
+}
+
+func (q *SpanExportQueue) Enqueue(ctx context.Context, req *coltracepb.ExportTraceServiceRequest) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	var immediateUpload []*tracev1.ResourceSpans
+	for _, resource := range req.ResourceSpans {
+		resourceInfo := newResourceInfo(resource.Resource, resource.SchemaUrl)
+		knownResources := &tracev1.ResourceSpans{
+			Resource:  resource.Resource,
+			SchemaUrl: resource.SchemaUrl,
+		}
+		for _, scope := range resource.ScopeSpans {
+			var knownSpans []*tracev1.Span
+			for _, span := range scope.Spans {
+				spanTraceId := oteltrace.TraceID(span.TraceId)
+				spanTraceIdHex := oteltrace.TraceID(span.TraceId).String()
+
+				formatSpanName(span)
+				if len(span.ParentSpanId) == 0 {
+					// observed a new root span
+					var pomeriumTraceparent string
+					for _, attr := range span.Attributes {
+						if attr.Key == "pomerium.traceparent" {
+							pomeriumTraceparent = attr.GetValue().GetStringValue()
+							break
+						}
+					}
+					var targetTraceID oteltrace.TraceID
+
+					if pomeriumTraceparent == "" {
+						// no replacement id, map the trace to itself and release pending spans
+						targetTraceID = spanTraceId
+					} else {
+						// this root span has an alternate traceparent. permanently rewrite
+						// all spans of the old trace id to use the new trace id
+						tp, err := ParseTraceparent(pomeriumTraceparent)
+						if err != nil {
+							log.Ctx(ctx).Err(err).Msg("error processing trace")
+							continue
+						}
+						targetTraceID = tp.TraceID()
+					}
+
+					q.resolveTraceIdMappingLocked(resourceInfo, scope.Scope, scope.SchemaUrl, span, targetTraceID)
+				} else {
+					if rewrite, ok := q.knownTraceIdMappings[spanTraceIdHex]; ok {
+						span.TraceId = rewrite[:]
+						knownSpans = append(knownSpans, span)
+					} else {
+						q.insertPendingSpanLocked(resourceInfo, scope.Scope, scope.SchemaUrl, span)
+					}
+				}
+			}
+			if len(knownSpans) > 0 {
+				knownResources.ScopeSpans = append(knownResources.ScopeSpans, &tracev1.ScopeSpans{
+					Scope:     scope.Scope,
+					SchemaUrl: scope.SchemaUrl,
+					Spans:     knownSpans,
+				})
+			}
+		}
+		if len(knownResources.ScopeSpans) > 0 {
+			immediateUpload = append(immediateUpload, knownResources)
+		}
+	}
+	if len(immediateUpload) > 0 {
+		q.uploadC <- immediateUpload
+	}
+}
+
+func formatSpanName(span *tracev1.Span) {
+	hasPath := strings.Contains(span.GetName(), "${path}")
+	hasHost := strings.Contains(span.GetName(), "${host}")
+	hasMethod := strings.Contains(span.GetName(), "${method}")
+	if hasPath || hasHost || hasMethod {
+		var u *url.URL
+		var method string
+		for _, attr := range span.Attributes {
+			if attr.Key == "http.url" {
+				u, _ = url.Parse(attr.Value.GetStringValue())
+			}
+			if attr.Key == "http.method" {
+				method = attr.Value.GetStringValue()
+			}
+		}
+		if u != nil {
+			if hasPath {
+				span.Name = strings.ReplaceAll(span.Name, "${path}", u.Path)
+			}
+			if hasHost {
+				span.Name = strings.ReplaceAll(span.Name, "${host}", u.Host)
+			}
+			if hasMethod {
+				span.Name = strings.ReplaceAll(span.Name, "${method}", method)
+			}
+		}
+	}
+}
+
+// Export implements ptraceotlp.GRPCServer.
+func (srv *Server) Export(ctx context.Context, req *coltracepb.ExportTraceServiceRequest) (*coltracepb.ExportTraceServiceResponse, error) {
+	srv.spanExportQueue.Enqueue(ctx, req)
+	return &coltracepb.ExportTraceServiceResponse{}, nil
+}
+
+type Server struct {
+	coltracepb.UnimplementedTraceServiceServer
+	spanExportQueue *SpanExportQueue
+}
+
+func NewServer(ctx context.Context, client otlptrace.Client) *Server {
+	client.Start(ctx)
+	return &Server{
+		spanExportQueue: NewSpanExportQueue(ctx, client),
+	}
+}
+
+func (srv *Server) Start(ctx context.Context) otlptrace.Client {
+	lis := bufconn.Listen(4096)
+	gs := grpc.NewServer(grpc.Creds(insecure.NewCredentials()))
+	coltracepb.RegisterTraceServiceServer(gs, srv)
+	go gs.Serve(lis)
+	cc, err := grpc.NewClient("passthrough://ignore",
+		grpc.WithContextDialer(func(context.Context, string) (net.Conn, error) {
+			return lis.Dial()
+		}), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		panic(err)
+	}
+	return otlptracegrpc.NewClient(otlptracegrpc.WithGRPCConn(cc))
+}

internal/telemetry/trace/trace.go

@@ -2,87 +2,231 @@ package trace
 
 import (
 	"context"
+	"encoding/hex"
+	"errors"
 	"fmt"
-	"net/url"
+	"os"
+	"runtime"
+	"strconv"
+	"strings"
 
-	octrace "go.opencensus.io/trace"
+	"go.opentelemetry.io/otel"
-
+	"go.opentelemetry.io/otel/attribute"
-	"github.com/pomerium/pomerium/internal/log"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
+	"go.opentelemetry.io/otel/trace"
+	"go.opentelemetry.io/otel/trace/embedded"
+	coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/stats"
 )
 
-const (
+type (
-	// DatadogTracingProviderName is the name of the tracing provider Datadog.
+	clientKeyType         struct{}
-	DatadogTracingProviderName = "datadog"
+	exporterKeyType       struct{}
-	// JaegerTracingProviderName is the name of the tracing provider Jaeger.
+	tracerProviderKeyType struct{}
-	JaegerTracingProviderName = "jaeger"
+	serverKeyType         struct{}
-	// ZipkinTracingProviderName is the name of the tracing provider Zipkin.
-	ZipkinTracingProviderName = "zipkin"
 )
 
-// Provider is a trace provider.
+var (
-type Provider interface {
+	exporterKey       exporterKeyType
-	Register(options *TracingOptions) error
+	tracerProviderKey tracerProviderKeyType
-	Unregister() error
+	serverKey         serverKeyType
+)
+
+type shutdownFunc func(options ...trace.SpanEndOption)
+
+func init() {
+	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
+	otel.SetTracerProvider(panicTracerProvider{})
 }
 
-// TracingOptions contains the configurations settings for a http server.
+type panicTracerProvider struct {
-type TracingOptions struct {
+	embedded.TracerProvider
-	// Shared
-	Provider string
-	Service  string
-	Debug    bool
-
-	// Datadog
-	DatadogAddress string
-
-	// Jaeger
-
-	// CollectorEndpoint is the full url to the Jaeger HTTP Thrift collector.
-	// For example, http://localhost:14268/api/traces
-	JaegerCollectorEndpoint *url.URL
-	// AgentEndpoint instructs exporter to send spans to jaeger-agent at this address.
-	// For example, localhost:6831.
-	JaegerAgentEndpoint string
-
-	// Zipkin
-
-	// ZipkinEndpoint configures the zipkin collector URI
-	// Example: http://zipkin:9411/api/v2/spans
-	ZipkinEndpoint *url.URL
-
-	// SampleRate is percentage of requests which are sampled
-	SampleRate float64
 }
 
-// Enabled indicates whether tracing is enabled on a given TracingOptions
+// Tracer implements trace.TracerProvider.
-func (t *TracingOptions) Enabled() bool {
+func (w panicTracerProvider) Tracer(name string, options ...trace.TracerOption) trace.Tracer {
-	return t.Provider != ""
+	panic("global tracer used")
 }
 
-// GetProvider creates a new trace provider from TracingOptions.
+func NewContext(ctx context.Context) context.Context {
-func GetProvider(opts *TracingOptions) (Provider, error) {
+	var realClient otlptrace.Client
-	var provider Provider
+	if os.Getenv("OTEL_EXPORTER_OTLP_PROTOCOL") == "http/protobuf" {
-	switch opts.Provider {
+		realClient = otlptracehttp.NewClient()
-	case DatadogTracingProviderName:
+	} else {
-		provider = new(datadogProvider)
+		realClient = otlptracegrpc.NewClient()
-	case JaegerTracingProviderName:
-		provider = new(jaegerProvider)
-	case ZipkinTracingProviderName:
-		provider = new(zipkinProvider)
-	default:
-		return nil, fmt.Errorf("telemetry/trace: provider %s unknown", opts.Provider)
 	}
-	octrace.ApplyConfig(octrace.Config{DefaultSampler: octrace.ProbabilitySampler(opts.SampleRate)})
+	srv := NewServer(ctx, realClient)
-
+	localClient := srv.Start(ctx)
-	log.Debug().Interface("Opts", opts).Msg("telemetry/trace: provider created")
+	exp, err := otlptrace.New(ctx, localClient)
-	return provider, nil
+	if err != nil {
+		panic(err)
+	}
+	ctx = context.WithValue(ctx, exporterKey, exp)
+	ctx = context.WithValue(ctx, serverKey, srv)
+	return ctx
 }
 
+func NewTracerProvider(ctx context.Context, serviceName string) trace.TracerProvider {
+	_, file, line, _ := runtime.Caller(1)
+	exp := ctx.Value(exporterKey).(sdktrace.SpanExporter)
+	r, err := resource.Merge(
+		resource.Default(),
+		resource.NewWithAttributes(
+			semconv.SchemaURL,
+			semconv.ServiceName(serviceName),
+			attribute.String("provider.created_at", fmt.Sprintf("%s:%d", file, line)),
+		),
+	)
+	if err != nil {
+		panic(err)
+	}
+	return sdktrace.NewTracerProvider(
+		sdktrace.WithSpanProcessor(&stackTraceProcessor{}),
+		sdktrace.WithBatcher(exp),
+		sdktrace.WithResource(r),
+	)
+}
+
+type stackTraceProcessor struct{}
+
+// ForceFlush implements trace.SpanProcessor.
+func (s *stackTraceProcessor) ForceFlush(ctx context.Context) error {
+	return nil
+}
+
+// OnEnd implements trace.SpanProcessor.
+func (*stackTraceProcessor) OnEnd(s sdktrace.ReadOnlySpan) {
+}
+
+// OnStart implements trace.SpanProcessor.
+func (*stackTraceProcessor) OnStart(parent context.Context, s sdktrace.ReadWriteSpan) {
+	_, file, line, _ := runtime.Caller(2)
+	s.SetAttributes(attribute.String("caller", fmt.Sprintf("%s:%d", file, line)))
+}
+
+// Shutdown implements trace.SpanProcessor.
+func (s *stackTraceProcessor) Shutdown(ctx context.Context) error {
+	return nil
+}
+
+func ForceFlush(ctx context.Context) error {
+	if tp, ok := trace.SpanFromContext(ctx).TracerProvider().(interface {
+		ForceFlush(context.Context) error
+	}); ok {
+		return tp.ForceFlush(context.Background())
+	}
+	return nil
+}
+
+func Shutdown(ctx context.Context) error {
+	_ = ForceFlush(ctx)
+	exporter := ctx.Value(exporterKey).(sdktrace.SpanExporter)
+	return exporter.Shutdown(context.Background())
+}
+
+func ExporterServerFromContext(ctx context.Context) coltracepb.TraceServiceServer {
+	return ctx.Value(serverKey).(coltracepb.TraceServiceServer)
+}
+
+const PomeriumCoreTracer = "pomerium.io/core"
+
 // StartSpan starts a new child span of the current span in the context. If
 // there is no span in the context, creates a new trace and span.
 //
 // Returned context contains the newly created span. You can use it to
 // propagate the returned span in process.
-func StartSpan(ctx context.Context, name string, o ...octrace.StartOption) (context.Context, *octrace.Span) {
+func Continue(ctx context.Context, name string, o ...trace.SpanStartOption) (context.Context, trace.Span) {
-	return octrace.StartSpan(ctx, name, o...)
+	return trace.SpanFromContext(ctx).TracerProvider().Tracer(PomeriumCoreTracer).Start(ctx, name, o...)
+}
+
+func ParseTraceparent(traceparent string) (trace.SpanContext, error) {
+	parts := strings.Split(traceparent, "-")
+	if len(parts) != 4 {
+		return trace.SpanContext{}, errors.New("malformed traceparent")
+	}
+	traceId, err := trace.TraceIDFromHex(parts[1])
+	if err != nil {
+		return trace.SpanContext{}, err
+	}
+	spanId, err := trace.SpanIDFromHex(parts[2])
+	if err != nil {
+		return trace.SpanContext{}, err
+	}
+	traceFlags, err := strconv.ParseUint(parts[3], 6, 32)
+	if err != nil {
+		return trace.SpanContext{}, err
+	}
+	if len(traceId) != 16 || len(spanId) != 8 {
+		return trace.SpanContext{}, errors.New("malformed traceparent")
+	}
+	return trace.NewSpanContext(trace.SpanContextConfig{
+		TraceID:    traceId,
+		SpanID:     spanId,
+		TraceFlags: trace.TraceFlags(traceFlags),
+	}), nil
+}
+
+func ReplaceTraceID(traceparent string, newTraceID trace.TraceID) string {
+	parts := strings.Split(traceparent, "-")
+	if len(parts) != 4 {
+		return traceparent
+	}
+	parts[1] = hex.EncodeToString(newTraceID[:])
+	return strings.Join(parts, "-")
+}
+
+func NewStatsHandler(base stats.Handler) stats.Handler {
+	return &wrapperStatsHandler{
+		base: base,
+	}
+}
+
+type wrapperStatsHandler struct {
+	base stats.Handler
+}
+
+func (w *wrapperStatsHandler) wrapContext(ctx context.Context) context.Context {
+	md, ok := metadata.FromIncomingContext(ctx)
+	if !ok {
+		return ctx
+	}
+	traceparent := md.Get("traceparent")
+	xPomeriumTraceparent := md.Get("x-pomerium-traceparent")
+	if len(traceparent) > 0 && traceparent[0] != "" && len(xPomeriumTraceparent) > 0 && xPomeriumTraceparent[0] != "" {
+		newTracectx, err := ParseTraceparent(xPomeriumTraceparent[0])
+		if err != nil {
+			return ctx
+		}
+
+		md.Set("traceparent", ReplaceTraceID(traceparent[0], newTracectx.TraceID()))
+		return metadata.NewIncomingContext(ctx, md)
+	}
+	return ctx
+}
+
+// HandleConn implements stats.Handler.
+func (w *wrapperStatsHandler) HandleConn(ctx context.Context, stats stats.ConnStats) {
+	w.base.HandleConn(w.wrapContext(ctx), stats)
+}
+
+// HandleRPC implements stats.Handler.
+func (w *wrapperStatsHandler) HandleRPC(ctx context.Context, stats stats.RPCStats) {
+	w.base.HandleRPC(w.wrapContext(ctx), stats)
+}
+
+// TagConn implements stats.Handler.
+func (w *wrapperStatsHandler) TagConn(ctx context.Context, info *stats.ConnTagInfo) context.Context {
+	return w.base.TagConn(w.wrapContext(ctx), info)
+}
+
+// TagRPC implements stats.Handler.
+func (w *wrapperStatsHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context {
+	return w.base.TagRPC(w.wrapContext(ctx), info)
 }

internal/telemetry/trace/trace_test.go

@@ -1,27 +0,0 @@
-package trace
-
-import (
-	"net/url"
-	"testing"
-)
-
-func TestGetProvider(t *testing.T) {
-	tests := []struct {
-		name    string
-		opts    *TracingOptions
-		wantErr bool
-	}{
-		{"jaeger", &TracingOptions{JaegerAgentEndpoint: "localhost:6831", Service: "all", Provider: "jaeger"}, false},
-		{"jaeger with debug", &TracingOptions{JaegerAgentEndpoint: "localhost:6831", Service: "all", Provider: "jaeger", Debug: true}, false},
-		{"jaeger no endpoint", &TracingOptions{JaegerAgentEndpoint: "", Service: "all", Provider: "jaeger"}, false},
-		{"unknown provider", &TracingOptions{JaegerAgentEndpoint: "localhost:0", Service: "all", Provider: "Lucius Cornelius Sulla"}, true},
-		{"zipkin with debug", &TracingOptions{ZipkinEndpoint: &url.URL{Host: "localhost"}, Service: "all", Provider: "zipkin", Debug: true}, false},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if _, err := GetProvider(tt.opts); (err != nil) != tt.wantErr {
-				t.Errorf("RegisterTracing() error = %v, wantErr %v", err, tt.wantErr)
-			}
-		})
-	}
-}

internal/telemetry/trace/zipkin.go

@@ -1,49 +0,0 @@
-package trace
-
-import (
-	"fmt"
-	stdlog "log"
-
-	oczipkin "contrib.go.opencensus.io/exporter/zipkin"
-	"github.com/openzipkin/zipkin-go"
-	"github.com/openzipkin/zipkin-go/reporter"
-	zipkinHTTP "github.com/openzipkin/zipkin-go/reporter/http"
-	octrace "go.opencensus.io/trace"
-
-	"github.com/pomerium/pomerium/internal/log"
-)
-
-type zipkinProvider struct {
-	reporter reporter.Reporter
-	exporter *oczipkin.Exporter
-}
-
-func (provider *zipkinProvider) Register(opts *TracingOptions) error {
-	localEndpoint, err := zipkin.NewEndpoint(opts.Service, "")
-	if err != nil {
-		return fmt.Errorf("telemetry/trace: could not create local endpoint: %w", err)
-	}
-
-	logger := log.With().Str("service", "zipkin").Logger()
-	logWriter := &log.StdLogWrapper{Logger: &logger}
-	stdLogger := stdlog.New(logWriter, "", 0)
-
-	provider.reporter = zipkinHTTP.NewReporter(opts.ZipkinEndpoint.String(), zipkinHTTP.Logger(stdLogger))
-	provider.exporter = oczipkin.NewExporter(provider.reporter, localEndpoint)
-	octrace.RegisterExporter(provider.exporter)
-	return nil
-}
-
-func (provider *zipkinProvider) Unregister() error {
-	if provider.exporter != nil {
-		octrace.UnregisterExporter(provider.exporter)
-		provider.exporter = nil
-	}
-
-	var err error
-	if provider.reporter != nil {
-		err = provider.reporter.Close()
-		provider.reporter = nil
-	}
-	return err
-}

internal/testenv/environment.go

@@ -34,6 +34,7 @@ import (
 	"github.com/pomerium/pomerium/config"
 	"github.com/pomerium/pomerium/config/envoyconfig/filemgr"
 	"github.com/pomerium/pomerium/internal/log"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/testenv/envutil"
 	"github.com/pomerium/pomerium/internal/testenv/values"
 	"github.com/pomerium/pomerium/pkg/cmd/pomerium"
@@ -45,6 +46,8 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/attribute"
+	oteltrace "go.opentelemetry.io/otel/trace"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc/grpclog"
 )
@@ -56,6 +59,7 @@ type Environment interface {
 	// top-level logger scoped to this environment. It will be canceled when
 	// Stop() is called, or during test cleanup.
 	Context() context.Context
+	Tracer() oteltrace.Tracer
 
 	Assert() *assert.Assertions
 	Require() *require.Assertions
@@ -133,9 +137,18 @@ type Environment interface {
 	// the Pomerium server and Envoy.
 	NewLogRecorder(opts ...LogRecorderOption) *LogRecorder
 
+	// GetState returns the current state of the test environment.
+	GetState() EnvironmentState
+
 	// OnStateChanged registers a callback to be invoked when the environment's
-	// state changes to the given state. The callback is invoked in a separate
+	// state changes to the given state. Each callback is invoked in a separate
-	// goroutine.
+	// goroutine, but the test environment will wait for all callbacks to return
+	// before continuing, after triggering the state change.
+	// State changes are triggered in the following places:
+	// - NotRunning->Starting: in Start(), as the first operation
+	// - Starting->Running: in Start(), just before returning
+	// - Running->Stopping: in Stop(), just before the env context is canceled
+	// - Stopping->Stopped: in Stop(), after all tasks have completed
 	OnStateChanged(state EnvironmentState, callback func())
 }
 
@@ -192,10 +205,12 @@ type environment struct {
 	workspaceFolder string
 	silent          bool
 
-	ctx         context.Context
+	ctx            context.Context
-	cancel      context.CancelCauseFunc
+	cancel         context.CancelCauseFunc
-	cleanupOnce sync.Once
+	cleanupOnce    sync.Once
-	logWriter   *log.MultiWriter
+	logWriter      *log.MultiWriter
+	tracerProvider oteltrace.TracerProvider
+	tracer         oteltrace.Tracer
 
 	mods         []WithCaller[Modifier]
 	tasks        []WithCaller[Task]
@@ -305,7 +320,14 @@ func New(t testing.TB, opts ...EnvironmentOption) Environment {
 	})
 	logger := zerolog.New(writer).With().Timestamp().Logger().Level(zerolog.DebugLevel)
 
-	ctx, cancel := context.WithCancelCause(logger.WithContext(context.Background()))
+	ctx, cancel := context.WithCancelCause(logger.WithContext(trace.NewContext(context.Background())))
+	t.Cleanup(func() {
+		trace.Shutdown(ctx)
+	})
+	tracerProvider := trace.NewTracerProvider(ctx, "Test Environment")
+	tracer := tracerProvider.Tracer(trace.PomeriumCoreTracer)
+	ctx, span := tracer.Start(ctx, t.Name())
+	require.NoError(t, err)
 	taskErrGroup, ctx := errgroup.WithContext(ctx)
 
 	e := &environment{
@@ -325,13 +347,19 @@ func New(t testing.TB, opts ...EnvironmentOption) Environment {
 			Debug:        values.Deferred[int](),
 			ALPN:         values.Deferred[int](),
 		},
-		workspaceFolder: workspaceFolder,
+		workspaceFolder:      workspaceFolder,
-		silent:          silent,
+		silent:               silent,
-		ctx:             ctx,
+		ctx:                  ctx,
-		cancel:          cancel,
+		cancel:               cancel,
-		logWriter:       writer,
+		tracerProvider:       tracerProvider,
-		taskErrGroup:    taskErrGroup,
+		tracer:               tracerProvider.Tracer(trace.PomeriumCoreTracer),
+		logWriter:            writer,
+		taskErrGroup:         taskErrGroup,
+		stateChangeListeners: make(map[EnvironmentState][]func()),
 	}
+	e.OnStateChanged(Stopped, func() {
+		span.End()
+	})
 	_, err = rand.Read(e.sharedSecret[:])
 	require.NoError(t, err)
 	_, err = rand.Read(e.cookieSecret[:])
@@ -394,6 +422,10 @@ func (e *environment) Context() context.Context {
 	return ContextWithEnv(e.ctx, e)
 }
 
+func (e *environment) Tracer() oteltrace.Tracer {
+	return e.tracer
+}
+
 func (e *environment) Assert() *assert.Assertions {
 	return e.assert
 }
@@ -455,6 +487,8 @@ var ErrCauseTestCleanup = errors.New("test cleanup")
 var ErrCauseManualStop = errors.New("Stop() called")
 
 func (e *environment) Start() {
+	_, span := e.tracer.Start(e.ctx, "Start")
+	defer span.End()
 	e.debugf("Start()")
 	e.advanceState(Starting)
 	e.t.Cleanup(e.cleanup)
@@ -515,6 +549,7 @@ func (e *environment) Start() {
 		log.AccessLogFieldUserAgent,
 		log.AccessLogFieldClientCertificate,
 	}
+	cfg.Options.TracingSampleRate = 1.0
 
 	e.src = &configSource{cfg: cfg}
 	e.AddTask(TaskFunc(func(ctx context.Context) error {
@@ -524,7 +559,7 @@ func (e *environment) Start() {
 			require.NoError(e.t, cfg.Options.Validate(), "invoking modifier resulted in an invalid configuration:\nadded by: "+mod.Caller)
 		}
 
-		opts := []pomerium.RunOption{
+		opts := []pomerium.Option{
 			pomerium.WithOverrideFileManager(fileMgr),
 		}
 		envoyBinaryPath := filepath.Join(e.workspaceFolder, fmt.Sprintf("pkg/envoy/files/envoy-%s-%s", runtime.GOOS, runtime.GOARCH))
@@ -565,7 +600,12 @@ func (e *environment) Start() {
 			e.debugf("envoy profiling not available")
 		}
 
-		return pomerium.Run(ctx, e.src, opts...)
+		pom := pomerium.New(opts...)
+		e.OnStateChanged(Stopping, func() {
+			pom.Shutdown()
+		})
+		pom.Start(ctx, e.tracerProvider, e.src)
+		return pom.Wait()
 	}))
 
 	for i, task := range e.tasks {
@@ -740,7 +780,7 @@ func (e *environment) Add(m Modifier) {
 	e.t.Helper()
 	caller := getCaller()
 	e.debugf("Add: %T from %s", m, caller)
-	switch e.getState() {
+	switch e.GetState() {
 	case NotRunning:
 		for _, mod := range e.mods {
 			if mod.Value == m {
@@ -761,7 +801,7 @@ func (e *environment) Add(m Modifier) {
 	case Stopped, Stopping:
 		panic("test bug: cannot call Add() after Stop()")
 	default:
-		panic(fmt.Sprintf("unexpected environment state: %s", e.getState()))
+		panic(fmt.Sprintf("unexpected environment state: %s", e.GetState()))
 	}
 }
 
@@ -805,13 +845,25 @@ func (e *environment) advanceState(newState EnvironmentState) {
 	}
 	e.debugf("state %s -> %s", e.state.String(), newState.String())
 	e.state = newState
-	e.debugf("notifying %d listeners of state change", len(e.stateChangeListeners[newState]))
+	if len(e.stateChangeListeners[newState]) > 0 {
-	for _, listener := range e.stateChangeListeners[newState] {
+		e.debugf("notifying %d listeners of state change", len(e.stateChangeListeners[newState]))
-		go listener()
+		var wg sync.WaitGroup
+		for _, listener := range e.stateChangeListeners[newState] {
+			wg.Add(1)
+			go func() {
+				_, span := e.tracer.Start(e.ctx, "State Change Callback")
+				span.SetAttributes(attribute.String("state", newState.String()))
+				defer span.End()
+				defer wg.Done()
+				listener()
+			}()
+		}
+		wg.Wait()
+		e.debugf("done notifying state change listeners")
 	}
 }
 
-func (e *environment) getState() EnvironmentState {
+func (e *environment) GetState() EnvironmentState {
 	e.stateMu.Lock()
 	defer e.stateMu.Unlock()
 	return e.state
@@ -828,7 +880,7 @@ func (e *environment) OnStateChanged(state EnvironmentState, callback func()) {
 
 	// add change listeners for all states, if there are multiple bits set
 	for state > 0 {
-		stateBit := EnvironmentState(bits.TrailingZeros32(uint32(state)))
+		stateBit := EnvironmentState(1 << bits.TrailingZeros32(uint32(state)))
 		state &= (state - 1)
 		e.stateChangeListeners[stateBit] = append(e.stateChangeListeners[stateBit], callback)
 	}

internal/testenv/scenarios/mock_idp.go

@@ -45,7 +45,7 @@ type IDP struct {
 func (idp *IDP) Attach(ctx context.Context) {
 	env := testenv.EnvFromContext(ctx)
 
-	router := upstreams.HTTP(nil)
+	router := upstreams.HTTP(nil, upstreams.WithDisplayName("IDP"))
 
 	idp.url = values.Bind2(env.SubdomainURL("mock-idp"), router.Port(), func(urlStr string, port int) string {
 		u, _ := url.Parse(urlStr)

internal/testenv/snippets/wait.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"time"
 
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/testenv"
 	"github.com/pomerium/pomerium/pkg/grpcutil"
 	"google.golang.org/grpc"
@@ -12,6 +13,11 @@ import (
 )
 
 func WaitStartupComplete(env testenv.Environment, timeout ...time.Duration) time.Duration {
+	if env.GetState() == testenv.NotRunning {
+		panic("test bug: WaitStartupComplete called before starting the test environment")
+	}
+	_, span := trace.Continue(env.Context(), "snippets.WaitStartupComplete")
+	defer span.End()
 	start := time.Now()
 	recorder := env.NewLogRecorder()
 	if len(timeout) == 0 {

internal/testenv/types.go

@@ -84,7 +84,7 @@ type Aggregate struct {
 
 func (d *Aggregate) Add(mod Modifier) {
 	if d.env != nil {
-		if d.env.(*environment).getState() == NotRunning {
+		if d.env.(*environment).GetState() == NotRunning {
 			// If the test environment is running, adding to an aggregate is a no-op.
 			// If the test environment has not been started yet, the aggregate is
 			// being used like in the following example, which is incorrect:

internal/testenv/upstreams/http.go

@@ -20,12 +20,18 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/pomerium/pomerium/integration/forms"
 	"github.com/pomerium/pomerium/internal/retry"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/testenv"
 	"github.com/pomerium/pomerium/internal/testenv/values"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+
+	"go.opentelemetry.io/otel/attribute"
+	oteltrace "go.opentelemetry.io/otel/trace"
 	"google.golang.org/protobuf/proto"
 )
 
 type RequestOptions struct {
+	requestCtx     context.Context
 	path           string
 	query          url.Values
 	headers        map[string]string
@@ -77,6 +83,12 @@ func Client(c *http.Client) RequestOption {
 	}
 }
 
+func Context(ctx context.Context) RequestOption {
+	return func(o *RequestOptions) {
+		o.requestCtx = ctx
+	}
+}
+
 // Body sets the body of the request.
 // The argument can be one of the following types:
 // - string
@@ -102,6 +114,24 @@ func ClientCert[T interface {
 	}
 }
 
+type HTTPUpstreamOptions struct {
+	displayName string
+}
+
+type HTTPUpstreamOption func(*HTTPUpstreamOptions)
+
+func (o *HTTPUpstreamOptions) apply(opts ...HTTPUpstreamOption) {
+	for _, op := range opts {
+		op(o)
+	}
+}
+
+func WithDisplayName(displayName string) HTTPUpstreamOption {
+	return func(o *HTTPUpstreamOptions) {
+		o.displayName = displayName
+	}
+}
+
 // HTTPUpstream represents a HTTP server which can be used as the target for
 // one or more Pomerium routes in a test environment.
 //
@@ -119,13 +149,15 @@ type HTTPUpstream interface {
 }
 
 type httpUpstream struct {
+	HTTPUpstreamOptions
 	testenv.Aggregate
 	serverPort values.MutableValue[int]
 	tlsConfig  values.Value[*tls.Config]
 
 	clientCache sync.Map // map[testenv.Route]*http.Client
 
-	router *mux.Router
+	router         *mux.Router
+	tracerProvider oteltrace.TracerProvider
 }
 
 var (
@@ -134,11 +166,16 @@ var (
 )
 
 // HTTP creates a new HTTP upstream server.
-func HTTP(tlsConfig values.Value[*tls.Config]) HTTPUpstream {
+func HTTP(tlsConfig values.Value[*tls.Config], opts ...HTTPUpstreamOption) HTTPUpstream {
+	options := HTTPUpstreamOptions{
+		displayName: "HTTP Upstream",
+	}
+	options.apply(opts...)
 	up := &httpUpstream{
-		serverPort: values.Deferred[int](),
+		HTTPUpstreamOptions: options,
-		router:     mux.NewRouter(),
+		serverPort:          values.Deferred[int](),
-		tlsConfig:  tlsConfig,
+		router:              mux.NewRouter(),
+		tlsConfig:           tlsConfig,
 	}
 	up.RecordCaller()
 	return up
@@ -176,6 +213,9 @@ func (h *httpUpstream) Run(ctx context.Context) error {
 	if h.tlsConfig != nil {
 		tlsConfig = h.tlsConfig.Value()
 	}
+	h.router.Use(trace.NewHTTPMiddleware(otelhttp.WithTracerProvider(h.tracerProvider)))
+	h.tracerProvider = trace.NewTracerProvider(ctx, h.displayName)
+
 	server := &http.Server{
 		Handler:   h.router,
 		TLSConfig: tlsConfig,
@@ -208,7 +248,9 @@ func (h *httpUpstream) Post(r testenv.Route, opts ...RequestOption) (*http.Respo
 
 // Do implements HTTPUpstream.
 func (h *httpUpstream) Do(method string, r testenv.Route, opts ...RequestOption) (*http.Response, error) {
-	options := RequestOptions{}
+	options := RequestOptions{
+		requestCtx: h.Env().Context(),
+	}
 	options.apply(opts...)
 	u, err := url.Parse(r.URL().Value())
 	if err != nil {
@@ -220,7 +262,8 @@ func (h *httpUpstream) Do(method string, r testenv.Route, opts ...RequestOption)
 			RawQuery: options.query.Encode(),
 		})
 	}
-	req, err := http.NewRequest(method, u.String(), nil)
+
+	req, err := http.NewRequestWithContext(options.requestCtx, method, u.String(), nil)
 	if err != nil {
 		return nil, err
 	}
@@ -250,12 +293,17 @@ func (h *httpUpstream) Do(method string, r testenv.Route, opts ...RequestOption)
 
 	newClient := func() *http.Client {
 		c := http.Client{
-			Transport: &http.Transport{
+			Transport: otelhttp.NewTransport(&http.Transport{
 				TLSClientConfig: &tls.Config{
 					RootCAs:      h.Env().ServerCAs(),
 					Certificates: options.clientCerts,
 				},
 			},
+				otelhttp.WithTracerProvider(h.tracerProvider),
+				otelhttp.WithSpanNameFormatter(func(operation string, r *http.Request) string {
+					return fmt.Sprintf("Client: %s %s", r.Method, r.URL.Path)
+				}),
+			),
 		}
 		c.Jar, _ = cookiejar.New(&cookiejar.Options{})
 		return &c
@@ -273,7 +321,7 @@ func (h *httpUpstream) Do(method string, r testenv.Route, opts ...RequestOption)
 	}
 
 	var resp *http.Response
-	if err := retry.Retry(h.Env().Context(), "http", func(ctx context.Context) error {
+	if err := retry.Retry(options.requestCtx, "http", func(ctx context.Context) error {
 		var err error
 		if options.authenticateAs != "" {
 			resp, err = authenticateFlow(ctx, client, req, options.authenticateAs) //nolint:bodyclose
@@ -284,11 +332,15 @@ func (h *httpUpstream) Do(method string, r testenv.Route, opts ...RequestOption)
 		if err != nil {
 			var opErr *net.OpError
 			if errors.As(err, &opErr) && opErr.Op == "dial" && opErr.Err.Error() == "connect: connection refused" {
+				oteltrace.SpanFromContext(ctx).AddEvent("Retrying on dial error")
 				return err
 			}
 			return retry.NewTerminalError(err)
 		}
-		if resp.StatusCode == http.StatusInternalServerError {
+		if resp.StatusCode/100 == 5 {
+			oteltrace.SpanFromContext(ctx).AddEvent("Retrying on 5xx error", oteltrace.WithAttributes(
+				attribute.String("status", resp.Status),
+			))
 			return errors.New(http.StatusText(resp.StatusCode))
 		}
 		return nil

internal/urlutil/proxy.go

@@ -47,6 +47,14 @@ func GetCallbackURLForRedirectURI(r *http.Request, encodedSessionJWT, rawRedirec
 	if r.FormValue(QueryIsProgrammatic) == "true" {
 		callbackParams.Set(QueryIsProgrammatic, "true")
 	}
+	// propagate trace context
+	if tracecontext := r.FormValue(QueryTraceparent); tracecontext != "" {
+		callbackParams.Set(QueryTraceparent, tracecontext)
+	}
+	if tracestate := r.FormValue(QueryTracestate); tracestate != "" {
+		callbackParams.Set(QueryTracestate, tracestate)
+	}
+
 	// add our encoded and encrypted route-session JWT to a query param
 	callbackParams.Set(QuerySessionEncrypted, encodedSessionJWT)
 	callbackParams.Set(QueryRedirectURI, redirectURI.String())

internal/urlutil/query_params.go

@@ -20,6 +20,8 @@ const (
 	QuerySessionState       = "pomerium_session_state"
 	QueryVersion            = "pomerium_version"
 	QueryRequestUUID        = "pomerium_request_uuid"
+	QueryTraceparent        = "pomerium_traceparent"
+	QueryTracestate         = "pomerium_tracestate"
 )
 
 // URL signature based query params used for verifying the authenticity of a URL.

internal/zero/controller/usagereporter/usagereporter_test.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc"
 	"google.golang.org/protobuf/types/known/timestamppb"
@@ -37,7 +38,7 @@ func TestUsageReporter(t *testing.T) {
 	t.Cleanup(cancel)
 
 	cc := testutil.NewGRPCServer(t, func(srv *grpc.Server) {
-		databrokerpb.RegisterDataBrokerServiceServer(srv, databroker.New(ctx))
+		databrokerpb.RegisterDataBrokerServiceServer(srv, databroker.New(ctx, trace.NewNoopTracerProvider()))
 	})
 	t.Cleanup(func() { cc.Close() })
 

pkg/cmd/pomerium/pomerium.go

@@ -3,11 +3,9 @@ package pomerium
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/http"
-	"os"
-	"os/signal"
-	"syscall"
 
 	envoy_service_auth_v3 "github.com/envoyproxy/go-control-plane/envoy/service/auth/v3"
 	"go.uber.org/automaxprocs/maxprocs"
@@ -24,43 +22,72 @@ import (
 	"github.com/pomerium/pomerium/internal/events"
 	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/internal/registry"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/version"
 	derivecert_config "github.com/pomerium/pomerium/pkg/derivecert/config"
 	"github.com/pomerium/pomerium/pkg/envoy"
 	"github.com/pomerium/pomerium/pkg/envoy/files"
 	"github.com/pomerium/pomerium/proxy"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
-type RunOptions struct {
+type Options struct {
 	fileMgr            *filemgr.Manager
 	envoyServerOptions []envoy.ServerOption
 }
 
-type RunOption func(*RunOptions)
+type Option func(*Options)
 
-func (o *RunOptions) apply(opts ...RunOption) {
+func (o *Options) apply(opts ...Option) {
 	for _, op := range opts {
 		op(o)
 	}
 }
 
-func WithOverrideFileManager(fileMgr *filemgr.Manager) RunOption {
+func WithOverrideFileManager(fileMgr *filemgr.Manager) Option {
-	return func(o *RunOptions) {
+	return func(o *Options) {
 		o.fileMgr = fileMgr
 	}
 }
 
-func WithEnvoyServerOptions(opts ...envoy.ServerOption) RunOption {
+func WithEnvoyServerOptions(opts ...envoy.ServerOption) Option {
-	return func(o *RunOptions) {
+	return func(o *Options) {
 		o.envoyServerOptions = append(o.envoyServerOptions, opts...)
 	}
 }
 
 // Run runs the main pomerium application.
-func Run(ctx context.Context, src config.Source, opts ...RunOption) error {
+func Run(ctx context.Context, src config.Source, opts ...Option) error {
-	options := RunOptions{}
+	p := New(opts...)
+	tracerProvider := trace.NewTracerProvider(ctx, "Pomerium")
+
+	if err := p.Start(ctx, tracerProvider, src); err != nil {
+		return err
+	}
+	return p.Wait()
+}
+
+var ErrShutdown = errors.New("Shutdown() called")
+
+type Pomerium struct {
+	Options
+	errGroup *errgroup.Group
+
+	cancel      context.CancelCauseFunc
+	envoyServer *envoy.Server
+}
+
+func New(opts ...Option) *Pomerium {
+	options := Options{}
 	options.apply(opts...)
 
+	return &Pomerium{
+		Options: options,
+	}
+}
+
+func (p *Pomerium) Start(ctx context.Context, tracerProvider oteltrace.TracerProvider, src config.Source) error {
+	ctx, p.cancel = context.WithCancelCause(ctx)
 	_, _ = maxprocs.Set(maxprocs.Logger(func(s string, i ...any) { log.Ctx(ctx).Debug().Msgf(s, i...) }))
 
 	evt := log.Ctx(ctx).Info().
@@ -75,9 +102,8 @@ func Run(ctx context.Context, src config.Source, opts ...RunOption) error {
 	if err != nil {
 		return err
 	}
-	src = databroker.NewConfigSource(ctx, src, databroker.EnableConfigValidation(true))
+	src = databroker.NewConfigSource(ctx, tracerProvider, src, databroker.EnableConfigValidation(true))
-	logMgr := config.NewLogManager(ctx, src)
+	_ = config.NewLogManager(ctx, src)
-	defer logMgr.Close()
 
 	// trigger changes when underlying files are changed
 	src = config.NewFileWatcherSource(ctx, src)
@@ -91,13 +117,10 @@ func Run(ctx context.Context, src config.Source, opts ...RunOption) error {
 	http.DefaultTransport = config.NewHTTPTransport(src)
 
 	metricsMgr := config.NewMetricsManager(ctx, src)
-	defer metricsMgr.Close()
-	traceMgr := config.NewTraceManager(ctx, src)
-	defer traceMgr.Close()
 
 	eventsMgr := events.New()
 
-	fileMgr := options.fileMgr
+	fileMgr := p.fileMgr
 	if fileMgr == nil {
 		fileMgr = filemgr.NewManager()
 	}
@@ -130,11 +153,13 @@ func Run(ctx context.Context, src config.Source, opts ...RunOption) error {
 		Msg("server started")
 
 	// create envoy server
-	envoyServer, err := envoy.NewServer(ctx, src, controlPlane.Builder, options.envoyServerOptions...)
+	p.envoyServer, err = envoy.NewServer(ctx, src, controlPlane.Builder, p.envoyServerOptions...)
 	if err != nil {
 		return fmt.Errorf("error creating envoy server: %w", err)
 	}
-	defer envoyServer.Close()
+	context.AfterFunc(ctx, func() {
+		p.envoyServer.Close()
+	})
 
 	// add services
 	if err := setupAuthenticate(ctx, src, controlPlane); err != nil {
@@ -155,44 +180,43 @@ func Run(ctx context.Context, src config.Source, opts ...RunOption) error {
 		}
 	}
 
-	if err = setupRegistryReporter(ctx, src); err != nil {
+	if err = setupRegistryReporter(ctx, tracerProvider, src); err != nil {
 		return fmt.Errorf("setting up registry reporter: %w", err)
 	}
 	if err := setupProxy(ctx, src, controlPlane); err != nil {
 		return err
 	}
 
-	ctx, cancel := context.WithCancel(ctx)
-	go func(ctx context.Context) {
-		ch := make(chan os.Signal, 2)
-		defer signal.Stop(ch)
-
-		signal.Notify(ch, os.Interrupt)
-		signal.Notify(ch, syscall.SIGTERM)
-
-		select {
-		case <-ch:
-		case <-ctx.Done():
-		}
-		cancel()
-	}(ctx)
-
 	// run everything
-	eg, ctx := errgroup.WithContext(ctx)
+	p.errGroup, ctx = errgroup.WithContext(ctx)
 	if authorizeServer != nil {
-		eg.Go(func() error {
+		p.errGroup.Go(func() error {
 			return authorizeServer.Run(ctx)
 		})
 	}
-	eg.Go(func() error {
+	p.errGroup.Go(func() error {
 		return controlPlane.Run(ctx)
 	})
 	if dataBrokerServer != nil {
-		eg.Go(func() error {
+		p.errGroup.Go(func() error {
 			return dataBrokerServer.Run(ctx)
 		})
 	}
-	return eg.Wait()
+	return nil
+}
+
+func (p *Pomerium) Shutdown() error {
+	_ = p.envoyServer.Close() // this only errors if signaling envoy fails
+	p.cancel(ErrShutdown)
+	return p.Wait()
+}
+
+func (p *Pomerium) Wait() error {
+	err := p.errGroup.Wait()
+	if errors.Is(err, ErrShutdown) {
+		return nil
+	}
+	return err
 }
 
 func setupAuthenticate(ctx context.Context, src config.Source, controlPlane *controlplane.Server) error {
@@ -245,8 +269,8 @@ func setupDataBroker(ctx context.Context,
 	return svc, nil
 }
 
-func setupRegistryReporter(ctx context.Context, src config.Source) error {
+func setupRegistryReporter(ctx context.Context, tracerProvider oteltrace.TracerProvider, src config.Source) error {
-	reporter := registry.NewReporter()
+	reporter := registry.NewReporter(tracerProvider)
 	src.OnConfigChange(ctx, reporter.OnConfigChange)
 	reporter.OnConfigChange(ctx, src.GetConfig())
 	return nil

pkg/envoy/envoy.go

@@ -16,6 +16,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	stdatomic "sync/atomic"
 	"syscall"
 	"time"
 
@@ -42,6 +43,7 @@ type Server struct {
 	wd        string
 	cmd       *exec.Cmd
 	cmdExited chan struct{}
+	closing   stdatomic.Bool
 
 	builder            *envoyconfig.Builder
 	resourceMonitor    ResourceMonitor
@@ -122,8 +124,13 @@ func NewServer(ctx context.Context, src config.Source, builder *envoyconfig.Buil
 	return srv, nil
 }
 
-// Close kills any underlying envoy process.
+// Close attempts to gracefully shut down a running envoy server. If envoy
+// does not exit within the defined grace period, it will be killed. Server
+// cannot be used again after Close is called.
 func (srv *Server) Close() error {
+	if !srv.closing.CompareAndSwap(false, true) {
+		return nil
+	}
 	srv.monitorProcessCancel()
 
 	srv.mu.Lock()
@@ -156,6 +163,10 @@ func (srv *Server) Close() error {
 }
 
 func (srv *Server) onConfigChange(ctx context.Context, cfg *config.Config) {
+	if srv.closing.Load() {
+		// do not attempt to update the configuration after Close is called
+		return
+	}
 	srv.update(ctx, cfg)
 }
 
@@ -233,10 +244,10 @@ func (srv *Server) run(ctx context.Context, cfg *config.Config) error {
 	go func() {
 		pid := cmd.Process.Pid
 		err := srv.monitorProcess(monitorProcessCtx, int32(pid))
-		if err != nil && ctx.Err() == nil {
+		if err != nil && ctx.Err() == nil && !srv.closing.Load() {
-			// If the envoy subprocess exits and ctx is not done, issue a fatal error.
+			// If the envoy subprocess exits and ctx is not done (or waiting for envoy
-			// If ctx is done, the server is already exiting, and envoy is expected
+			// to gracefully stop), issue a fatal error. If ctx is done, the server is
-			// to be stopped along with it.
+			// already exiting, and envoy is expected to be stopped along with it.
 			log.Ctx(ctx).
 				Fatal().
 				Int("pid", pid).

pkg/grpc/client.go

@@ -7,13 +7,14 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/connectivity"
 
 	"github.com/pomerium/pomerium/internal/log"
-	"github.com/pomerium/pomerium/internal/telemetry"
 	"github.com/pomerium/pomerium/pkg/grpcutil"
 	"github.com/pomerium/pomerium/pkg/telemetry/requestid"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
 // Options contains options for connecting to a pomerium rpc service.
@@ -32,12 +33,9 @@ type Options struct {
 }
 
 // NewGRPCClientConn returns a new gRPC pomerium service client connection.
-func NewGRPCClientConn(ctx context.Context, opts *Options, other ...grpc.DialOption) (*grpc.ClientConn, error) {
+func NewGRPCClientConn(ctx context.Context, tracerProvider oteltrace.TracerProvider, opts *Options, other ...grpc.DialOption) (*grpc.ClientConn, error) {
-	clientStatsHandler := telemetry.NewGRPCClientStatsHandler(opts.ServiceName)
-
 	unaryClientInterceptors := []grpc.UnaryClientInterceptor{
 		requestid.UnaryClientInterceptor(),
-		clientStatsHandler.UnaryInterceptor,
 	}
 	streamClientInterceptors := []grpc.StreamClientInterceptor{
 		requestid.StreamClientInterceptor(),
@@ -50,7 +48,7 @@ func NewGRPCClientConn(ctx context.Context, opts *Options, other ...grpc.DialOpt
 	dialOptions := []grpc.DialOption{
 		grpc.WithChainUnaryInterceptor(unaryClientInterceptors...),
 		grpc.WithChainStreamInterceptor(streamClientInterceptors...),
-		grpc.WithStatsHandler(clientStatsHandler.Handler),
+		grpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(tracerProvider))),
 		grpc.WithDisableServiceConfig(),
 		grpc.WithInsecure(),
 	}
@@ -87,8 +85,8 @@ type OutboundOptions struct {
 }
 
 // newOutboundGRPCClientConn gets a new outbound gRPC client.
-func newOutboundGRPCClientConn(ctx context.Context, opts *OutboundOptions) (*grpc.ClientConn, error) {
+func newOutboundGRPCClientConn(ctx context.Context, tracerProvider oteltrace.TracerProvider, opts *OutboundOptions) (*grpc.ClientConn, error) {
-	return NewGRPCClientConn(ctx, &Options{
+	return NewGRPCClientConn(ctx, tracerProvider, &Options{
 		Address:        net.JoinHostPort("127.0.0.1", opts.OutboundPort),
 		InstallationID: opts.InstallationID,
 		ServiceName:    opts.ServiceName,
@@ -104,7 +102,7 @@ type CachedOutboundGRPClientConn struct {
 }
 
 // Get gets the cached outbound gRPC client, or creates a new one if the options have changed.
-func (cache *CachedOutboundGRPClientConn) Get(ctx context.Context, opts *OutboundOptions) (*grpc.ClientConn, error) {
+func (cache *CachedOutboundGRPClientConn) Get(ctx context.Context, tracerProvider oteltrace.TracerProvider, opts *OutboundOptions) (*grpc.ClientConn, error) {
 	cache.mu.Lock()
 	defer cache.mu.Unlock()
 
@@ -118,7 +116,7 @@ func (cache *CachedOutboundGRPClientConn) Get(ctx context.Context, opts *Outboun
 	}
 
 	var err error
-	cache.current, err = newOutboundGRPCClientConn(ctx, opts)
+	cache.current, err = newOutboundGRPCClientConn(ctx, tracerProvider, opts)
 	if err != nil {
 		return nil, err
 	}

pkg/grpc/databroker/sync_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace"
 	grpc "google.golang.org/grpc"
 
 	"github.com/pomerium/pomerium/internal/databroker"
@@ -24,7 +25,7 @@ func Test_SyncLatestRecords(t *testing.T) {
 	defer clearTimeout()
 
 	cc := testutil.NewGRPCServer(t, func(s *grpc.Server) {
-		databrokerpb.RegisterDataBrokerServiceServer(s, databroker.New(ctx))
+		databrokerpb.RegisterDataBrokerServiceServer(s, databroker.New(ctx, trace.NewNoopTracerProvider()))
 	})
 
 	c := databrokerpb.NewDataBrokerServiceClient(cc)

pkg/identity/providers.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"net/http"
 
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"golang.org/x/oauth2"
 
 	"github.com/pomerium/pomerium/pkg/identity/identity"
@@ -22,6 +23,7 @@ import (
 	"github.com/pomerium/pomerium/pkg/identity/oidc/okta"
 	"github.com/pomerium/pomerium/pkg/identity/oidc/onelogin"
 	"github.com/pomerium/pomerium/pkg/identity/oidc/ping"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
 // State is the identity state.
@@ -64,13 +66,20 @@ func init() {
 }
 
 // NewAuthenticator returns a new identity provider based on its name.
-func NewAuthenticator(o oauth.Options) (a Authenticator, err error) {
+func NewAuthenticator(ctx context.Context, tracerProvider oteltrace.TracerProvider, o oauth.Options) (a Authenticator, err error) {
-	ctx := context.Background()
-
 	if o.ProviderName == "" {
 		return nil, fmt.Errorf("identity: provider is not defined")
 	}
 
+	ctx = context.WithValue(ctx, oauth2.HTTPClient, &http.Client{
+		Transport: otelhttp.NewTransport(nil,
+			otelhttp.WithTracerProvider(tracerProvider),
+			otelhttp.WithSpanNameFormatter(func(operation string, r *http.Request) string {
+				return fmt.Sprintf("OAuth2 Client: %s %s", r.Method, r.URL.Path)
+			}),
+		),
+	})
+
 	ctor, ok := registry[o.ProviderName]
 	if !ok {
 		return nil, fmt.Errorf("identity: unknown provider: %s", o.ProviderName)

proxy/data_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace"
 	"google.golang.org/grpc"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/timestamppb"
@@ -32,7 +33,7 @@ func Test_getUserInfoData(t *testing.T) {
 	defer clearTimeout()
 
 	cc := testutil.NewGRPCServer(t, func(srv *grpc.Server) {
-		databrokerpb.RegisterDataBrokerServiceServer(srv, databroker.New(ctx))
+		databrokerpb.RegisterDataBrokerServiceServer(srv, databroker.New(ctx, trace.NewNoopTracerProvider()))
 	})
 	t.Cleanup(func() { cc.Close() })
 

proxy/handlers.go

@@ -9,11 +9,13 @@ import (
 
 	"github.com/go-jose/go-jose/v3/jwt"
 	"github.com/gorilla/mux"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 
 	"github.com/pomerium/pomerium/config"
 	"github.com/pomerium/pomerium/internal/handlers"
 	"github.com/pomerium/pomerium/internal/httputil"
 	"github.com/pomerium/pomerium/internal/middleware"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/urlutil"
 )
 
@@ -21,6 +23,7 @@ import (
 func (p *Proxy) registerDashboardHandlers(r *mux.Router, opts *config.Options) *mux.Router {
 	h := httputil.DashboardSubrouter(r)
 	h.Use(middleware.SetHeaders(httputil.HeadersContentSecurityPolicy))
+	h.Use(trace.NewHTTPMiddleware(otelhttp.WithTracerProvider(p.tracerProvider)))
 
 	// special pomerium endpoints for users to view their session
 	h.Path("/").Handler(httputil.HandlerFunc(p.userInfo)).Methods(http.MethodGet)

proxy/proxy.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	oteltrace "go.opentelemetry.io/otel/trace"
 
 	"github.com/pomerium/pomerium/config"
 	"github.com/pomerium/pomerium/internal/atomicutil"
@@ -17,6 +18,7 @@ import (
 	"github.com/pomerium/pomerium/internal/httputil"
 	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/internal/telemetry/metrics"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/pkg/cryptutil"
 )
 
@@ -56,17 +58,20 @@ type Proxy struct {
 	currentOptions *atomicutil.Value[*config.Options]
 	currentRouter  *atomicutil.Value[*mux.Router]
 	webauthn       *webauthn.Handler
+	tracerProvider oteltrace.TracerProvider
 }
 
 // New takes a Proxy service from options and a validation function.
 // Function returns an error if options fail to validate.
 func New(ctx context.Context, cfg *config.Config) (*Proxy, error) {
-	state, err := newProxyStateFromConfig(ctx, cfg)
+	tracerProvider := trace.NewTracerProvider(ctx, "Proxy")
+	state, err := newProxyStateFromConfig(ctx, tracerProvider, cfg)
 	if err != nil {
 		return nil, err
 	}
 
 	p := &Proxy{
+		tracerProvider: tracerProvider,
 		state:          atomicutil.NewValue(state),
 		currentOptions: config.NewAtomicOptions(),
 		currentRouter:  atomicutil.NewValue(httputil.NewRouter()),
@@ -96,7 +101,7 @@ func (p *Proxy) OnConfigChange(ctx context.Context, cfg *config.Config) {
 	if err := p.setHandlers(ctx, cfg.Options); err != nil {
 		log.Ctx(ctx).Error().Err(err).Msg("proxy: failed to update proxy handlers from configuration settings")
 	}
-	if state, err := newProxyStateFromConfig(ctx, cfg); err != nil {
+	if state, err := newProxyStateFromConfig(ctx, p.tracerProvider, cfg); err != nil {
 		log.Ctx(ctx).Error().Err(err).Msg("proxy: failed to update proxy state from configuration settings")
 	} else {
 		p.state.Store(state)

proxy/state.go

@@ -9,6 +9,7 @@ import (
 	"github.com/pomerium/pomerium/internal/authenticateflow"
 	"github.com/pomerium/pomerium/pkg/grpc"
 	"github.com/pomerium/pomerium/pkg/grpc/databroker"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
 var outboundGRPCConnection = new(grpc.CachedOutboundGRPClientConn)
@@ -31,7 +32,7 @@ type proxyState struct {
 	authenticateFlow                    authenticateFlow
 }
 
-func newProxyStateFromConfig(ctx context.Context, cfg *config.Config) (*proxyState, error) {
+func newProxyStateFromConfig(ctx context.Context, tracerProvider oteltrace.TracerProvider, cfg *config.Config) (*proxyState, error) {
 	err := ValidateOptions(cfg.Options)
 	if err != nil {
 		return nil, err
@@ -57,7 +58,7 @@ func newProxyStateFromConfig(ctx context.Context, cfg *config.Config) (*proxySta
 		return nil, err
 	}
 
-	dataBrokerConn, err := outboundGRPCConnection.Get(ctx, &grpc.OutboundOptions{
+	dataBrokerConn, err := outboundGRPCConnection.Get(ctx, tracerProvider, &grpc.OutboundOptions{
 		OutboundPort:   cfg.OutboundPort,
 		InstallationID: cfg.Options.InstallationID,
 		ServiceName:    cfg.Options.Services,
@@ -71,10 +72,10 @@ func newProxyStateFromConfig(ctx context.Context, cfg *config.Config) (*proxySta
 	state.programmaticRedirectDomainWhitelist = cfg.Options.ProgrammaticRedirectDomainWhitelist
 
 	if cfg.Options.UseStatelessAuthenticateFlow() {
-		state.authenticateFlow, err = authenticateflow.NewStateless(ctx,
+		state.authenticateFlow, err = authenticateflow.NewStateless(ctx, tracerProvider,
 			cfg, state.sessionStore, nil, nil, nil)
 	} else {
-		state.authenticateFlow, err = authenticateflow.NewStateful(ctx, cfg, state.sessionStore)
+		state.authenticateFlow, err = authenticateflow.NewStateful(ctx, tracerProvider, cfg, state.sessionStore)
 	}
 	if err != nil {
 		return nil, err