New tracing system (#5388)

* update tracing config definitions

* new tracing system

* performance improvements

* only configure tracing in envoy if it is enabled in pomerium

* [tracing] refactor to use custom extension for trace id editing (#5420)

refactor to use custom extension for trace id editing

* set default tracing sample rate to 1.0

* fix proxy service http middleware

* improve some existing auth related traces

* test fixes

* bump envoyproxy/go-control-plane

* code cleanup

* test fixes

* Fix missing spans for well-known endpoints

* import extension apis from pomerium/envoy-custom

authenticate/authenticate.go

@@ -10,7 +10,9 @@ import (
 	"github.com/pomerium/pomerium/config"
 	"github.com/pomerium/pomerium/internal/atomicutil"
 	"github.com/pomerium/pomerium/internal/log"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/pkg/cryptutil"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
 // ValidateOptions checks that configuration are complete and valid.
@@ -38,23 +40,31 @@ func ValidateOptions(o *config.Options) error {
 
 // Authenticate contains data required to run the authenticate service.
 type Authenticate struct {
-	cfg     *authenticateConfig
-	options *atomicutil.Value[*config.Options]
-	state   *atomicutil.Value[*authenticateState]
+	cfg            *authenticateConfig
+	options        *atomicutil.Value[*config.Options]
+	state          *atomicutil.Value[*authenticateState]
+	tracerProvider oteltrace.TracerProvider
+	tracer         oteltrace.Tracer
 }
 
 // New validates and creates a new authenticate service from a set of Options.
 func New(ctx context.Context, cfg *config.Config, options ...Option) (*Authenticate, error) {
 	authenticateConfig := getAuthenticateConfig(options...)
+
+	tracerProvider := trace.NewTracerProvider(ctx, "Authenticate")
+	tracer := tracerProvider.Tracer(trace.PomeriumCoreTracer)
+
 	a := &Authenticate{
-		cfg:     authenticateConfig,
-		options: config.NewAtomicOptions(),
-		state:   atomicutil.NewValue(newAuthenticateState()),
+		cfg:            authenticateConfig,
+		options:        config.NewAtomicOptions(),
+		state:          atomicutil.NewValue(newAuthenticateState()),
+		tracerProvider: tracerProvider,
+		tracer:         tracer,
 	}
 
 	a.options.Store(cfg.Options)
 
-	state, err := newAuthenticateStateFromConfig(ctx, cfg, authenticateConfig)
+	state, err := newAuthenticateStateFromConfig(ctx, tracerProvider, cfg, authenticateConfig)
 	if err != nil {
 		return nil, err
 	}
@@ -70,7 +80,7 @@ func (a *Authenticate) OnConfigChange(ctx context.Context, cfg *config.Config) {
 	}
 
 	a.options.Store(cfg.Options)
-	if state, err := newAuthenticateStateFromConfig(ctx, cfg, a.cfg); err != nil {
+	if state, err := newAuthenticateStateFromConfig(ctx, a.tracerProvider, cfg, a.cfg); err != nil {
 		log.Ctx(ctx).Error().Err(err).Msg("authenticate: failed to update state")
 	} else {
 		a.state.Store(state)

authenticate/config.go

@@ -1,14 +1,17 @@
 package authenticate
 
 import (
+	"context"
+
 	"github.com/pomerium/pomerium/authenticate/events"
 	"github.com/pomerium/pomerium/config"
 	identitypb "github.com/pomerium/pomerium/pkg/grpc/identity"
 	"github.com/pomerium/pomerium/pkg/identity"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
 type authenticateConfig struct {
-	getIdentityProvider func(options *config.Options, idpID string) (identity.Authenticator, error)
+	getIdentityProvider func(ctx context.Context, tracerProvider oteltrace.TracerProvider, options *config.Options, idpID string) (identity.Authenticator, error)
 	profileTrimFn       func(*identitypb.Profile)
 	authEventFn         events.AuthEventFn
 }
@@ -26,7 +29,7 @@ func getAuthenticateConfig(options ...Option) *authenticateConfig {
 }
 
 // WithGetIdentityProvider sets the getIdentityProvider function in the config.
-func WithGetIdentityProvider(getIdentityProvider func(options *config.Options, idpID string) (identity.Authenticator, error)) Option {
+func WithGetIdentityProvider(getIdentityProvider func(ctx context.Context, tracerProvider oteltrace.TracerProvider, options *config.Options, idpID string) (identity.Authenticator, error)) Option {
 	return func(cfg *authenticateConfig) {
 		cfg.getIdentityProvider = getIdentityProvider
 	}

authenticate/handlers.go

@@ -3,6 +3,7 @@ package authenticate
 import (
 	"context"
 	"encoding/base64"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"net/http"
@@ -13,6 +14,9 @@ import (
 	"github.com/google/uuid"
 	"github.com/gorilla/mux"
 	"github.com/rs/cors"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel/attribute"
+	oteltrace "go.opentelemetry.io/otel/trace"
 
 	"github.com/pomerium/csrf"
 	"github.com/pomerium/pomerium/internal/authenticateflow"
@@ -58,6 +62,7 @@ func (a *Authenticate) Mount(r *mux.Router) {
 		}
 		return csrf.Protect(state.cookieSecret, csrfOptions...)(h)
 	})
+	r.Use(trace.NewHTTPMiddleware(otelhttp.WithTracerProvider(a.tracerProvider)))
 
 	// redirect / to /.pomerium/
 	r.Path("/").Handler(http.RedirectHandler("/.pomerium/", http.StatusFound))
@@ -114,7 +119,7 @@ func (a *Authenticate) RetrieveSession(next http.Handler) http.Handler {
 // session state is attached to the users's request context.
 func (a *Authenticate) VerifySession(next http.Handler) http.Handler {
 	return httputil.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
-		ctx, span := trace.StartSpan(r.Context(), "authenticate.VerifySession")
+		ctx, span := a.tracer.Start(r.Context(), "authenticate.VerifySession")
 		defer span.End()
 
 		state := a.state.Load()
@@ -126,6 +131,8 @@ func (a *Authenticate) VerifySession(next http.Handler) http.Handler {
 				Err(err).
 				Str("idp_id", idpID).
 				Msg("authenticate: session load error")
+			span.AddEvent("session load error",
+				oteltrace.WithAttributes(attribute.String("error", err.Error())))
 			return a.reauthenticateOrFail(w, r, err)
 		}
 
@@ -135,6 +142,7 @@ func (a *Authenticate) VerifySession(next http.Handler) http.Handler {
 				Str("session_idp_id", sessionState.IdentityProviderID).
 				Str("id", sessionState.ID).
 				Msg("authenticate: session not associated with identity provider")
+			span.AddEvent("session not associated with identity provider")
 			return a.reauthenticateOrFail(w, r, err)
 		}
 
@@ -143,6 +151,8 @@ func (a *Authenticate) VerifySession(next http.Handler) http.Handler {
 				Err(err).
 				Str("idp_id", idpID).
 				Msg("authenticate: couldn't verify session")
+			span.AddEvent("couldn't verify session",
+				oteltrace.WithAttributes(attribute.String("error", err.Error())))
 			return a.reauthenticateOrFail(w, r, err)
 		}
 
@@ -160,7 +170,7 @@ func (a *Authenticate) RobotsTxt(w http.ResponseWriter, _ *http.Request) {
 
 // SignIn handles authenticating a user.
 func (a *Authenticate) SignIn(w http.ResponseWriter, r *http.Request) error {
-	ctx, span := trace.StartSpan(r.Context(), "authenticate.SignIn")
+	ctx, span := a.tracer.Start(r.Context(), "authenticate.SignIn")
 	defer span.End()
 
 	state := a.state.Load()
@@ -197,13 +207,13 @@ func (a *Authenticate) SignOut(w http.ResponseWriter, r *http.Request) error {
 }
 
 func (a *Authenticate) signOutRedirect(w http.ResponseWriter, r *http.Request) error {
-	ctx, span := trace.StartSpan(r.Context(), "authenticate.SignOut")
+	ctx, span := a.tracer.Start(r.Context(), "authenticate.SignOut")
 	defer span.End()
 
 	options := a.options.Load()
 	idpID := a.getIdentityProviderIDForRequest(r)
 
-	authenticator, err := a.cfg.getIdentityProvider(options, idpID)
+	authenticator, err := a.cfg.getIdentityProvider(ctx, a.tracerProvider, options, idpID)
 	if err != nil {
 		return err
 	}
@@ -274,7 +284,7 @@ func (a *Authenticate) reauthenticateOrFail(w http.ResponseWriter, r *http.Reque
 	options := a.options.Load()
 	idpID := a.getIdentityProviderIDForRequest(r)
 
-	authenticator, err := a.cfg.getIdentityProvider(options, idpID)
+	authenticator, err := a.cfg.getIdentityProvider(r.Context(), a.tracerProvider, options, idpID)
 	if err != nil {
 		return err
 	}
@@ -283,9 +293,20 @@ func (a *Authenticate) reauthenticateOrFail(w http.ResponseWriter, r *http.Reque
 
 	state.sessionStore.ClearSession(w, r)
 	redirectURL := state.redirectURL.ResolveReference(r.URL)
+	redirectURLValues := redirectURL.Query()
+	var traceID string
+	if tp := trace.PomeriumURLQueryCarrier(redirectURLValues).Get("traceparent"); len(tp) == 55 {
+		if traceIDBytes, err := hex.DecodeString(tp[3:35]); err == nil {
+			traceFlags, _ := hex.DecodeString(tp[53:55])
+			if len(traceFlags) != 1 {
+				traceFlags = []byte{0}
+			}
+			traceID = base64.RawURLEncoding.EncodeToString(append(traceIDBytes, traceFlags[0]))
+		}
+	}
 	nonce := csrf.Token(r)
 	now := time.Now().Unix()
-	b := []byte(fmt.Sprintf("%s|%d|", nonce, now))
+	b := []byte(fmt.Sprintf("%s|%d|%s|", nonce, now, traceID))
 	enc := cryptutil.Encrypt(state.cookieCipher, []byte(redirectURL.String()), b)
 	b = append(b, enc...)
 	encodedState := base64.URLEncoding.EncodeToString(b)
@@ -321,7 +342,7 @@ func (a *Authenticate) statusForErrorCode(errorCode string) int {
 }
 
 func (a *Authenticate) getOAuthCallback(w http.ResponseWriter, r *http.Request) (*url.URL, error) {
-	ctx, span := trace.StartSpan(r.Context(), "authenticate.getOAuthCallback")
+	ctx, span := a.tracer.Start(r.Context(), "authenticate.OAuthCallback")
 	defer span.End()
 
 	state := a.state.Load()
@@ -347,21 +368,20 @@ func (a *Authenticate) getOAuthCallback(w http.ResponseWriter, r *http.Request)
 	}
 
 	// split state into concat'd components
-	// (nonce|timestamp|redirect_url|encrypted_data(redirect_url)+mac(nonce,ts))
-	statePayload := strings.SplitN(string(bytes), "|", 3)
-	if len(statePayload) != 3 {
+	// (nonce|timestamp|trace_id+flags|encrypted_data(redirect_url)+mac(nonce,ts))
+	statePayload := strings.SplitN(string(bytes), "|", 4)
+	if len(statePayload) != 4 {
 		return nil, httputil.NewError(http.StatusBadRequest, fmt.Errorf("state malformed, size: %d", len(statePayload)))
 	}
 
 	// Use our AEAD construct to enforce secrecy and authenticity:
 	// mac: to validate the nonce again, and above timestamp
 	// decrypt: to prevent leaking 'redirect_uri' to IdP or logs
-	b := []byte(fmt.Sprint(statePayload[0], "|", statePayload[1], "|"))
-	redirectString, err := cryptutil.Decrypt(state.cookieCipher, []byte(statePayload[2]), b)
+	b := []byte(fmt.Sprint(statePayload[0], "|", statePayload[1], "|", statePayload[2], "|"))
+	redirectString, err := cryptutil.Decrypt(state.cookieCipher, []byte(statePayload[3]), b)
 	if err != nil {
 		return nil, httputil.NewError(http.StatusBadRequest, err)
 	}
-
 	redirectURL, err := urlutil.ParseAndValidateURL(string(redirectString))
 	if err != nil {
 		return nil, httputil.NewError(http.StatusBadRequest, err)
@@ -380,7 +400,7 @@ Or contact your administrator.
 
 	idpID := state.flow.GetIdentityProviderIDForURLValues(redirectURL.Query())
 
-	authenticator, err := a.cfg.getIdentityProvider(options, idpID)
+	authenticator, err := a.cfg.getIdentityProvider(ctx, a.tracerProvider, options, idpID)
 	if err != nil {
 		return nil, err
 	}
@@ -432,7 +452,7 @@ func (a *Authenticate) getSessionFromCtx(ctx context.Context) (*sessions.State,
 }
 
 func (a *Authenticate) userInfo(w http.ResponseWriter, r *http.Request) error {
-	ctx, span := trace.StartSpan(r.Context(), "authenticate.userInfo")
+	ctx, span := a.tracer.Start(r.Context(), "authenticate.userInfo")
 	defer span.End()
 
 	options := a.options.Load()
@@ -484,7 +504,7 @@ func (a *Authenticate) revokeSession(ctx context.Context, w http.ResponseWriter,
 
 	idpID := r.FormValue(urlutil.QueryIdentityProviderID)
 
-	authenticator, err := a.cfg.getIdentityProvider(options, idpID)
+	authenticator, err := a.cfg.getIdentityProvider(ctx, a.tracerProvider, options, idpID)
 	if err != nil {
 		return ""
 	}

authenticate/handlers_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	oteltrace "go.opentelemetry.io/otel/trace"
 	"go.uber.org/mock/gomock"
 	"golang.org/x/crypto/chacha20poly1305"
 	"golang.org/x/oauth2"
@@ -39,23 +40,21 @@ import (
 	"github.com/pomerium/pomerium/pkg/identity/oidc"
 )
 
-func testAuthenticate() *Authenticate {
-	redirectURL, _ := url.Parse("https://auth.example.com/oauth/callback")
-	var auth Authenticate
-	auth.state = atomicutil.NewValue(&authenticateState{
-		redirectURL:  redirectURL,
-		cookieSecret: cryptutil.NewKey(),
-		flow:         new(stubFlow),
+func testAuthenticate(t *testing.T) *Authenticate {
+	opts := newTestOptions(t)
+	opts.AuthenticateURLString = "https://auth.example.com/oauth/callback"
+	auth, err := New(context.Background(), &config.Config{
+		Options: opts,
 	})
-	auth.options = config.NewAtomicOptions()
-	auth.options.Store(&config.Options{
-		SharedKey: cryptutil.NewBase64Key(),
-	})
-	return &auth
+	if err != nil {
+		panic(err)
+	}
+	auth.state.Load().flow = new(stubFlow)
+	return auth
 }
 
 func TestAuthenticate_RobotsTxt(t *testing.T) {
-	auth := testAuthenticate()
+	auth := testAuthenticate(t)
 	req, err := http.NewRequest(http.MethodGet, "/robots.txt", nil)
 	if err != nil {
 		t.Fatal(err)
@@ -73,7 +72,7 @@ func TestAuthenticate_RobotsTxt(t *testing.T) {
 }
 
 func TestAuthenticate_Handler(t *testing.T) {
-	auth := testAuthenticate()
+	auth := testAuthenticate(t)
 
 	h := auth.Handler()
 	if h == nil {
@@ -224,17 +223,16 @@ func TestAuthenticate_SignOut(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			ctrl := gomock.NewController(t)
 			defer ctrl.Finish()
-			a := &Authenticate{
-				cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ *config.Options, _ string) (identity.Authenticator, error) {
-					return tt.provider, nil
-				})),
-				state: atomicutil.NewValue(&authenticateState{
-					sessionStore:  tt.sessionStore,
-					sharedEncoder: mock.Encoder{},
-					flow:          new(stubFlow),
-				}),
-				options: config.NewAtomicOptions(),
-			}
+			a := testAuthenticate(t)
+			a.cfg = getAuthenticateConfig(WithGetIdentityProvider(func(_ context.Context, _ oteltrace.TracerProvider, _ *config.Options, _ string) (identity.Authenticator, error) {
+				return tt.provider, nil
+			}))
+			a.state = atomicutil.NewValue(&authenticateState{
+				sessionStore:  tt.sessionStore,
+				sharedEncoder: mock.Encoder{},
+				flow:          new(stubFlow),
+			})
+			a.options = config.NewAtomicOptions()
 			if tt.signoutRedirectURL != "" {
 				opts := a.options.Load()
 				opts.SignOutRedirectURLString = tt.signoutRedirectURL
@@ -280,7 +278,7 @@ func TestAuthenticate_SignOutDoesNotRequireSession(t *testing.T) {
 
 	sessionStore := &mstore.Store{LoadError: errors.New("no session")}
 	a := &Authenticate{
-		cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ *config.Options, _ string) (identity.Authenticator, error) {
+		cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ context.Context, _ oteltrace.TracerProvider, _ *config.Options, _ string) (identity.Authenticator, error) {
 			return identity.MockProvider{}, nil
 		})),
 		state: atomicutil.NewValue(&authenticateState{
@@ -354,26 +352,24 @@ func TestAuthenticate_OAuthCallback(t *testing.T) {
 				t.Fatal(err)
 			}
 			authURL, _ := url.Parse(tt.authenticateURL)
-			a := &Authenticate{
-				cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ *config.Options, _ string) (identity.Authenticator, error) {
-					return tt.provider, nil
-				})),
-				state: atomicutil.NewValue(&authenticateState{
-					redirectURL:  authURL,
-					sessionStore: tt.session,
-					cookieCipher: aead,
-					flow:         new(stubFlow),
-				}),
-				options: config.NewAtomicOptions(),
-			}
+			a := testAuthenticate(t)
+			a.cfg = getAuthenticateConfig(WithGetIdentityProvider(func(_ context.Context, _ oteltrace.TracerProvider, _ *config.Options, _ string) (identity.Authenticator, error) {
+				return tt.provider, nil
+			}))
+			a.state = atomicutil.NewValue(&authenticateState{
+				redirectURL:  authURL,
+				sessionStore: tt.session,
+				cookieCipher: aead,
+				flow:         new(stubFlow),
+			})
+			a.options = config.NewAtomicOptions()
 			u, _ := url.Parse("/oauthGet")
 			params, _ := url.ParseQuery(u.RawQuery)
 			params.Add("error", tt.paramErr)
 			params.Add("code", tt.code)
 			nonce := cryptutil.NewBase64Key() // mock csrf
-			// (nonce|timestamp|redirect_url|encrypt(redirect_url),mac(nonce,ts))
-			b := []byte(fmt.Sprintf("%s|%d|%s", nonce, tt.ts, tt.extraMac))
-
+			// (nonce|timestamp|trace_id+flags|encrypt(redirect_url),mac(nonce,ts))
+			b := []byte(fmt.Sprintf("%s|%d||%s", nonce, tt.ts, tt.extraMac))
 			enc := cryptutil.Encrypt(a.state.Load().cookieCipher, []byte(tt.redirectURI), b)
 			b = append(b, enc...)
 			encodedState := base64.URLEncoding.EncodeToString(b)
@@ -466,20 +462,19 @@ func TestAuthenticate_SessionValidatorMiddleware(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			a := &Authenticate{
-				cfg: getAuthenticateConfig(WithGetIdentityProvider(func(_ *config.Options, _ string) (identity.Authenticator, error) {
-					return tt.provider, nil
-				})),
-				state: atomicutil.NewValue(&authenticateState{
-					cookieSecret:  cryptutil.NewKey(),
-					redirectURL:   uriParseHelper("https://authenticate.corp.beyondperimeter.com"),
-					sessionStore:  tt.session,
-					cookieCipher:  aead,
-					sharedEncoder: signer,
-					flow:          new(stubFlow),
-				}),
-				options: config.NewAtomicOptions(),
-			}
+			a := testAuthenticate(t)
+			a.cfg = getAuthenticateConfig(WithGetIdentityProvider(func(_ context.Context, _ oteltrace.TracerProvider, _ *config.Options, _ string) (identity.Authenticator, error) {
+				return tt.provider, nil
+			}))
+			a.state = atomicutil.NewValue(&authenticateState{
+				cookieSecret:  cryptutil.NewKey(),
+				redirectURL:   uriParseHelper("https://authenticate.corp.beyondperimeter.com"),
+				sessionStore:  tt.session,
+				cookieCipher:  aead,
+				sharedEncoder: signer,
+				flow:          new(stubFlow),
+			})
+			a.options = config.NewAtomicOptions()
 			r := httptest.NewRequest(http.MethodGet, "/", nil)
 			state, err := tt.session.LoadSession(r)
 			if err != nil {
@@ -512,7 +507,7 @@ func TestAuthenticate_userInfo(t *testing.T) {
 	t.Run("cookie-redirect-uri", func(t *testing.T) {
 		w := httptest.NewRecorder()
 		r := httptest.NewRequest(http.MethodGet, "https://authenticate.service.cluster.local/.pomerium/?pomerium_redirect_uri=https://www.example.com", nil)
-		var a Authenticate
+		a := testAuthenticate(t)
 		a.state = atomicutil.NewValue(&authenticateState{
 			cookieSecret: cryptutil.NewKey(),
 			flow:         new(stubFlow),
@@ -577,14 +572,13 @@ func TestAuthenticate_userInfo(t *testing.T) {
 			if !tt.validSignature {
 				f.verifySignatureErr = errors.New("bad signature")
 			}
-			a := &Authenticate{
-				options: o,
-				state: atomicutil.NewValue(&authenticateState{
-					sessionStore:  tt.sessionStore,
-					sharedEncoder: signer,
-					flow:          f,
-				}),
-			}
+			a := testAuthenticate(t)
+			a.options = o
+			a.state = atomicutil.NewValue(&authenticateState{
+				sessionStore:  tt.sessionStore,
+				sharedEncoder: signer,
+				flow:          f,
+			})
 			r := httptest.NewRequest(http.MethodGet, tt.url, nil)
 			state, err := tt.sessionStore.LoadSession(r)
 			if err != nil {
@@ -606,7 +600,7 @@ func TestAuthenticate_userInfo(t *testing.T) {
 
 func TestAuthenticate_CORS(t *testing.T) {
 	f := new(stubFlow)
-	auth := testAuthenticate()
+	auth := testAuthenticate(t)
 	state := auth.state.Load()
 	state.sessionLoader = &mstore.Store{Session: &sessions.State{}}
 	state.sharedEncoder = mock.Encoder{}
@@ -645,7 +639,7 @@ func TestAuthenticate_CORS(t *testing.T) {
 func TestSignOutBranding(t *testing.T) {
 	t.Parallel()
 
-	auth := testAuthenticate()
+	auth := testAuthenticate(t)
 	auth.state.Load().flow.(*stubFlow).verifySignatureErr = errors.New("unsigned URL")
 	auth.options.Store(&config.Options{
 		BrandingOptions: &configproto.Settings{

authenticate/identity.go

@@ -1,13 +1,16 @@
 package authenticate
 
 import (
+	"context"
+
 	"github.com/pomerium/pomerium/config"
 	"github.com/pomerium/pomerium/internal/urlutil"
 	"github.com/pomerium/pomerium/pkg/identity"
 	"github.com/pomerium/pomerium/pkg/identity/oauth"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
-func defaultGetIdentityProvider(options *config.Options, idpID string) (identity.Authenticator, error) {
+func defaultGetIdentityProvider(ctx context.Context, tracerProvider oteltrace.TracerProvider, options *config.Options, idpID string) (identity.Authenticator, error) {
 	authenticateURL, err := options.GetAuthenticateURL()
 	if err != nil {
 		return nil, err
@@ -23,7 +26,7 @@ func defaultGetIdentityProvider(options *config.Options, idpID string) (identity
 	if err != nil {
 		return nil, err
 	}
-	return identity.NewAuthenticator(oauth.Options{
+	return identity.NewAuthenticator(ctx, tracerProvider, oauth.Options{
 		RedirectURL:     redirectURL,
 		ProviderName:    idp.GetType(),
 		ProviderURL:     idp.GetUrl(),

authenticate/state.go

@@ -8,6 +8,7 @@ import (
 	"net/url"
 
 	"github.com/go-jose/go-jose/v3"
+	oteltrace "go.opentelemetry.io/otel/trace"
 	"golang.org/x/oauth2"
 
 	"github.com/pomerium/pomerium/config"
@@ -65,7 +66,9 @@ func newAuthenticateState() *authenticateState {
 
 func newAuthenticateStateFromConfig(
 	ctx context.Context,
-	cfg *config.Config, authenticateConfig *authenticateConfig,
+	tracerProvider oteltrace.TracerProvider,
+	cfg *config.Config,
+	authenticateConfig *authenticateConfig,
 ) (*authenticateState, error) {
 	err := ValidateOptions(cfg.Options)
 	if err != nil {
@@ -147,6 +150,7 @@ func newAuthenticateStateFromConfig(
 
 	if cfg.Options.UseStatelessAuthenticateFlow() {
 		state.flow, err = authenticateflow.NewStateless(ctx,
+			tracerProvider,
 			cfg,
 			cookieStore,
 			authenticateConfig.getIdentityProvider,
@@ -154,7 +158,7 @@ func newAuthenticateStateFromConfig(
 			authenticateConfig.authEventFn,
 		)
 	} else {
-		state.flow, err = authenticateflow.NewStateful(ctx, cfg, cookieStore)
+		state.flow, err = authenticateflow.NewStateful(ctx, tracerProvider, cfg, cookieStore)
 	}
 	if err != nil {
 		return nil, err

authorize/authorize.go

@@ -21,6 +21,7 @@ import (
 	"github.com/pomerium/pomerium/pkg/cryptutil"
 	"github.com/pomerium/pomerium/pkg/grpc/databroker"
 	"github.com/pomerium/pomerium/pkg/storage"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
 // Authorize struct holds
@@ -35,18 +36,25 @@ type Authorize struct {
 	// This should provide a consistent view of the data at a given server/record version and
 	// avoid partial updates.
 	stateLock sync.RWMutex
+
+	tracerProvider oteltrace.TracerProvider
+	tracer         oteltrace.Tracer
 }
 
 // New validates and creates a new Authorize service from a set of config options.
 func New(ctx context.Context, cfg *config.Config) (*Authorize, error) {
+	tracerProvider := trace.NewTracerProvider(ctx, "Authorize")
+	tracer := tracerProvider.Tracer(trace.PomeriumCoreTracer)
 	a := &Authorize{
 		currentOptions: config.NewAtomicOptions(),
 		store:          store.New(),
 		globalCache:    storage.NewGlobalCache(time.Minute),
+		tracerProvider: tracerProvider,
+		tracer:         tracer,
 	}
 	a.accessTracker = NewAccessTracker(a, accessTrackerMaxSize, accessTrackerDebouncePeriod)
 
-	state, err := newAuthorizeStateFromConfig(ctx, cfg, a.store, nil)
+	state, err := newAuthorizeStateFromConfig(ctx, tracerProvider, cfg, a.store, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -88,7 +96,7 @@ func newPolicyEvaluator(
 	ctx = log.WithContext(ctx, func(c zerolog.Context) zerolog.Context {
 		return c.Str("service", "authorize")
 	})
-	ctx, span := trace.StartSpan(ctx, "authorize.newPolicyEvaluator")
+	ctx, span := trace.Continue(ctx, "authorize.newPolicyEvaluator")
 	defer span.End()
 
 	clientCA, err := opts.DownstreamMTLS.GetCA()
@@ -142,7 +150,7 @@ func newPolicyEvaluator(
 func (a *Authorize) OnConfigChange(ctx context.Context, cfg *config.Config) {
 	currentState := a.state.Load()
 	a.currentOptions.Store(cfg.Options)
-	if state, err := newAuthorizeStateFromConfig(ctx, cfg, a.store, currentState.evaluator); err != nil {
+	if state, err := newAuthorizeStateFromConfig(ctx, a.tracerProvider, cfg, a.store, currentState.evaluator); err != nil {
 		log.Ctx(ctx).Error().Err(err).Msg("authorize: error updating state")
 	} else {
 		a.state.Store(state)

authorize/check_response.go

@@ -228,16 +228,21 @@ func (a *Authorize) requireLoginResponse(
 	// always assume https scheme
 	checkRequestURL := getCheckRequestURL(in)
 	checkRequestURL.Scheme = "https"
+	var signInURLQuery url.Values
 
+	headers := http.Header{}
+	if id := in.GetAttributes().GetRequest().GetHttp().GetHeaders()["traceparent"]; id != "" {
+		signInURLQuery = url.Values{}
+		signInURLQuery.Add("pomerium_traceparent", id)
+	}
 	redirectTo, err := state.authenticateFlow.AuthenticateSignInURL(
-		ctx, nil, &checkRequestURL, idp.GetId())
+		ctx, signInURLQuery, &checkRequestURL, idp.GetId())
 	if err != nil {
 		return nil, err
 	}
+	headers["Location"] = []string{redirectTo}
 
-	return a.deniedResponse(ctx, in, http.StatusFound, "Login", http.Header{
-		"Location": {redirectTo},
-	})
+	return a.deniedResponse(ctx, in, http.StatusFound, "Login", headers)
 }
 
 func (a *Authorize) requireWebAuthnResponse(

authorize/databroker.go

@@ -3,7 +3,6 @@ package authorize
 import (
 	"context"
 
-	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/pkg/grpc/databroker"
 	"github.com/pomerium/pomerium/pkg/grpc/session"
 	"github.com/pomerium/pomerium/pkg/grpc/user"
@@ -63,7 +62,7 @@ func (a *Authorize) getDataBrokerSessionOrServiceAccount(
 	sessionID string,
 	dataBrokerRecordVersion uint64,
 ) (s sessionOrServiceAccount, err error) {
-	ctx, span := trace.StartSpan(ctx, "authorize.getDataBrokerSessionOrServiceAccount")
+	ctx, span := a.tracer.Start(ctx, "authorize.getDataBrokerSessionOrServiceAccount")
 	defer span.End()
 
 	record, err := getDataBrokerRecord(ctx, grpcutil.GetTypeURL(new(session.Session)), sessionID, dataBrokerRecordVersion)
@@ -96,7 +95,7 @@ func (a *Authorize) getDataBrokerUser(
 	ctx context.Context,
 	userID string,
 ) (*user.User, error) {
-	ctx, span := trace.StartSpan(ctx, "authorize.getDataBrokerUser")
+	ctx, span := a.tracer.Start(ctx, "authorize.getDataBrokerUser")
 	defer span.End()
 
 	record, err := getDataBrokerRecord(ctx, grpcutil.GetTypeURL(new(user.User)), userID, 0)

authorize/evaluator/evaluator.go

@@ -199,7 +199,7 @@ func getOrCreatePolicyEvaluators(
 
 // Evaluate evaluates the rego for the given policy and generates the identity headers.
 func (e *Evaluator) Evaluate(ctx context.Context, req *Request) (*Result, error) {
-	ctx, span := trace.StartSpan(ctx, "authorize.Evaluator.Evaluate")
+	ctx, span := trace.Continue(ctx, "authorize.Evaluator.Evaluate")
 	defer span.End()
 
 	eg, ctx := errgroup.WithContext(ctx)

authorize/evaluator/headers_evaluator.go

@@ -30,7 +30,7 @@ func NewHeadersEvaluator(store *store.Store) *HeadersEvaluator {
 
 // Evaluate evaluates the headers.rego script.
 func (e *HeadersEvaluator) Evaluate(ctx context.Context, req *Request, options ...rego.EvalOption) (*HeadersResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "authorize.HeadersEvaluator.Evaluate")
+	ctx, span := trace.Continue(ctx, "authorize.HeadersEvaluator.Evaluate")
 	defer span.End()
 
 	ectx := new(rego.EvalContext)

authorize/evaluator/policy_evaluator.go

@@ -6,7 +6,7 @@ import (
 	"strings"
 
 	"github.com/open-policy-agent/opa/rego"
-	octrace "go.opencensus.io/trace"
+	"go.opentelemetry.io/otel/attribute"
 
 	"github.com/pomerium/pomerium/authorize/internal/store"
 	"github.com/pomerium/pomerium/config"
@@ -209,9 +209,9 @@ func (e *PolicyEvaluator) Evaluate(ctx context.Context, req *PolicyRequest) (*Po
 }
 
 func (e *PolicyEvaluator) evaluateQuery(ctx context.Context, req *PolicyRequest, query policyQuery) (*PolicyResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "authorize.PolicyEvaluator.evaluateQuery")
+	ctx, span := trace.Continue(ctx, "authorize.PolicyEvaluator.evaluateQuery")
 	defer span.End()
-	span.AddAttributes(octrace.StringAttribute("script_checksum", query.checksum()))
+	span.SetAttributes(attribute.String("script_checksum", query.checksum()))
 
 	rs, err := safeEval(ctx, query.PreparedEvalQuery,
 		rego.EvalInput(req),

authorize/grpc.go

@@ -19,7 +19,6 @@ import (
 	"github.com/pomerium/pomerium/internal/httputil"
 	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/internal/sessions"
-	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/urlutil"
 	"github.com/pomerium/pomerium/pkg/contextutil"
 	"github.com/pomerium/pomerium/pkg/grpc/user"
@@ -29,7 +28,7 @@ import (
 
 // Check implements the envoy auth server gRPC endpoint.
 func (a *Authorize) Check(ctx context.Context, in *envoy_service_auth_v3.CheckRequest) (*envoy_service_auth_v3.CheckResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "authorize.grpc.Check")
+	ctx, span := a.tracer.Start(ctx, "authorize.grpc.Check")
 	defer span.End()
 
 	querier := storage.NewTracingQuerier(

authorize/internal/store/store.go

@@ -14,7 +14,7 @@ import (
 	opastorage "github.com/open-policy-agent/opa/storage"
 	"github.com/open-policy-agent/opa/storage/inmem"
 	"github.com/open-policy-agent/opa/types"
-	octrace "go.opencensus.io/trace"
+	"go.opentelemetry.io/otel/attribute"
 	"google.golang.org/grpc"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/timestamppb"
@@ -145,20 +145,20 @@ func (s *Store) GetDataBrokerRecordOption() func(*rego.Rego) {
 			types.NewObject(nil, types.NewDynamicProperty(types.S, types.S)),
 		),
 	}, func(bctx rego.BuiltinContext, op1 *ast.Term, op2 *ast.Term) (*ast.Term, error) {
-		ctx, span := trace.StartSpan(bctx.Context, "rego.get_databroker_record")
+		ctx, span := trace.Continue(bctx.Context, "rego.get_databroker_record")
 		defer span.End()
 
 		recordType, ok := op1.Value.(ast.String)
 		if !ok {
 			return nil, fmt.Errorf("invalid record type: %T", op1)
 		}
-		span.AddAttributes(octrace.StringAttribute("record_type", recordType.String()))
+		span.SetAttributes(attribute.String("record_type", recordType.String()))
 
 		recordIDOrIndex, ok := op2.Value.(ast.String)
 		if !ok {
 			return nil, fmt.Errorf("invalid record id: %T", op2)
 		}
-		span.AddAttributes(octrace.StringAttribute("record_id", recordIDOrIndex.String()))
+		span.SetAttributes(attribute.String("record_id", recordIDOrIndex.String()))
 
 		msg := s.GetDataBrokerRecord(ctx, string(recordType), string(recordIDOrIndex))
 		if msg == nil {

authorize/log.go

@@ -7,10 +7,10 @@ import (
 	envoy_service_auth_v3 "github.com/envoyproxy/go-control-plane/envoy/service/auth/v3"
 	"github.com/go-jose/go-jose/v3/jwt"
 	"github.com/rs/zerolog"
+	"go.opentelemetry.io/otel/attribute"
 
 	"github.com/pomerium/pomerium/authorize/evaluator"
 	"github.com/pomerium/pomerium/internal/log"
-	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/pkg/grpc/databroker"
 	"github.com/pomerium/pomerium/pkg/grpc/session"
 	"github.com/pomerium/pomerium/pkg/grpc/user"
@@ -24,7 +24,7 @@ func (a *Authorize) logAuthorizeCheck(
 	in *envoy_service_auth_v3.CheckRequest,
 	res *evaluator.Result, s sessionOrServiceAccount, u *user.User,
 ) {
-	ctx, span := trace.StartSpan(ctx, "authorize.grpc.LogAuthorizeCheck")
+	ctx, span := a.tracer.Start(ctx, "authorize.grpc.LogAuthorizeCheck")
 	defer span.End()
 
 	hdrs := getCheckRequestHeaders(in)
@@ -39,17 +39,24 @@ func (a *Authorize) logAuthorizeCheck(
 
 	// result
 	if res != nil {
+		span.SetAttributes(attribute.Bool("result.allow", res.Allow.Value))
 		evt = evt.Bool("allow", res.Allow.Value)
+		allowReasons := res.Allow.Reasons.Strings()
 		if res.Allow.Value {
-			evt = evt.Strs("allow-why-true", res.Allow.Reasons.Strings())
+			span.SetAttributes(attribute.StringSlice("result.allow-why-true", allowReasons))
+			evt = evt.Strs("allow-why-true", allowReasons)
 		} else {
-			evt = evt.Strs("allow-why-false", res.Allow.Reasons.Strings())
+			span.SetAttributes(attribute.StringSlice("result.allow-why-false", allowReasons))
+			evt = evt.Strs("allow-why-false", allowReasons)
 		}
 		evt = evt.Bool("deny", res.Deny.Value)
+		denyReasons := res.Deny.Reasons.Strings()
 		if res.Deny.Value {
-			evt = evt.Strs("deny-why-true", res.Deny.Reasons.Strings())
+			span.SetAttributes(attribute.StringSlice("result.deny-why-true", denyReasons))
+			evt = evt.Strs("deny-why-true", denyReasons)
 		} else {
-			evt = evt.Strs("deny-why-false", res.Deny.Reasons.Strings())
+			span.SetAttributes(attribute.StringSlice("result.deny-why-false", denyReasons))
+			evt = evt.Strs("deny-why-false", denyReasons)
 		}
 	}
 

authorize/state.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"net/url"
 
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
+	oteltrace "go.opentelemetry.io/otel/trace"
 	googlegrpc "google.golang.org/grpc"
 
 	"github.com/pomerium/pomerium/authorize/evaluator"
@@ -32,7 +34,10 @@ type authorizeState struct {
 
 func newAuthorizeStateFromConfig(
 	ctx context.Context,
-	cfg *config.Config, store *store.Store, previousPolicyEvaluator *evaluator.Evaluator,
+	tracerProvider oteltrace.TracerProvider,
+	cfg *config.Config,
+	store *store.Store,
+	previousPolicyEvaluator *evaluator.Evaluator,
 ) (*authorizeState, error) {
 	if err := validateOptions(cfg.Options); err != nil {
 		return nil, fmt.Errorf("authorize: bad options: %w", err)
@@ -62,7 +67,7 @@ func newAuthorizeStateFromConfig(
 		InstallationID: cfg.Options.InstallationID,
 		ServiceName:    cfg.Options.Services,
 		SignedJWTKey:   sharedKey,
-	})
+	}, googlegrpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(tracerProvider))))
 	if err != nil {
 		return nil, fmt.Errorf("authorize: error creating databroker connection: %w", err)
 	}
@@ -75,9 +80,9 @@ func newAuthorizeStateFromConfig(
 	}
 
 	if cfg.Options.UseStatelessAuthenticateFlow() {
-		state.authenticateFlow, err = authenticateflow.NewStateless(ctx, cfg, nil, nil, nil, nil)
+		state.authenticateFlow, err = authenticateflow.NewStateless(ctx, tracerProvider, cfg, nil, nil, nil, nil)
 	} else {
-		state.authenticateFlow, err = authenticateflow.NewStateful(ctx, cfg, nil)
+		state.authenticateFlow, err = authenticateflow.NewStateful(ctx, tracerProvider, cfg, nil)
 	}
 	if err != nil {
 		return nil, err

cmd/pomerium/main.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/pomerium/pomerium/config"
 	"github.com/pomerium/pomerium/internal/log"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/version"
 	_ "github.com/pomerium/pomerium/internal/zero/bootstrap/writers/filesystem"
 	_ "github.com/pomerium/pomerium/internal/zero/bootstrap/writers/k8s"
@@ -30,9 +31,15 @@ func main() {
 	}
 	root.AddCommand(zero_cmd.BuildRootCmd())
 	root.PersistentFlags().StringVar(&configFile, "config", "", "Specify configuration file location")
-
-	ctx := context.Background()
 	log.SetLevel(zerolog.InfoLevel)
+	ctx := trace.Options{
+		RemoteClient: trace.NewSyncClient(trace.NewRemoteClientFromEnv()),
+	}.NewContext(context.Background())
+	defer func() {
+		if err := trace.ShutdownContext(ctx); err != nil {
+			log.Error().Err(err).Send()
+		}
+	}()
 	runFn := run
 	if zero_cmd.IsManagedMode(configFile) {
 		runFn = zero_cmd.Run

config/envoyconfig/bootstrap.go

@@ -3,6 +3,7 @@ package envoyconfig
 import (
 	"context"
 	"fmt"
+	"math"
 	"os"
 	"path/filepath"
 
@@ -37,7 +38,7 @@ func (b *Builder) BuildBootstrap(
 	cfg *config.Config,
 	fullyStatic bool,
 ) (bootstrap *envoy_config_bootstrap_v3.Bootstrap, err error) {
-	ctx, span := trace.StartSpan(ctx, "envoyconfig.Builder.BuildBootstrap")
+	ctx, span := trace.Continue(ctx, "envoyconfig.Builder.BuildBootstrap")
 	defer span.End()
 
 	bootstrap = new(envoy_config_bootstrap_v3.Bootstrap)
@@ -52,7 +53,7 @@ func (b *Builder) BuildBootstrap(
 		return nil, fmt.Errorf("error building bootstrap dynamic resources: %w", err)
 	}
 
-	bootstrap.LayeredRuntime, err = b.BuildBootstrapLayeredRuntime()
+	bootstrap.LayeredRuntime, err = b.BuildBootstrapLayeredRuntime(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("error building bootstrap layered runtime: %w", err)
 	}
@@ -148,7 +149,13 @@ func (b *Builder) BuildBootstrapDynamicResources(
 }
 
 // BuildBootstrapLayeredRuntime builds the layered runtime for the envoy bootstrap.
-func (b *Builder) BuildBootstrapLayeredRuntime() (*envoy_config_bootstrap_v3.LayeredRuntime, error) {
+func (b *Builder) BuildBootstrapLayeredRuntime(ctx context.Context) (*envoy_config_bootstrap_v3.LayeredRuntime, error) {
+	flushIntervalMs := 5000
+	minFlushSpans := 3
+	if trace.DebugFlagsFromContext(ctx).Check(trace.EnvoyFlushEverySpan) {
+		minFlushSpans = 1
+		flushIntervalMs = math.MaxInt32
+	}
 	layer, err := structpb.NewStruct(map[string]any{
 		"re2": map[string]any{
 			"max_program_size": map[string]any{
@@ -156,6 +163,21 @@ func (b *Builder) BuildBootstrapLayeredRuntime() (*envoy_config_bootstrap_v3.Lay
 				"warn_level":  1024,
 			},
 		},
+		"tracing": map[string]any{
+			"opentelemetry": map[string]any{
+				"flush_interval_ms": flushIntervalMs,
+				// For most requests, envoy generates 3 spans:
+				// - ingress (downstream->envoy)
+				// - ext_authz check request (envoy->pomerium)
+				// - egress (envoy->upstream)
+				// The default value is 5, which usually leads to delayed exports.
+				// This can be set lower, e.g. 1 to have envoy export every span
+				// individually (useful for testing), but 3 is a reasonable default.
+				// If set to 1, also set flush_interval_ms to a very large number to
+				// effectively disable it.
+				"min_flush_spans": minFlushSpans,
+			},
+		},
 	})
 	if err != nil {
 		return nil, fmt.Errorf("envoyconfig: failed to create layered runtime layer: %w", err)
@@ -180,7 +202,7 @@ func (b *Builder) BuildBootstrapStaticResources(
 	cfg *config.Config,
 	fullyStatic bool,
 ) (staticResources *envoy_config_bootstrap_v3.Bootstrap_StaticResources, err error) {
-	ctx, span := trace.StartSpan(ctx, "envoyconfig.Builder.BuildBootstrapStaticResources")
+	ctx, span := trace.Continue(ctx, "envoyconfig.Builder.BuildBootstrapStaticResources")
 	defer span.End()
 
 	staticResources = new(envoy_config_bootstrap_v3.Bootstrap_StaticResources)

config/envoyconfig/bootstrap_test.go

@@ -36,7 +36,7 @@ func TestBuilder_BuildBootstrapAdmin(t *testing.T) {
 
 func TestBuilder_BuildBootstrapLayeredRuntime(t *testing.T) {
 	b := New("localhost:1111", "localhost:2222", "localhost:3333", filemgr.NewManager(), nil)
-	staticCfg, err := b.BuildBootstrapLayeredRuntime()
+	staticCfg, err := b.BuildBootstrapLayeredRuntime(context.Background())
 	assert.NoError(t, err)
 	testutil.AssertProtoJSONEqual(t, `
 		{ "layers": [{
@@ -47,6 +47,12 @@ func TestBuilder_BuildBootstrapLayeredRuntime(t *testing.T) {
 						"error_level": 1048576,
 						"warn_level": 1024
 					}
+				},
+				"tracing": {
+					"opentelemetry": {
+						"flush_interval_ms": 5000,
+						"min_flush_spans": 3
+					}
 				}
 			}
 		}] }

config/envoyconfig/clusters.go

@@ -26,7 +26,7 @@ import (
 
 // BuildClusters builds envoy clusters from the given config.
 func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*envoy_config_cluster_v3.Cluster, error) {
-	ctx, span := trace.StartSpan(ctx, "envoyconfig.Builder.BuildClusters")
+	ctx, span := trace.Continue(ctx, "envoyconfig.Builder.BuildClusters")
 	defer span.End()
 
 	grpcURLs := []*url.URL{{
@@ -104,13 +104,6 @@ func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*env
 		envoyAdminCluster,
 	}
 
-	tracingCluster, err := buildTracingCluster(cfg.Options)
-	if err != nil {
-		return nil, err
-	} else if tracingCluster != nil {
-		clusters = append(clusters, tracingCluster)
-	}
-
 	if config.IsProxy(cfg.Options.Services) {
 		for policy := range cfg.Options.GetAllPolicies() {
 			if len(policy.To) > 0 {

config/envoyconfig/listeners.go

@@ -20,7 +20,7 @@ func (b *Builder) BuildListeners(
 	cfg *config.Config,
 	fullyStatic bool,
 ) ([]*envoy_config_listener_v3.Listener, error) {
-	ctx, span := trace.StartSpan(ctx, "envoyconfig.Builder.BuildListeners")
+	ctx, span := trace.Continue(ctx, "envoyconfig.Builder.BuildListeners")
 	defer span.End()
 
 	var listeners []*envoy_config_listener_v3.Listener

config/envoyconfig/listeners_grpc.go

@@ -73,6 +73,9 @@ func (b *Builder) buildGRPCHTTPConnectionManagerFilter() *envoy_config_listener_
 				PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: fmt.Sprintf("/%s/", svc)},
 				Grpc:          &envoy_config_route_v3.RouteMatch_GrpcRouteMatchOptions{},
 			},
+			Decorator: &envoy_config_route_v3.Decorator{
+				Operation: fmt.Sprintf("pomerium-control-plane-grpc %s", svc),
+			},
 			Action: &envoy_config_route_v3.Route_Route{
 				Route: &envoy_config_route_v3.RouteAction{
 					ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{

config/envoyconfig/listeners_main.go

@@ -10,7 +10,6 @@ import (
 	envoy_config_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	envoy_extensions_access_loggers_grpc_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/grpc/v3"
 	envoy_extensions_filters_network_http_connection_manager "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
-	envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3"
 	"google.golang.org/protobuf/types/known/durationpb"
 	"google.golang.org/protobuf/types/known/wrapperspb"
 
@@ -177,20 +176,11 @@ func (b *Builder) buildMainHTTPConnectionManagerFilter(
 		maxStreamDuration = durationpb.New(cfg.Options.WriteTimeout)
 	}
 
-	tracingProvider, err := buildTracingHTTP(cfg.Options)
-	if err != nil {
-		return nil, err
-	}
-
 	localReply, err := b.buildLocalReplyConfig(cfg.Options)
 	if err != nil {
 		return nil, err
 	}
 
-	sampleRate := 1.0
-	if cfg.Options.TracingSampleRate != nil {
-		sampleRate = *cfg.Options.TracingSampleRate
-	}
 	mgr := &envoy_extensions_filters_network_http_connection_manager.HttpConnectionManager{
 		AlwaysSetRequestIdInResponse: true,
 		StatPrefix:                   "ingress",
@@ -202,10 +192,6 @@ func (b *Builder) buildMainHTTPConnectionManagerFilter(
 		},
 		HttpProtocolOptions: http1ProtocolOptions,
 		RequestTimeout:      durationpb.New(cfg.Options.ReadTimeout),
-		Tracing: &envoy_extensions_filters_network_http_connection_manager.HttpConnectionManager_Tracing{
-			RandomSampling: &envoy_type_v3.Percent{Value: max(0.0, min(1.0, sampleRate)) * 100},
-			Provider:       tracingProvider,
-		},
 		// See https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for
 		UseRemoteAddress:  &wrapperspb.BoolValue{Value: true},
 		SkipXffAppend:     cfg.Options.SkipXffAppend,
@@ -228,6 +214,8 @@ func (b *Builder) buildMainHTTPConnectionManagerFilter(
 		mgr.CodecType = cfg.Options.GetCodecType().ToEnvoy()
 	}
 
+	applyTracingConfig(ctx, mgr, cfg.Options)
+
 	if fullyStatic {
 		routeConfiguration, err := b.buildMainRouteConfiguration(ctx, cfg)
 		if err != nil {

config/envoyconfig/listeners_test.go

@@ -149,7 +149,9 @@ func Test_buildMainHTTPConnectionManagerFilter(t *testing.T) {
 	options.SkipXffAppend = true
 	options.XffNumTrustedHops = 1
 	options.AuthenticateURLString = "https://authenticate.example.com"
+	options.TracingProvider = "otlp"
 	filter, err := b.buildMainHTTPConnectionManagerFilter(context.Background(), &config.Config{Options: options}, false, false)
 	require.NoError(t, err)
+
 	testutil.AssertProtoJSONEqual(t, testData(t, "main_http_connection_manager_filter.json", nil), filter)
 }

config/envoyconfig/outbound.go

@@ -101,6 +101,9 @@ func (b *Builder) buildOutboundRoutes() []*envoy_config_route_v3.Route {
 					PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: prefix},
 					Grpc:          &envoy_config_route_v3.RouteMatch_GrpcRouteMatchOptions{},
 				},
+				Decorator: &envoy_config_route_v3.Decorator{
+					Operation: fmt.Sprintf("Outbound (grpc): %s %s", def.Cluster, prefix),
+				},
 				Action: &envoy_config_route_v3.Route_Route{
 					Route: &envoy_config_route_v3.RouteAction{
 						ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
@@ -123,6 +126,9 @@ func (b *Builder) buildOutboundRoutes() []*envoy_config_route_v3.Route {
 		Match: &envoy_config_route_v3.RouteMatch{
 			PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: "/envoy/stats/prometheus"},
 		},
+		Decorator: &envoy_config_route_v3.Decorator{
+			Operation: "Outbound: envoy-metrics /envoy/stats/prometheus/*",
+		},
 		Action: &envoy_config_route_v3.Route_Route{
 			Route: &envoy_config_route_v3.RouteAction{
 				ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{

config/envoyconfig/outbound_test.go

@@ -15,6 +15,9 @@ func Test_buildOutboundRoutes(t *testing.T) {
 				"grpc": {},
 				"prefix": "/envoy.service.auth.v3.Authorization/"
 			},
+			"decorator": {
+				"operation": "Outbound (grpc): pomerium-authorize /envoy.service.auth.v3.Authorization/"
+			},
 			"name": "pomerium-authorize",
 			"route": {
 				"autoHostRewrite": true,
@@ -28,6 +31,9 @@ func Test_buildOutboundRoutes(t *testing.T) {
 				"grpc": {},
 				"prefix": "/databroker.DataBrokerService/"
 			},
+			"decorator": {
+				"operation": "Outbound (grpc): pomerium-databroker /databroker.DataBrokerService/"
+			},
 			"name": "pomerium-databroker",
 			"route": {
 				"autoHostRewrite": true,
@@ -41,6 +47,9 @@ func Test_buildOutboundRoutes(t *testing.T) {
 				"grpc": {},
 				"prefix": "/registry.Registry/"
 			},
+			"decorator": {
+				"operation": "Outbound (grpc): pomerium-databroker /registry.Registry/"
+			},
 			"name": "pomerium-databroker",
 			"route": {
 				"autoHostRewrite": true,
@@ -54,6 +63,9 @@ func Test_buildOutboundRoutes(t *testing.T) {
 				"grpc": {},
 				"prefix": "/"
 			},
+			"decorator": {
+				"operation": "Outbound (grpc): pomerium-control-plane-grpc /"
+			},
 			"name": "pomerium-control-plane-grpc",
 			"route": {
 				"autoHostRewrite": true,
@@ -66,6 +78,9 @@ func Test_buildOutboundRoutes(t *testing.T) {
 			"match": {
 				"prefix": "/envoy/stats/prometheus"
 			},
+			"decorator": {
+				"operation": "Outbound: envoy-metrics /envoy/stats/prometheus/*"
+			},
 			"name": "envoy-metrics",
 			"route": {
 				"cluster": "pomerium-envoy-admin",

config/envoyconfig/route_configurations.go

@@ -20,7 +20,7 @@ func (b *Builder) BuildRouteConfigurations(
 	ctx context.Context,
 	cfg *config.Config,
 ) ([]*envoy_config_route_v3.RouteConfiguration, error) {
-	ctx, span := trace.StartSpan(ctx, "envoyconfig.Builder.BuildRouteConfigurations")
+	ctx, span := trace.Continue(ctx, "envoyconfig.Builder.BuildRouteConfigurations")
 	defer span.End()
 
 	var routeConfigurations []*envoy_config_route_v3.RouteConfiguration

config/envoyconfig/route_configurations_test.go

@@ -51,6 +51,10 @@ func TestBuilder_buildMainRouteConfiguration(t *testing.T) {
 					`+protojson.Format(b.buildControlPlanePrefixRoute(cfg.Options, "/.well-known/pomerium/"))+`,
 					{
 						"name": "policy-0",
+						"decorator": {
+							"operation": "ingress: ${method} ${host}${path}",
+							"propagate": false
+						},
 						"match": {
 							"headers": [
 								{ "name": ":authority", "stringMatch": { "safeRegex": { "regex": "^(.*)\\.example\\.com$" } }}
@@ -104,6 +108,10 @@ func TestBuilder_buildMainRouteConfiguration(t *testing.T) {
 					},
 					{
 						"name": "policy-0",
+						"decorator": {
+							"operation": "ingress: ${method} ${host}${path}",
+							"propagate": false
+						},
 						"match": {
 							"headers": [
 								{ "name": ":authority", "stringMatch": { "safeRegex": { "regex": "^(.*)\\.example\\.com:443$" } }}

config/envoyconfig/routes.go

@@ -114,6 +114,9 @@ func (b *Builder) buildControlPlanePathRoute(
 		Match: &envoy_config_route_v3.RouteMatch{
 			PathSpecifier: &envoy_config_route_v3.RouteMatch_Path{Path: path},
 		},
+		Decorator: &envoy_config_route_v3.Decorator{
+			Operation: "internal: ${method} ${host}${path}",
+		},
 		Action: &envoy_config_route_v3.Route_Route{
 			Route: &envoy_config_route_v3.RouteAction{
 				ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
@@ -138,6 +141,9 @@ func (b *Builder) buildControlPlanePrefixRoute(
 		Match: &envoy_config_route_v3.RouteMatch{
 			PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: prefix},
 		},
+		Decorator: &envoy_config_route_v3.Decorator{
+			Operation: "internal: ${method} ${host}${path}",
+		},
 		Action: &envoy_config_route_v3.Route_Route{
 			Route: &envoy_config_route_v3.RouteAction{
 				ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
@@ -269,8 +275,12 @@ func (b *Builder) buildRouteForPolicyAndMatch(
 	}
 
 	route := &envoy_config_route_v3.Route{
-		Name:                   name,
-		Match:                  match,
+		Name:  name,
+		Match: match,
+		Decorator: &envoy_config_route_v3.Decorator{
+			Operation: "ingress: ${method} ${host}${path}",
+			Propagate: wrapperspb.Bool(false),
+		},
 		Metadata:               &envoy_config_core_v3.Metadata{},
 		RequestHeadersToRemove: getRequestHeadersToRemove(cfg.Options, policy),
 		ResponseHeadersToAdd:   toEnvoyHeaders(cfg.Options.GetSetResponseHeadersForPolicy(policy)),

config/envoyconfig/routes_test.go

@@ -59,6 +59,9 @@ func Test_buildPomeriumHTTPRoutes(t *testing.T) {
 	routeString := func(typ, name string) string {
 		str := `{
 			"name": "pomerium-` + typ + `-` + name + `",
+			"decorator": {
+				"operation": "internal: ${method} ${host}${path}"
+			},
 			"match": {
 				"` + typ + `": "` + name + `"
 			},
@@ -135,6 +138,9 @@ func Test_buildControlPlanePathRoute(t *testing.T) {
 	testutil.AssertProtoJSONEqual(t, `
 		{
 			"name": "pomerium-path-/hello/world",
+			"decorator": {
+				"operation": "internal: ${method} ${host}${path}"
+			},
 			"match": {
 				"path": "/hello/world"
 			},
@@ -179,6 +185,9 @@ func Test_buildControlPlanePrefixRoute(t *testing.T) {
 	testutil.AssertProtoJSONEqual(t, `
 		{
 			"name": "pomerium-prefix-/hello/world/",
+			"decorator": {
+				"operation": "internal: ${method} ${host}${path}"
+			},
 			"match": {
 				"prefix": "/hello/world/"
 			},
@@ -406,6 +415,10 @@ func Test_buildPolicyRoutes(t *testing.T) {
 		[
 			{
 				"name": "policy-1",
+				"decorator": {
+					"operation": "ingress: ${method} ${host}${path}",
+					"propagate": false
+				},
 				"match": {
 					"prefix": "/"
 				},
@@ -476,6 +489,10 @@ func Test_buildPolicyRoutes(t *testing.T) {
 			},
 			{
 				"name": "policy-2",
+				"decorator": {
+					"operation": "ingress: ${method} ${host}${path}",
+					"propagate": false
+				},
 				"match": {
 					"path": "/some/path"
 				},
@@ -547,6 +564,10 @@ func Test_buildPolicyRoutes(t *testing.T) {
 			},
 			{
 				"name": "policy-3",
+				"decorator": {
+					"operation": "ingress: ${method} ${host}${path}",
+					"propagate": false
+				},
 				"match": {
 					"prefix": "/some/prefix/"
 				},
@@ -617,6 +638,10 @@ func Test_buildPolicyRoutes(t *testing.T) {
 			},
 			{
 				"name": "policy-4",
+				"decorator": {
+					"operation": "ingress: ${method} ${host}${path}",
+					"propagate": false
+				},
 				"match": {
 					"safeRegex": {
 						"regex": "^/[a]+$"
@@ -689,6 +714,10 @@ func Test_buildPolicyRoutes(t *testing.T) {
 			},
 			{
 				"name": "policy-5",
+				"decorator": {
+					"operation": "ingress: ${method} ${host}${path}",
+					"propagate": false
+				},
 				"match": {
 					"prefix": "/some/prefix/"
 				},
@@ -760,6 +789,10 @@ func Test_buildPolicyRoutes(t *testing.T) {
 			},
 			{
 				"name": "policy-6",
+				"decorator": {
+					"operation": "ingress: ${method} ${host}${path}",
+					"propagate": false
+				},
 				"match": {
 					"path": "/some/path"
 				},
@@ -830,6 +863,10 @@ func Test_buildPolicyRoutes(t *testing.T) {
 			},
 			{
 				"name": "policy-7",
+				"decorator": {
+					"operation": "ingress: ${method} ${host}${path}",
+					"propagate": false
+				},
 				"match": {
 					"path": "/some/path"
 				},
@@ -901,6 +938,10 @@ func Test_buildPolicyRoutes(t *testing.T) {
 			},
 			{
 				"name": "policy-8",
+				"decorator": {
+					"operation": "ingress: ${method} ${host}${path}",
+					"propagate": false
+				},
 				"match": {
 					"path": "/websocket-timeout"
 				},
@@ -994,6 +1035,10 @@ func Test_buildPolicyRoutes(t *testing.T) {
 			[
 				{
 					"name": "policy-0",
+					"decorator": {
+						"operation": "ingress: ${method} ${host}${path}",
+						"propagate": false
+					},
 					"match": {
 						"prefix": "/"
 					},
@@ -1083,6 +1128,10 @@ func Test_buildPolicyRoutes(t *testing.T) {
 		[
 			{
 				"name": "policy-0",
+				"decorator": {
+					"operation": "ingress: ${method} ${host}${path}",
+					"propagate": false
+				},
 				"match": {
 					"connectMatcher": {}
 				},
@@ -1155,6 +1204,10 @@ func Test_buildPolicyRoutes(t *testing.T) {
 			},
 			{
 				"name": "policy-1",
+				"decorator": {
+					"operation": "ingress: ${method} ${host}${path}",
+					"propagate": false
+				},
 				"match": {
 					"connectMatcher": {}
 				},
@@ -1248,6 +1301,10 @@ func Test_buildPolicyRoutes(t *testing.T) {
 		[
 			{
 				"name": "policy-0",
+				"decorator": {
+					"operation": "ingress: ${method} ${host}${path}",
+					"propagate": false
+				},
 				"match": {
 					"connectMatcher": {}
 				},
@@ -1345,6 +1402,10 @@ func Test_buildPolicyRoutes(t *testing.T) {
 			[
 				{
 					"name": "policy-0",
+					"decorator": {
+						"operation": "ingress: ${method} ${host}${path}",
+						"propagate": false
+					},
 					"match": {
 						"prefix": "/"
 					},
@@ -1440,6 +1501,10 @@ func Test_buildPolicyRoutes(t *testing.T) {
 			[
 				{
 					"name": "policy-0",
+					"decorator": {
+						"operation": "ingress: ${method} ${host}${path}",
+						"propagate": false
+					},
 					"match": {
 						"prefix": "/"
 					},
@@ -1588,6 +1653,10 @@ func Test_buildPolicyRoutesRewrite(t *testing.T) {
 		[
 			{
 				"name": "policy-0",
+				"decorator": {
+					"operation": "ingress: ${method} ${host}${path}",
+					"propagate": false
+				},
 				"match": {
 					"prefix": "/"
 				},
@@ -1659,6 +1728,10 @@ func Test_buildPolicyRoutesRewrite(t *testing.T) {
 			},
 			{
 				"name": "policy-1",
+				"decorator": {
+					"operation": "ingress: ${method} ${host}${path}",
+					"propagate": false
+				},
 				"match": {
 					"prefix": "/"
 				},
@@ -1730,6 +1803,10 @@ func Test_buildPolicyRoutesRewrite(t *testing.T) {
 			},
 			{
 				"name": "policy-2",
+				"decorator": {
+					"operation": "ingress: ${method} ${host}${path}",
+					"propagate": false
+				},
 				"match": {
 					"prefix": "/"
 				},
@@ -1806,6 +1883,10 @@ func Test_buildPolicyRoutesRewrite(t *testing.T) {
 			},
 			{
 				"name": "policy-3",
+				"decorator": {
+					"operation": "ingress: ${method} ${host}${path}",
+					"propagate": false
+				},
 				"match": {
 					"prefix": "/"
 				},
@@ -1877,6 +1958,10 @@ func Test_buildPolicyRoutesRewrite(t *testing.T) {
 			},
 			{
 				"name": "policy-4",
+				"decorator": {
+					"operation": "ingress: ${method} ${host}${path}",
+					"propagate": false
+				},
 				"match": {
 					"prefix": "/"
 				},
@@ -1948,6 +2033,10 @@ func Test_buildPolicyRoutesRewrite(t *testing.T) {
 			},
 			{
 				"name": "policy-5",
+				"decorator": {
+					"operation": "ingress: ${method} ${host}${path}",
+					"propagate": false
+				},
 				"match": {
 					"prefix": "/"
 				},

config/envoyconfig/testdata/main_http_connection_manager_filter.json

@@ -23,6 +23,14 @@
     "commonHttpProtocolOptions": {
       "idleTimeout": "300s"
     },
+    "earlyHeaderMutationExtensions": [
+      {
+        "name": "envoy.http.early_header_mutation.trace_context",
+        "typedConfig": {
+          "@type": "type.googleapis.com/pomerium.extensions.TraceContext"
+        }
+      }
+    ],
     "httpFilters": [
       {
         "name": "envoy.filters.http.lua",
@@ -52,13 +60,13 @@
             },
             "timeout": "10s"
           },
-          "metadataContextNamespaces": [
-            "com.pomerium.client-certificate-info"
-          ],
+          "transportApiVersion": "V3",
           "statusOnError": {
             "code": "InternalServerError"
           },
-          "transportApiVersion": "V3"
+          "metadataContextNamespaces": [
+            "com.pomerium.client-certificate-info"
+          ]
         }
       },
       {
@@ -75,7 +83,7 @@
         "typedConfig": {
           "@type": "type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua",
           "defaultSourceCode": {
-            "inlineString": "function has_prefix(str, prefix)\n    return str ~= nil and str:sub(1, #prefix) == prefix\nend\n\nfunction remove_pomerium_cookie(cookie_name, cookie)\n    local result = \"\"\n    for c in cookie:gmatch(\"([^;]+)\") do\n        c = c:gsub(\"^ +\",\"\")\n        local name = c:match(\"^([^=]+)\")\n        if name ~= cookie_name then\n            if string.len(result) \u003e 0 then\n                result = result .. \"; \" .. c\n            else\n                result = result .. c\n            end\n        end\n    end\n    return result\nend\n\nfunction envoy_on_request(request_handle)\n    local headers = request_handle:headers()\n    local metadata = request_handle:metadata()\n\n    local remove_cookie_name = metadata:get(\"remove_pomerium_cookie\")\n    if remove_cookie_name then\n        local cookie = headers:get(\"cookie\")\n        if cookie ~= nil then\n            local newcookie = remove_pomerium_cookie(remove_cookie_name, cookie)\n            headers:replace(\"cookie\", newcookie)\n        end\n    end\n\n    local remove_authorization = metadata:get(\"remove_pomerium_authorization\")\n    if remove_authorization then\n        local authorization = headers:get(\"authorization\")\n        local authorization_prefix = \"Pomerium \"\n        if has_prefix(authorization, authorization_prefix) then\n            headers:remove(\"authorization\")\n        end\n\n        headers:remove('x-pomerium-authorization')\n    end\nend\n\nfunction envoy_on_response(response_handle) end\n"
+            "inlineString": "function has_prefix(str, prefix)\n    return str ~= nil and str:sub(1, #prefix) == prefix\nend\n\nfunction remove_pomerium_cookie(cookie_name, cookie)\n    local result = \"\"\n    for c in cookie:gmatch(\"([^;]+)\") do\n        c = c:gsub(\"^ +\",\"\")\n        local name = c:match(\"^([^=]+)\")\n        if name ~= cookie_name then\n            if string.len(result) > 0 then\n                result = result .. \"; \" .. c\n            else\n                result = result .. c\n            end\n        end\n    end\n    return result\nend\n\nfunction envoy_on_request(request_handle)\n    local headers = request_handle:headers()\n    local metadata = request_handle:metadata()\n\n    local remove_cookie_name = metadata:get(\"remove_pomerium_cookie\")\n    if remove_cookie_name then\n        local cookie = headers:get(\"cookie\")\n        if cookie ~= nil then\n            local newcookie = remove_pomerium_cookie(remove_cookie_name, cookie)\n            headers:replace(\"cookie\", newcookie)\n        end\n    end\n\n    local remove_authorization = metadata:get(\"remove_pomerium_authorization\")\n    if remove_authorization then\n        local authorization = headers:get(\"authorization\")\n        local authorization_prefix = \"Pomerium \"\n        if has_prefix(authorization, authorization_prefix) then\n            headers:remove(\"authorization\")\n        end\n\n        headers:remove('x-pomerium-authorization')\n    end\nend\n\nfunction envoy_on_response(response_handle) end\n"
           }
         }
       },
@@ -169,6 +177,13 @@
         }
       ]
     },
+    "requestIdExtension": {
+      "typedConfig": {
+        "@type": "type.googleapis.com/pomerium.extensions.UuidxRequestIdConfig",
+        "packTraceReason": true,
+        "useRequestIdForTraceSampling": true
+      }
+    },
     "requestTimeout": "30s",
     "normalizePath": true,
     "rds": {
@@ -183,7 +198,33 @@
     "tracing": {
       "randomSampling": {
         "value": 100
-      }
+      },
+      "verbose": true,
+      "maxPathTagLength": 1024,
+      "provider": {
+        "name": "envoy.tracers.pomerium_otel",
+        "typedConfig": {
+          "@type": "type.googleapis.com/pomerium.extensions.OpenTelemetryConfig",
+          "grpcService": {
+            "envoyGrpc": {
+              "clusterName": "pomerium-control-plane-grpc"
+            }
+          },
+          "serviceName": "Envoy",
+          "resourceDetectors": [
+            {
+              "name": "envoy.tracers.opentelemetry.resource_detectors.static_config",
+              "typedConfig": {
+                "@type": "type.googleapis.com/envoy.extensions.tracers.opentelemetry.resource_detectors.v3.StaticConfigResourceDetectorConfig",
+                "attributes": {
+                  "pomerium.envoy": "true"
+                }
+              }
+            }
+          ]
+        }
+      },
+      "spawnUpstreamSpan": true
     },
     "useRemoteAddress": true,
     "xffNumTrustedHops": 1,

config/envoyconfig/tracing.go

@@ -1,134 +1,151 @@
 package envoyconfig
 
 import (
-	"fmt"
-	"net"
-
-	envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
-	envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
-	envoy_config_trace_v3 "github.com/envoyproxy/go-control-plane/envoy/config/trace/v3"
-	"google.golang.org/protobuf/types/known/durationpb"
+	"context"
+	"os"
+	"strconv"
 
+	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	tracev3 "github.com/envoyproxy/go-control-plane/envoy/config/trace/v3"
+	envoy_extensions_filters_http_header_to_metadata "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/header_to_metadata/v3"
+	envoy_extensions_filters_network_http_connection_manager "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	envoy_extensions_tracers_otel "github.com/envoyproxy/go-control-plane/envoy/extensions/tracers/opentelemetry/resource_detectors/v3"
+	metadatav3 "github.com/envoyproxy/go-control-plane/envoy/type/metadata/v3"
+	envoy_tracing_v3 "github.com/envoyproxy/go-control-plane/envoy/type/tracing/v3"
+	envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3"
+	extensions_trace_context "github.com/pomerium/envoy-custom/api/extensions/http/early_header_mutation/trace_context"
+	extensions_uuidx "github.com/pomerium/envoy-custom/api/extensions/request_id/uuidx"
+	extensions_pomerium_otel "github.com/pomerium/envoy-custom/api/extensions/tracers/pomerium_otel"
 	"github.com/pomerium/pomerium/config"
 	"github.com/pomerium/pomerium/internal/telemetry/trace"
-	"github.com/pomerium/pomerium/pkg/protoutil"
+	"google.golang.org/protobuf/types/known/wrapperspb"
 )
 
-func buildTracingCluster(options *config.Options) (*envoy_config_cluster_v3.Cluster, error) {
-	tracingOptions, err := config.NewTracingOptions(options)
-	if err != nil {
-		return nil, fmt.Errorf("envoyconfig: invalid tracing config: %w", err)
+func isTracingEnabled(cfg *config.Options) bool {
+	if os.Getenv("OTEL_SDK_DISABLED") == "true" {
+		return false
 	}
-
-	switch tracingOptions.Provider {
-	case trace.DatadogTracingProviderName:
-		addr, _ := parseAddress("127.0.0.1:8126")
-
-		if options.TracingDatadogAddress != "" {
-			addr, err = parseAddress(options.TracingDatadogAddress)
-			if err != nil {
-				return nil, fmt.Errorf("envoyconfig: invalid tracing datadog address: %w", err)
-			}
-		}
-
-		endpoints := []*envoy_config_endpoint_v3.LbEndpoint{{
-			HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{
-				Endpoint: &envoy_config_endpoint_v3.Endpoint{
-					Address: addr,
-				},
-			},
-		}}
-
-		return &envoy_config_cluster_v3.Cluster{
-			Name: "datadog-apm",
-			ConnectTimeout: &durationpb.Duration{
-				Seconds: 5,
-			},
-			ClusterDiscoveryType: getClusterDiscoveryType(endpoints),
-			LbPolicy:             envoy_config_cluster_v3.Cluster_ROUND_ROBIN,
-			LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{
-				ClusterName: "datadog-apm",
-				Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{{
-					LbEndpoints: endpoints,
-				}},
-			},
-		}, nil
-	case trace.ZipkinTracingProviderName:
-		host := tracingOptions.ZipkinEndpoint.Host
-		if _, port, _ := net.SplitHostPort(host); port == "" {
-			if tracingOptions.ZipkinEndpoint.Scheme == "https" {
-				host = net.JoinHostPort(host, "443")
-			} else {
-				host = net.JoinHostPort(host, "80")
-			}
-		}
-
-		addr, err := parseAddress(host)
-		if err != nil {
-			return nil, fmt.Errorf("envoyconfig: invalid tracing zipkin address: %w", err)
-		}
-
-		endpoints := []*envoy_config_endpoint_v3.LbEndpoint{{
-			HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{
-				Endpoint: &envoy_config_endpoint_v3.Endpoint{
-					Address: addr,
-				},
-			},
-		}}
-		return &envoy_config_cluster_v3.Cluster{
-			Name: "zipkin",
-			ConnectTimeout: &durationpb.Duration{
-				Seconds: 5,
-			},
-			ClusterDiscoveryType: getClusterDiscoveryType(endpoints),
-			LbPolicy:             envoy_config_cluster_v3.Cluster_ROUND_ROBIN,
-			LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{
-				ClusterName: "zipkin",
-				Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{{
-					LbEndpoints: endpoints,
-				}},
-			},
-		}, nil
-	default:
-		return nil, nil
+	switch cfg.TracingProvider {
+	case "none", "noop": // explicitly disabled from config
+		return false
+	case "": // unset
+		return trace.IsEnabledViaEnvironment()
+	default: // set to a non-empty value
+		return !trace.IsDisabledViaEnvironment()
 	}
 }
 
-func buildTracingHTTP(options *config.Options) (*envoy_config_trace_v3.Tracing_Http, error) {
-	tracingOptions, err := config.NewTracingOptions(options)
-	if err != nil {
-		return nil, fmt.Errorf("invalid tracing config: %w", err)
+func applyTracingConfig(
+	ctx context.Context,
+	mgr *envoy_extensions_filters_network_http_connection_manager.HttpConnectionManager,
+	opts *config.Options,
+) {
+	if !isTracingEnabled(opts) {
+		return
+	}
+	mgr.EarlyHeaderMutationExtensions = []*envoy_config_core_v3.TypedExtensionConfig{
+		{
+			Name:        "envoy.http.early_header_mutation.trace_context",
+			TypedConfig: marshalAny(&extensions_trace_context.TraceContext{}),
+		},
+	}
+	mgr.RequestIdExtension = &envoy_extensions_filters_network_http_connection_manager.RequestIDExtension{
+		TypedConfig: marshalAny(&extensions_uuidx.UuidxRequestIdConfig{
+			PackTraceReason:              wrapperspb.Bool(true),
+			UseRequestIdForTraceSampling: wrapperspb.Bool(true),
+		}),
 	}
 
-	switch tracingOptions.Provider {
-	case trace.DatadogTracingProviderName:
-		tracingTC := protoutil.NewAny(&envoy_config_trace_v3.DatadogConfig{
-			CollectorCluster: "datadog-apm",
-			ServiceName:      tracingOptions.Service,
-		})
-		return &envoy_config_trace_v3.Tracing_Http{
-			Name: "envoy.tracers.datadog",
-			ConfigType: &envoy_config_trace_v3.Tracing_Http_TypedConfig{
-				TypedConfig: tracingTC,
-			},
-		}, nil
-	case trace.ZipkinTracingProviderName:
-		path := tracingOptions.ZipkinEndpoint.Path
-		if path == "" {
-			path = "/"
+	maxPathTagLength := uint32(1024)
+	if value, ok := os.LookupEnv("OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT"); ok {
+		if num, err := strconv.ParseUint(value, 10, 32); err == nil {
+			maxPathTagLength = max(64, uint32(num))
 		}
-		tracingTC := protoutil.NewAny(&envoy_config_trace_v3.ZipkinConfig{
-			CollectorCluster:         "zipkin",
-			CollectorEndpoint:        path,
-			CollectorEndpointVersion: envoy_config_trace_v3.ZipkinConfig_HTTP_JSON,
-		})
-		return &envoy_config_trace_v3.Tracing_Http{
-			Name: "envoy.tracers.zipkin",
-			ConfigType: &envoy_config_trace_v3.Tracing_Http_TypedConfig{
-				TypedConfig: tracingTC,
+	}
+	sampleRate := 1.0
+	if value, ok := os.LookupEnv("OTEL_TRACES_SAMPLER_ARG"); ok {
+		if rate, err := strconv.ParseFloat(value, 64); err == nil {
+			sampleRate = rate
+		}
+	}
+	if opts.TracingSampleRate != nil {
+		sampleRate = *opts.TracingSampleRate
+	}
+	mgr.Tracing = &envoy_extensions_filters_network_http_connection_manager.HttpConnectionManager_Tracing{
+		RandomSampling:    &envoy_type_v3.Percent{Value: max(0.0, min(1.0, sampleRate)) * 100},
+		Verbose:           true,
+		SpawnUpstreamSpan: wrapperspb.Bool(true),
+		Provider: &tracev3.Tracing_Http{
+			Name: "envoy.tracers.pomerium_otel",
+			ConfigType: &tracev3.Tracing_Http_TypedConfig{
+				TypedConfig: marshalAny(&extensions_pomerium_otel.OpenTelemetryConfig{
+					GrpcService: &envoy_config_core_v3.GrpcService{
+						TargetSpecifier: &envoy_config_core_v3.GrpcService_EnvoyGrpc_{
+							EnvoyGrpc: &envoy_config_core_v3.GrpcService_EnvoyGrpc{
+								ClusterName: "pomerium-control-plane-grpc",
+							},
+						},
+					},
+					ServiceName: "Envoy",
+					ResourceDetectors: []*envoy_config_core_v3.TypedExtensionConfig{
+						{
+							Name: "envoy.tracers.opentelemetry.resource_detectors.static_config",
+							TypedConfig: marshalAny(&envoy_extensions_tracers_otel.StaticConfigResourceDetectorConfig{
+								Attributes: map[string]string{
+									"pomerium.envoy": "true",
+								},
+							}),
+						},
+					},
+				}),
 			},
-		}, nil
-	default:
-		return nil, nil
+		},
+		// this allows full URLs to be displayed in traces, they are otherwise truncated
+		MaxPathTagLength: wrapperspb.UInt32(maxPathTagLength),
+	}
+
+	debugFlags := trace.DebugFlagsFromContext(ctx)
+	if debugFlags.Check(trace.TrackSpanReferences) {
+		mgr.HttpFilters = append([]*envoy_extensions_filters_network_http_connection_manager.HttpFilter{
+			{
+				Name: "envoy.filters.http.header_to_metadata",
+				ConfigType: &envoy_extensions_filters_network_http_connection_manager.HttpFilter_TypedConfig{
+					TypedConfig: marshalAny(&envoy_extensions_filters_http_header_to_metadata.Config{
+						RequestRules: []*envoy_extensions_filters_http_header_to_metadata.Config_Rule{
+							{
+								Header: "x-pomerium-external-parent-span",
+								OnHeaderPresent: &envoy_extensions_filters_http_header_to_metadata.Config_KeyValuePair{
+									MetadataNamespace: "pomerium.internal",
+									Key:               "external-parent-span",
+								},
+								Remove: true,
+							},
+						},
+					}),
+				},
+			},
+		}, mgr.HttpFilters...)
+		mgr.Tracing.CustomTags = append(mgr.Tracing.CustomTags, &envoy_tracing_v3.CustomTag{
+			Tag: "pomerium.external-parent-span",
+			Type: &envoy_tracing_v3.CustomTag_Metadata_{
+				Metadata: &envoy_tracing_v3.CustomTag_Metadata{
+					Kind: &metadatav3.MetadataKind{
+						Kind: &metadatav3.MetadataKind_Request_{
+							Request: &metadatav3.MetadataKind_Request{},
+						},
+					},
+					MetadataKey: &metadatav3.MetadataKey{
+						Key: "pomerium.internal",
+						Path: []*metadatav3.MetadataKey_PathSegment{
+							{
+								Segment: &metadatav3.MetadataKey_PathSegment_Key{
+									Key: "external-parent-span",
+								},
+							},
+						},
+					},
+				},
+			},
+		})
 	}
 }

config/envoyconfig/tracing_test.go

@@ -1,135 +0,0 @@
-package envoyconfig
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/pomerium/pomerium/config"
-	"github.com/pomerium/pomerium/internal/testutil"
-)
-
-func TestBuildTracingCluster(t *testing.T) {
-	t.Run("datadog", func(t *testing.T) {
-		c, err := buildTracingCluster(&config.Options{
-			TracingProvider: "datadog",
-		})
-		require.NoError(t, err)
-		testutil.AssertProtoJSONEqual(t, `
-			{
-				"name": "datadog-apm",
-				"type": "STATIC",
-				"connectTimeout": "5s",
-				"loadAssignment": {
-					"clusterName": "datadog-apm",
-					"endpoints": [{
-						"lbEndpoints": [{
-							"endpoint": {
-								"address": {
-									"socketAddress": {
-										"address": "127.0.0.1",
-										"portValue": 8126
-									}
-								}
-							}
-						}]
-					}]
-				}
-			}
-		`, c)
-
-		c, err = buildTracingCluster(&config.Options{
-			TracingProvider:       "datadog",
-			TracingDatadogAddress: "example.com:8126",
-		})
-		require.NoError(t, err)
-		testutil.AssertProtoJSONEqual(t, `
-			{
-				"name": "datadog-apm",
-				"type": "STRICT_DNS",
-				"connectTimeout": "5s",
-				"loadAssignment": {
-					"clusterName": "datadog-apm",
-					"endpoints": [{
-						"lbEndpoints": [{
-							"endpoint": {
-								"address": {
-									"socketAddress": {
-										"address": "example.com",
-										"portValue": 8126
-									}
-								}
-							}
-						}]
-					}]
-				}
-			}
-		`, c)
-	})
-	t.Run("zipkin", func(t *testing.T) {
-		c, err := buildTracingCluster(&config.Options{
-			TracingProvider: "zipkin",
-			ZipkinEndpoint:  "https://example.com/api/v2/spans",
-		})
-		require.NoError(t, err)
-		testutil.AssertProtoJSONEqual(t, `
-			{
-				"name": "zipkin",
-				"type": "STRICT_DNS",
-				"connectTimeout": "5s",
-				"loadAssignment": {
-					"clusterName": "zipkin",
-					"endpoints": [{
-						"lbEndpoints": [{
-							"endpoint": {
-								"address": {
-									"socketAddress": {
-										"address": "example.com",
-										"portValue": 443
-									}
-								}
-							}
-						}]
-					}]
-				}
-			}
-		`, c)
-	})
-}
-
-func TestBuildTracingHTTP(t *testing.T) {
-	t.Run("datadog", func(t *testing.T) {
-		h, err := buildTracingHTTP(&config.Options{
-			TracingProvider: "datadog",
-		})
-		require.NoError(t, err)
-		testutil.AssertProtoJSONEqual(t, `
-			{
-				"name": "envoy.tracers.datadog",
-				"typedConfig": {
-					"@type": "type.googleapis.com/envoy.config.trace.v3.DatadogConfig",
-					"collectorCluster": "datadog-apm",
-					"serviceName": "pomerium"
-				}
-			}
-		`, h)
-	})
-	t.Run("zipkin", func(t *testing.T) {
-		h, err := buildTracingHTTP(&config.Options{
-			TracingProvider: "zipkin",
-			ZipkinEndpoint:  "https://example.com/api/v2/spans",
-		})
-		require.NoError(t, err)
-		testutil.AssertProtoJSONEqual(t, `
-			{
-				"name": "envoy.tracers.zipkin",
-				"typedConfig": {
-					"@type": "type.googleapis.com/envoy.config.trace.v3.ZipkinConfig",
-					"collectorCluster": "zipkin",
-					"collectorEndpoint": "/api/v2/spans",
-					"collectorEndpointVersion": "HTTP_JSON"
-				}
-			}
-		`, h)
-	})
-}

config/log.go

@@ -20,11 +20,6 @@ func NewLogManager(ctx context.Context, src Source) *LogManager {
 	return mgr
 }
 
-// Close closes the log manager.
-func (mgr *LogManager) Close() error {
-	return nil
-}
-
 // OnConfigChange is called whenever configuration changes.
 func (mgr *LogManager) OnConfigChange(_ context.Context, cfg *Config) {
 	if cfg == nil || cfg.Options == nil {

config/metrics.go

@@ -45,11 +45,6 @@ func NewMetricsManager(ctx context.Context, src Source) *MetricsManager {
 	return mgr
 }
 
-// Close closes any underlying http server.
-func (mgr *MetricsManager) Close() error {
-	return nil
-}
-
 // OnConfigChange updates the metrics manager when configuration is changed.
 func (mgr *MetricsManager) OnConfigChange(ctx context.Context, cfg *Config) {
 	mgr.mu.Lock()

config/options.go

@@ -210,30 +210,10 @@ type Options struct {
 	MetricsClientCA           string `mapstructure:"metrics_client_ca" yaml:"metrics_client_ca,omitempty"`
 	MetricsClientCAFile       string `mapstructure:"metrics_client_ca_file" yaml:"metrics_client_ca_file,omitempty"`
 
-	// Tracing shared settings
-	TracingProvider   string   `mapstructure:"tracing_provider" yaml:"tracing_provider,omitempty"`
-	TracingSampleRate *float64 `mapstructure:"tracing_sample_rate" yaml:"tracing_sample_rate,omitempty"`
-
-	TracingOTLPEndpoint string `mapstructure:"tracing_otlp_endpoint" yaml:"tracing_otlp_endpoint,omitempty"`
-	TracingOTLPProtocol string `mapstructure:"tracing_otlp_protocol" yaml:"tracing_otlp_protocol,omitempty"`
-
-	// Datadog tracing address
-	TracingDatadogAddress string `mapstructure:"tracing_datadog_address" yaml:"tracing_datadog_address,omitempty"`
-
-	//  Jaeger
-	//
-	// CollectorEndpoint is the full url to the Jaeger HTTP Thrift collector.
-	// For example, http://localhost:14268/api/traces
-	TracingJaegerCollectorEndpoint string `mapstructure:"tracing_jaeger_collector_endpoint" yaml:"tracing_jaeger_collector_endpoint,omitempty"`
-	// AgentEndpoint instructs exporter to send spans to jaeger-agent at this address.
-	// For example, localhost:6831.
-	TracingJaegerAgentEndpoint string `mapstructure:"tracing_jaeger_agent_endpoint" yaml:"tracing_jaeger_agent_endpoint,omitempty"`
-
-	// Zipkin
-	//
-	// ZipkinEndpoint configures the zipkin collector URI
-	// Example: http://zipkin:9411/api/v2/spans
-	ZipkinEndpoint string `mapstructure:"tracing_zipkin_endpoint" yaml:"tracing_zipkin_endpoint"`
+	TracingSampleRate   *float64 `mapstructure:"tracing_sample_rate" yaml:"tracing_sample_rate,omitempty"`
+	TracingProvider     string   `mapstructure:"tracing_provider" yaml:"tracing_provider,omitempty"`
+	TracingOTLPEndpoint string   `mapstructure:"tracing_otlp_endpoint" yaml:"tracing_otlp_endpoint,omitempty"`
+	TracingOTLPProtocol string   `mapstructure:"tracing_otlp_protocol" yaml:"tracing_otlp_protocol,omitempty"`
 
 	// GRPC Service Settings
 
@@ -1527,10 +1507,6 @@ func (o *Options) ApplySettings(ctx context.Context, certsIndex *cryptutil.Certi
 	set(&o.TracingOTLPEndpoint, settings.TracingOtlpEndpoint)
 	set(&o.TracingOTLPProtocol, settings.TracingOtlpProtocol)
 	setOptional(&o.TracingSampleRate, settings.TracingSampleRate)
-	set(&o.TracingDatadogAddress, settings.TracingDatadogAddress)
-	set(&o.TracingJaegerCollectorEndpoint, settings.TracingJaegerCollectorEndpoint)
-	set(&o.TracingJaegerAgentEndpoint, settings.TracingJaegerAgentEndpoint)
-	set(&o.ZipkinEndpoint, settings.TracingZipkinEndpoint)
 	set(&o.GRPCAddr, settings.GrpcAddress)
 	setOptional(&o.GRPCInsecure, settings.GrpcInsecure)
 	setDuration(&o.GRPCClientTimeout, settings.GrpcClientTimeout)
@@ -1619,10 +1595,6 @@ func (o *Options) ToProto() *config.Config {
 	settings.TracingSampleRate = o.TracingSampleRate
 	copySrcToOptionalDest(&settings.TracingOtlpEndpoint, &o.TracingOTLPEndpoint)
 	copySrcToOptionalDest(&settings.TracingOtlpProtocol, &o.TracingOTLPProtocol)
-	copySrcToOptionalDest(&settings.TracingDatadogAddress, &o.TracingDatadogAddress)
-	copySrcToOptionalDest(&settings.TracingJaegerCollectorEndpoint, &o.TracingJaegerCollectorEndpoint)
-	copySrcToOptionalDest(&settings.TracingJaegerAgentEndpoint, &o.TracingJaegerAgentEndpoint)
-	copySrcToOptionalDest(&settings.TracingZipkinEndpoint, &o.ZipkinEndpoint)
 	copySrcToOptionalDest(&settings.GrpcAddress, &o.GRPCAddr)
 	settings.GrpcInsecure = o.GRPCInsecure
 	copyOptionalDuration(&settings.GrpcClientTimeout, o.GRPCClientTimeout)

config/options_check.go

@@ -31,14 +31,18 @@ var reKeyPath = regexp.MustCompile(`\[\d+\]`)
 var (
 	// options that were deprecated in the config
 	removedConfigFields = map[string]string{
-		"client_ca":                       "https://www.pomerium.com/docs/deploy/core/upgrading#new-downstream-mtls-settings",
-		"client_ca_file":                  "https://www.pomerium.com/docs/deploy/core/upgrading#new-downstream-mtls-settings",
-		"idp_service_account":             "https://docs.pomerium.com/docs/overview/upgrading#idp-directory-sync",
-		"idp_refresh_directory_timeout":   "https://docs.pomerium.com/docs/overview/upgrading#idp-directory-sync",
-		"idp_refresh_directory_interval":  "https://docs.pomerium.com/docs/overview/upgrading#idp-directory-sync",
-		"idp_qps":                         "https://docs.pomerium.com/docs/overview/upgrading#idp-directory-sync",
-		"routes.allowed_groups":           "https://docs.pomerium.com/docs/overview/upgrading#idp-groups-policy",
-		"routes.set_authorization_header": "https://www.pomerium.com/docs/deploy/core/upgrading#set-authorization-header",
+		"client_ca":                         "https://www.pomerium.com/docs/deploy/core/upgrading#new-downstream-mtls-settings",
+		"client_ca_file":                    "https://www.pomerium.com/docs/deploy/core/upgrading#new-downstream-mtls-settings",
+		"idp_service_account":               "https://docs.pomerium.com/docs/overview/upgrading#idp-directory-sync",
+		"idp_refresh_directory_timeout":     "https://docs.pomerium.com/docs/overview/upgrading#idp-directory-sync",
+		"idp_refresh_directory_interval":    "https://docs.pomerium.com/docs/overview/upgrading#idp-directory-sync",
+		"idp_qps":                           "https://docs.pomerium.com/docs/overview/upgrading#idp-directory-sync",
+		"routes.allowed_groups":             "https://docs.pomerium.com/docs/overview/upgrading#idp-groups-policy",
+		"routes.set_authorization_header":   "https://www.pomerium.com/docs/deploy/core/upgrading#set-authorization-header",
+		"tracing_datadog_address":           "https://docs.pomerium.com/docs/overview/upgrading#removed-tracing-options",
+		"tracing_jaeger_collector_endpoint": "https://docs.pomerium.com/docs/overview/upgrading#removed-tracing-options",
+		"tracing_jaeger_agent_endpoint":     "https://docs.pomerium.com/docs/overview/upgrading#removed-tracing-options",
+		"tracing_zipkin_endpoint":           "https://docs.pomerium.com/docs/overview/upgrading#removed-tracing-options",
 	}
 
 	ignoreConfigFields = map[string]struct{}{

config/trace.go

@@ -1,129 +0,0 @@
-package config
-
-import (
-	"context"
-	"fmt"
-	"reflect"
-	"sync"
-
-	"github.com/rs/zerolog"
-
-	"github.com/pomerium/pomerium/internal/log"
-	"github.com/pomerium/pomerium/internal/telemetry"
-	"github.com/pomerium/pomerium/internal/telemetry/trace"
-	"github.com/pomerium/pomerium/internal/urlutil"
-)
-
-// TracingOptions are the options for tracing.
-type TracingOptions = trace.TracingOptions
-
-// NewTracingOptions builds a new TracingOptions from core Options
-func NewTracingOptions(o *Options) (*TracingOptions, error) {
-	sampleRate := 1.0
-	if o.TracingSampleRate != nil {
-		sampleRate = *o.TracingSampleRate
-	}
-	tracingOpts := TracingOptions{
-		Provider:            o.TracingProvider,
-		Service:             telemetry.ServiceName(o.Services),
-		JaegerAgentEndpoint: o.TracingJaegerAgentEndpoint,
-		SampleRate:          sampleRate,
-	}
-
-	switch o.TracingProvider {
-	case trace.DatadogTracingProviderName:
-		tracingOpts.DatadogAddress = o.TracingDatadogAddress
-	case trace.JaegerTracingProviderName:
-		if o.TracingJaegerCollectorEndpoint != "" {
-			jaegerCollectorEndpoint, err := urlutil.ParseAndValidateURL(o.TracingJaegerCollectorEndpoint)
-			if err != nil {
-				return nil, fmt.Errorf("config: invalid jaeger endpoint url: %w", err)
-			}
-			tracingOpts.JaegerCollectorEndpoint = jaegerCollectorEndpoint
-			tracingOpts.JaegerAgentEndpoint = o.TracingJaegerAgentEndpoint
-		}
-	case trace.ZipkinTracingProviderName:
-		zipkinEndpoint, err := urlutil.ParseAndValidateURL(o.ZipkinEndpoint)
-		if err != nil {
-			return nil, fmt.Errorf("config: invalid zipkin endpoint url: %w", err)
-		}
-		tracingOpts.ZipkinEndpoint = zipkinEndpoint
-	case "":
-		return &TracingOptions{}, nil
-	default:
-		return nil, fmt.Errorf("config: provider %s unknown", o.TracingProvider)
-	}
-
-	return &tracingOpts, nil
-}
-
-// A TraceManager manages setting up a trace exporter based on configuration options.
-type TraceManager struct {
-	mu        sync.Mutex
-	traceOpts *TracingOptions
-	provider  trace.Provider
-}
-
-// NewTraceManager creates a new TraceManager.
-func NewTraceManager(ctx context.Context, src Source) *TraceManager {
-	ctx = log.WithContext(ctx, func(c zerolog.Context) zerolog.Context {
-		return c.Str("service", "trace_manager")
-	})
-	mgr := &TraceManager{}
-	src.OnConfigChange(ctx, mgr.OnConfigChange)
-	mgr.OnConfigChange(ctx, src.GetConfig())
-	return mgr
-}
-
-// Close closes any underlying trace exporter.
-func (mgr *TraceManager) Close() error {
-	mgr.mu.Lock()
-	defer mgr.mu.Unlock()
-
-	var err error
-	if mgr.provider != nil {
-		err = mgr.provider.Unregister()
-	}
-	return err
-}
-
-// OnConfigChange updates the manager whenever the configuration is changed.
-func (mgr *TraceManager) OnConfigChange(ctx context.Context, cfg *Config) {
-	mgr.mu.Lock()
-	defer mgr.mu.Unlock()
-
-	traceOpts, err := NewTracingOptions(cfg.Options)
-	if err != nil {
-		log.Ctx(ctx).Error().Err(err).Msg("trace: failed to build tracing options")
-		return
-	}
-
-	if reflect.DeepEqual(traceOpts, mgr.traceOpts) {
-		log.Ctx(ctx).Debug().Msg("no change detected in trace options")
-		return
-	}
-	mgr.traceOpts = traceOpts
-
-	if mgr.provider != nil {
-		_ = mgr.provider.Unregister()
-		mgr.provider = nil
-	}
-
-	if !traceOpts.Enabled() {
-		return
-	}
-
-	log.Ctx(ctx).Info().Interface("options", traceOpts).Msg("trace: starting exporter")
-
-	mgr.provider, err = trace.GetProvider(traceOpts)
-	if err != nil {
-		log.Ctx(ctx).Error().Err(err).Msg("trace: failed to register exporter")
-		return
-	}
-
-	err = mgr.provider.Register(traceOpts)
-	if err != nil {
-		log.Ctx(ctx).Error().Err(err).Msg("trace: failed to register exporter")
-		return
-	}
-}

config/trace_client.go

@@ -0,0 +1,44 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+)
+
+var ErrNoTracingConfig = errors.New("no tracing config")
+
+func NewTraceClientFromOptions(opts *Options) (otlptrace.Client, error) {
+	switch opts.TracingProvider {
+	case "otlp":
+		endpoint := opts.TracingOTLPEndpoint
+		protocol := opts.TracingOTLPProtocol
+		if protocol == "" && endpoint != "" {
+			// treat this field as equivalent to OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
+			protocol = trace.BestEffortProtocolFromOTLPEndpoint(opts.TracingOTLPEndpoint, true)
+		}
+		switch strings.ToLower(strings.TrimSpace(protocol)) {
+		case "grpc":
+			return otlptracegrpc.NewClient(
+				otlptracegrpc.WithEndpointURL(endpoint),
+			), nil
+		case "http/protobuf", "":
+			return otlptracehttp.NewClient(
+				otlptracehttp.WithEndpointURL(endpoint),
+			), nil
+		default:
+			return nil, fmt.Errorf(`unknown otlp trace exporter protocol %q, expected "grpc" or "http/protobuf"\n`, protocol)
+		}
+	case "none", "noop":
+		return trace.NoopClient{}, nil
+	case "":
+		return nil, ErrNoTracingConfig
+	default:
+		return nil, fmt.Errorf(`unknown tracing provider %q, expected one of ["otlp"]`, opts.TracingProvider)
+	}
+}

config/trace_test.go

@@ -1,159 +0,0 @@
-package config
-
-import (
-	"context"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"net/url"
-	"testing"
-	"time"
-
-	"github.com/google/go-cmp/cmp"
-	"github.com/stretchr/testify/assert"
-
-	"github.com/pomerium/pomerium/internal/telemetry/trace"
-)
-
-func Test_NewTracingOptions(t *testing.T) {
-	tests := []struct {
-		name    string
-		opts    *Options
-		want    *TracingOptions
-		wantErr bool
-	}{
-		{
-			"datadog_good",
-			&Options{TracingProvider: "datadog"},
-			&TracingOptions{Provider: "datadog", Service: "pomerium", SampleRate: 1},
-			false,
-		},
-		{
-			"jaeger_good",
-			&Options{TracingProvider: "jaeger", TracingJaegerAgentEndpoint: "foo", TracingJaegerCollectorEndpoint: "http://foo", Services: ServiceAll},
-			&TracingOptions{Provider: "jaeger", JaegerAgentEndpoint: "foo", JaegerCollectorEndpoint: &url.URL{Scheme: "http", Host: "foo"}, Service: "pomerium", SampleRate: 1},
-			false,
-		},
-		{
-			"jaeger_bad",
-			&Options{TracingProvider: "jaeger", TracingJaegerAgentEndpoint: "foo", TracingJaegerCollectorEndpoint: "badurl"},
-			nil,
-			true,
-		},
-		{
-			"zipkin_good",
-			&Options{TracingProvider: "zipkin", ZipkinEndpoint: "https://foo/api/v1/spans", Services: ServiceAuthorize},
-			&TracingOptions{Provider: "zipkin", ZipkinEndpoint: &url.URL{Scheme: "https", Host: "foo", Path: "/api/v1/spans"}, Service: "pomerium-authorize", SampleRate: 1},
-			false,
-		},
-		{
-			"zipkin_bad",
-			&Options{TracingProvider: "zipkin", ZipkinEndpoint: "notaurl"},
-			nil,
-			true,
-		},
-		{
-			"noprovider",
-			&Options{},
-			&TracingOptions{},
-			false,
-		},
-		{
-			"fakeprovider",
-			&Options{TracingProvider: "fake"},
-			nil,
-			true,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := NewTracingOptions(tt.opts)
-			assert.NotEqual(t, err == nil, tt.wantErr, "unexpected error value")
-			assert.Empty(t, cmp.Diff(tt.want, got))
-		})
-	}
-}
-
-func Test_TracingEnabled(t *testing.T) {
-	tests := []struct {
-		name string
-		opts *TracingOptions
-		want bool
-	}{
-		{"enabled", &TracingOptions{Provider: "zipkin"}, true},
-		{"not enabled", &TracingOptions{}, false},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			assert.Equal(t, tt.want, tt.opts.Enabled(), "unexpected tracing state")
-		})
-	}
-}
-
-func TestTraceManager(t *testing.T) {
-	ctx, clearTimeout := context.WithTimeout(context.Background(), time.Second*30)
-	defer clearTimeout()
-
-	type Request struct {
-		URL  string
-		Name string
-	}
-
-	incoming := make(chan Request, 100)
-
-	h := http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
-		var objs []struct {
-			Name string
-		}
-		json.NewDecoder(r.Body).Decode(&objs)
-		for _, obj := range objs {
-			incoming <- Request{Name: obj.Name, URL: r.Host}
-		}
-	})
-
-	srv1 := httptest.NewServer(h)
-	defer srv1.Close()
-	srv2 := httptest.NewServer(h)
-	defer srv2.Close()
-
-	src := NewStaticSource(&Config{Options: &Options{
-		TracingProvider: "zipkin",
-		ZipkinEndpoint:  srv1.URL,
-	}})
-
-	_ = NewTraceManager(ctx, src)
-
-	_, span := trace.StartSpan(ctx, "Example")
-	span.End()
-
-	src.SetConfig(ctx, &Config{Options: &Options{
-		TracingProvider: "zipkin",
-		ZipkinEndpoint:  srv2.URL,
-	}})
-
-	_, span = trace.StartSpan(ctx, "Example")
-	span.End()
-
-	expect := map[Request]struct{}{
-		{Name: "example", URL: srv1.Listener.Addr().String()}: {},
-		{Name: "example", URL: srv2.Listener.Addr().String()}: {},
-	}
-
-	for len(expect) > 0 {
-		var req Request
-		select {
-		case <-ctx.Done():
-			t.Error("timeout waiting for requests")
-			return
-		case req = <-incoming:
-		}
-
-		if _, ok := expect[req]; ok {
-			delete(expect, req)
-		} else {
-			t.Error("unexpected request", req)
-			return
-		}
-	}
-}

databroker/cache.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/rs/zerolog"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/metadata"
@@ -18,7 +19,7 @@ import (
 	"github.com/pomerium/pomerium/internal/atomicutil"
 	"github.com/pomerium/pomerium/internal/events"
 	"github.com/pomerium/pomerium/internal/log"
-	"github.com/pomerium/pomerium/internal/telemetry"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/version"
 	"github.com/pomerium/pomerium/pkg/cryptutil"
 	"github.com/pomerium/pomerium/pkg/envoy/files"
@@ -28,6 +29,7 @@ import (
 	"github.com/pomerium/pomerium/pkg/identity"
 	"github.com/pomerium/pomerium/pkg/identity/legacymanager"
 	"github.com/pomerium/pomerium/pkg/identity/manager"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
 // DataBroker represents the databroker service. The databroker service is a simple interface
@@ -43,6 +45,8 @@ type DataBroker struct {
 	localGRPCServer     *grpc.Server
 	localGRPCConnection *grpc.ClientConn
 	sharedKey           *atomicutil.Value[[]byte]
+	tracerProvider      oteltrace.TracerProvider
+	tracer              oteltrace.Tracer
 }
 
 type Options struct {
@@ -87,9 +91,12 @@ func New(ctx context.Context, cfg *config.Config, eventsMgr *events.Manager, opt
 		),
 	)
 
+	tracerProvider := trace.NewTracerProvider(ctx, "Data Broker")
+	tracer := tracerProvider.Tracer(trace.PomeriumCoreTracer)
 	// No metrics handler because we have one in the control plane.  Add one
 	// if we no longer register with that grpc Server
 	localGRPCServer := grpc.NewServer(
+		grpc.StatsHandler(otelgrpc.NewServerHandler(otelgrpc.WithTracerProvider(tracerProvider))),
 		grpc.ChainStreamInterceptor(log.StreamServerInterceptor(log.Ctx(ctx)), si),
 		grpc.ChainUnaryInterceptor(log.UnaryServerInterceptor(log.Ctx(ctx)), ui),
 	)
@@ -100,12 +107,11 @@ func New(ctx context.Context, cfg *config.Config, eventsMgr *events.Manager, opt
 	}
 
 	sharedKeyValue := atomicutil.NewValue(sharedKey)
-	clientStatsHandler := telemetry.NewGRPCClientStatsHandler(cfg.Options.Services)
 	clientDialOptions := []grpc.DialOption{
 		grpc.WithInsecure(),
-		grpc.WithChainUnaryInterceptor(clientStatsHandler.UnaryInterceptor, grpcutil.WithUnarySignedJWT(sharedKeyValue.Load)),
+		grpc.WithChainUnaryInterceptor(grpcutil.WithUnarySignedJWT(sharedKeyValue.Load)),
 		grpc.WithChainStreamInterceptor(grpcutil.WithStreamSignedJWT(sharedKeyValue.Load)),
-		grpc.WithStatsHandler(clientStatsHandler.Handler),
+		grpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(tracerProvider))),
 	}
 
 	ctx = log.WithContext(ctx, func(c zerolog.Context) zerolog.Context {
@@ -120,7 +126,7 @@ func New(ctx context.Context, cfg *config.Config, eventsMgr *events.Manager, opt
 		return nil, err
 	}
 
-	dataBrokerServer, err := newDataBrokerServer(ctx, cfg)
+	dataBrokerServer, err := newDataBrokerServer(ctx, tracerProvider, cfg)
 	if err != nil {
 		return nil, err
 	}
@@ -133,6 +139,8 @@ func New(ctx context.Context, cfg *config.Config, eventsMgr *events.Manager, opt
 		localGRPCConnection: localGRPCConnection,
 		sharedKey:           sharedKeyValue,
 		eventsMgr:           eventsMgr,
+		tracerProvider:      tracerProvider,
+		tracer:              tracer,
 	}
 	c.Register(c.localGRPCServer)
 
@@ -202,7 +210,7 @@ func (c *DataBroker) update(ctx context.Context, cfg *config.Config) error {
 	}, c.legacyManagerOptions...)
 
 	if cfg.Options.SupportsUserRefresh() {
-		authenticator, err := identity.NewAuthenticator(oauthOptions)
+		authenticator, err := identity.NewAuthenticator(ctx, c.tracerProvider, oauthOptions)
 		if err != nil {
 			log.Ctx(ctx).Error().Err(err).Msg("databroker: failed to create authenticator")
 		} else {

databroker/databroker.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 
+	oteltrace "go.opentelemetry.io/otel/trace"
 	"google.golang.org/protobuf/types/known/emptypb"
 
 	"github.com/pomerium/pomerium/config"
@@ -23,7 +24,7 @@ type dataBrokerServer struct {
 }
 
 // newDataBrokerServer creates a new databroker service server.
-func newDataBrokerServer(ctx context.Context, cfg *config.Config) (*dataBrokerServer, error) {
+func newDataBrokerServer(ctx context.Context, tracerProvider oteltrace.TracerProvider, cfg *config.Config) (*dataBrokerServer, error) {
 	srv := &dataBrokerServer{
 		sharedKey: atomicutil.NewValue([]byte{}),
 	}
@@ -33,7 +34,7 @@ func newDataBrokerServer(ctx context.Context, cfg *config.Config) (*dataBrokerSe
 		return nil, err
 	}
 
-	srv.server = databroker.New(ctx, opts...)
+	srv.server = databroker.New(ctx, tracerProvider, opts...)
 	srv.setKey(cfg)
 	return srv, nil
 }

databroker/databroker_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -29,7 +30,7 @@ var lis *bufconn.Listener
 func init() {
 	lis = bufconn.Listen(bufSize)
 	s := grpc.NewServer()
-	internalSrv := internal_databroker.New(context.Background())
+	internalSrv := internal_databroker.New(context.Background(), trace.NewNoopTracerProvider())
 	srv := &dataBrokerServer{server: internalSrv, sharedKey: atomicutil.NewValue([]byte{})}
 	databroker.RegisterDataBrokerServiceServer(s, srv)
 

go.mod

@@ -4,11 +4,8 @@ go 1.23.0
 
 require (
 	cloud.google.com/go/storage v1.49.0
-	contrib.go.opencensus.io/exporter/jaeger v0.2.1
 	contrib.go.opencensus.io/exporter/prometheus v0.4.2
-	contrib.go.opencensus.io/exporter/zipkin v0.1.2
 	github.com/CAFxX/httpcompression v0.0.9
-	github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20200406135749-5c268882acf0
 	github.com/VictoriaMetrics/fastcache v1.12.2
 	github.com/aws/aws-sdk-go-v2 v1.32.7
 	github.com/aws/aws-sdk-go-v2/config v1.28.7
@@ -20,7 +17,7 @@ require (
 	github.com/cloudflare/circl v1.5.0
 	github.com/coreos/go-oidc/v3 v3.11.0
 	github.com/docker/docker v27.4.1+incompatible
-	github.com/envoyproxy/go-control-plane/envoy v1.32.2
+	github.com/envoyproxy/go-control-plane/envoy v1.32.3
 	github.com/envoyproxy/protoc-gen-validate v1.1.0
 	github.com/go-chi/chi/v5 v5.2.0
 	github.com/go-jose/go-jose/v3 v3.0.3
@@ -46,11 +43,11 @@ require (
 	github.com/natefinch/atomic v1.0.1
 	github.com/oapi-codegen/runtime v1.1.1
 	github.com/open-policy-agent/opa v1.0.0
-	github.com/openzipkin/zipkin-go v0.4.3
 	github.com/peterbourgon/ff/v3 v3.4.0
 	github.com/pires/go-proxyproto v0.8.0
 	github.com/pomerium/csrf v1.7.0
 	github.com/pomerium/datasource v0.18.2-0.20221108160055-c6134b5ed524
+	github.com/pomerium/envoy-custom v1.32.4-0.20250114182541-6f6d2147bea6
 	github.com/pomerium/protoutil v0.0.0-20240813175624-47b7ac43ff46
 	github.com/pomerium/webauthn v0.0.0-20240603205124-0428df511172
 	github.com/prometheus/client_golang v1.20.5
@@ -70,15 +67,20 @@ require (
 	github.com/volatiletech/null/v9 v9.0.0
 	github.com/yuin/gopher-lua v1.1.1
 	go.opencensus.io v0.24.0
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.57.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0
+	go.opentelemetry.io/contrib/propagators/autoprop v0.57.0
 	go.opentelemetry.io/otel v1.33.0
-	go.opentelemetry.io/otel/bridge/opencensus v1.33.0
-	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.33.0
+	go.opentelemetry.io/otel/bridge/opencensus v1.32.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0
 	go.opentelemetry.io/otel/metric v1.33.0
 	go.opentelemetry.io/otel/sdk v1.33.0
-	go.opentelemetry.io/otel/sdk/metric v1.33.0
+	go.opentelemetry.io/otel/sdk/metric v1.32.0
 	go.opentelemetry.io/otel/trace v1.33.0
+	go.opentelemetry.io/proto/otlp v1.4.0
 	go.uber.org/automaxprocs v1.6.0
 	go.uber.org/mock v0.5.0
 	go.uber.org/zap v1.27.0
@@ -91,7 +93,7 @@ require (
 	google.golang.org/api v0.214.0
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576
 	google.golang.org/grpc v1.69.2
-	google.golang.org/protobuf v1.36.1
+	google.golang.org/protobuf v1.36.2
 	gopkg.in/yaml.v3 v3.0.1
 	namespacelabs.dev/go-filenotify v0.0.0-20220511192020-53ea11be7eaa
 	sigs.k8s.io/yaml v1.4.0
@@ -107,7 +109,6 @@ require (
 	cloud.google.com/go/monitoring v1.21.2 // indirect
 	dario.cat/mergo v1.0.0 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
-	github.com/DataDog/datadog-go v3.5.0+incompatible // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect
@@ -133,7 +134,7 @@ require (
 	github.com/aws/smithy-go v1.22.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/caddyserver/zerossl v0.1.3 // indirect
-	github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect
+	github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3 // indirect
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/containerd/platforms v0.2.1 // indirect
 	github.com/cpuguy83/dockercfg v0.3.2 // indirect
@@ -142,7 +143,7 @@ require (
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/envoyproxy/go-control-plane v0.13.2 // indirect
+	github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.8.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.6.0 // indirect
@@ -195,7 +196,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
-	github.com/philhofer/fwd v1.1.2 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
@@ -218,10 +219,8 @@ require (
 	github.com/tchap/go-patricia/v2 v2.3.1 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
-	github.com/tinylib/msgp v1.1.8 // indirect
 	github.com/tklauser/go-sysconf v0.3.14 // indirect
 	github.com/tklauser/numcpus v0.8.0 // indirect
-	github.com/uber/jaeger-client-go v2.25.0+incompatible // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
@@ -232,10 +231,10 @@ require (
 	github.com/zeebo/xxh3 v1.0.2 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/contrib/detectors/gcp v1.31.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.4.0 // indirect
+	go.opentelemetry.io/contrib/propagators/aws v1.32.0 // indirect
+	go.opentelemetry.io/contrib/propagators/b3 v1.32.0 // indirect
+	go.opentelemetry.io/contrib/propagators/jaeger v1.32.0 // indirect
+	go.opentelemetry.io/contrib/propagators/ot v1.32.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect
 	golang.org/x/mod v0.20.0 // indirect
@@ -243,7 +242,6 @@ require (
 	golang.org/x/tools v0.24.0 // indirect
 	google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect
-	gopkg.in/DataDog/dd-trace-go.v1 v1.22.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

go.sum

@@ -52,12 +52,8 @@ cloud.google.com/go/storage v1.49.0 h1:zenOPBOWHCnojRd9aJZAyQXBYqkJkdQS42dxL55CI
 cloud.google.com/go/storage v1.49.0/go.mod h1:k1eHhhpLvrPjVGfo0mOUPEJ4Y2+a/Hv5PiwehZI9qGU=
 cloud.google.com/go/trace v1.11.2 h1:4ZmaBdL8Ng/ajrgKqY5jfvzqMXbrDcBsUGXOT9aqTtI=
 cloud.google.com/go/trace v1.11.2/go.mod h1:bn7OwXd4pd5rFuAnTrzBuoZ4ax2XQeG3qNgYmfCy0Io=
-contrib.go.opencensus.io/exporter/jaeger v0.2.1 h1:yGBYzYMewVL0yO9qqJv3Z5+IRhPdU7e9o/2oKpX4YvI=
-contrib.go.opencensus.io/exporter/jaeger v0.2.1/go.mod h1:Y8IsLgdxqh1QxYxPC5IgXVmBaeLUeQFfBeBi9PbeZd0=
 contrib.go.opencensus.io/exporter/prometheus v0.4.2 h1:sqfsYl5GIY/L570iT+l93ehxaWJs2/OwXtiWwew3oAg=
 contrib.go.opencensus.io/exporter/prometheus v0.4.2/go.mod h1:dvEHbiKmgvbr5pjaF9fpw1KeYcjrnC1J8B+JKjsZyRQ=
-contrib.go.opencensus.io/exporter/zipkin v0.1.2 h1:YqE293IZrKtqPnpwDPH/lOqTWD/s3Iwabycam74JV3g=
-contrib.go.opencensus.io/exporter/zipkin v0.1.2/go.mod h1:mP5xM3rrgOjpn79MM8fZbj3gsxcuytSqtH0dxSWW1RE=
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
@@ -69,10 +65,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/CAFxX/httpcompression v0.0.9 h1:0ue2X8dOLEpxTm8tt+OdHcgA+gbDge0OqFQWGKSqgrg=
 github.com/CAFxX/httpcompression v0.0.9/go.mod h1:XX8oPZA+4IDcfZ0A71Hz0mZsv/YJOgYygkFhizVPilM=
-github.com/DataDog/datadog-go v3.5.0+incompatible h1:AShr9cqkF+taHjyQgcBcQUt/ZNK+iPq4ROaZwSX5c/U=
-github.com/DataDog/datadog-go v3.5.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
-github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20200406135749-5c268882acf0 h1:Y6HFfo8UuntPOpfmUmLb0o3MNYKfUuH2aNmvypsDbY4=
-github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20200406135749-5c268882acf0/go.mod h1:/VV3EFO/hTNQZHAqaj+CPGy2+ioFrP4EX3iRwozubhQ=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 h1:3c8yed4lgqTt+oTQ+JNMDo+F4xprBf+O/il4ZC0nRLw=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0/go.mod h1:obipzmGjfSjam60XLwGfqUkJsfiheAl+TUjG+4yzyPM=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 h1:UQ0AhxogsIRZDkElkblfnwjc3IaltCm2HUMvezQaL7s=
@@ -86,8 +78,6 @@ github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
-github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
-github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI=
 github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI=
 github.com/agnivade/levenshtein v1.2.0 h1:U9L4IOT0Y3i0TIlUIDJ7rVUziKi/zPbrJGaFrtYH3SY=
@@ -172,8 +162,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys=
 github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 h1:QVw89YDxXxEe+l8gU8ETbOasdwEV+avkR75ZzsVV9WI=
-github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
+github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3 h1:boJj011Hh+874zpIySeApCX4GeOjPl9qhRF3QuIZq+Q=
+github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
 github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
 github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
@@ -206,17 +196,13 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
-github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
-github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
-github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane v0.13.2 h1:HoRou/hxH5VAnylep8UAh0jFAllMY0UlP0TwPmIEgYI=
-github.com/envoyproxy/go-control-plane v0.13.2/go.mod h1:mcYj6+AKxG86c/jKeZsCIWv8oLzhR+SJynG0TB94Xw8=
-github.com/envoyproxy/go-control-plane/envoy v1.32.2 h1:zidqwmijfcbyKqVxjQDFx042PgX+p9U+/fu/f9VtSk8=
-github.com/envoyproxy/go-control-plane/envoy v1.32.2/go.mod h1:eR2SOX2IedqlPvmiKjUH7Wu//S602JKI7HPC/L3SRq8=
+github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M=
+github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA=
+github.com/envoyproxy/go-control-plane/envoy v1.32.3 h1:hVEaommgvzTjTd4xCaFd+kEQ2iYBtGxP6luyLrx6uOk=
+github.com/envoyproxy/go-control-plane/envoy v1.32.3/go.mod h1:F6hWupPfh75TBXGKA++MCT/CZHFq5r9/uwt/kQYkZfE=
 github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI=
 github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
@@ -277,9 +263,7 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA
 github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
 github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -316,7 +300,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
-github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/brotli/go/cbrotli v0.0.0-20230829110029-ed738e842d2f h1:jopqB+UTSdJGEJT8tEqYyE29zN91fi2827oLET8tl7k=
@@ -372,8 +355,6 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gax-go/v2 v2.14.0 h1:f+jMrjBPl+DL9nI4IQzLUxMq7XrAqFYB7hBPqMNIe8o=
 github.com/googleapis/gax-go/v2 v2.14.0/go.mod h1:lhBCnjdLrWRaPvLWhmc8IS24m9mr07qSYnHncrgo+zk=
-github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
 github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
 github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
@@ -450,7 +431,6 @@ github.com/libdns/libdns v0.2.2 h1:O6ws7bAfRPaBsgAYt8MDe2HcNBGC29hkZ9MX2eUSX3s=
 github.com/libdns/libdns v0.2.2/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ=
 github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae h1:dIZY4ULFcto4tAFlj1FYZl8ztUZ13bdq+PLY+NOfbyI=
 github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k=
-github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
 github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
 github.com/martinlindhe/base36 v1.1.1 h1:1F1MZ5MGghBXDZ2KJ3QfxmiydlWOGB8HCEtkap5NkVg=
@@ -505,13 +485,11 @@ github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+
 github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro=
 github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/ginkgo/v2 v2.19.1 h1:QXgq3Z8Crl5EL1WBAC98A5sEBHARrAJNzAmMxzLcRF0=
 github.com/onsi/ginkgo/v2 v2.19.1/go.mod h1:O3DtEWQkPa/F7fBMgmZQKKsluAy8pd3rEQdrjkPb9zA=
-github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
@@ -522,18 +500,10 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
 github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
-github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
-github.com/openzipkin/zipkin-go v0.4.3 h1:9EGwpqkgnwdEIJ+Od7QVSEIH+ocmm5nPat0G7sjsSdg=
-github.com/openzipkin/zipkin-go v0.4.3/go.mod h1:M9wCJZFWCo2RiY+o1eBCEMe0Dp2S5LDHcMZmk3RmK7c=
 github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
 github.com/peterbourgon/ff/v3 v3.4.0 h1:QBvM/rizZM1cB0p0lGMdmR7HxZeI/ZrBWB4DqLkMUBc=
 github.com/peterbourgon/ff/v3 v3.4.0/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ=
-github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
-github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw=
-github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
-github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1 h1:VGcrWe3yk6o+t7BdVNy5UDPWa4OZuDWtE1W1ZbS7Kyw=
-github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
 github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
 github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
@@ -543,7 +513,6 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
 github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
 github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -553,6 +522,8 @@ github.com/pomerium/csrf v1.7.0 h1:Qp4t6oyEod3svQtKfJZs589mdUTWKVf7q0PgCKYCshY=
 github.com/pomerium/csrf v1.7.0/go.mod h1:hAPZV47mEj2T9xFs+ysbum4l7SF1IdrryYaY6PdoIqw=
 github.com/pomerium/datasource v0.18.2-0.20221108160055-c6134b5ed524 h1:3YQY1sb54tEEbr0L73rjHkpLB0IB6qh3zl1+XQbMLis=
 github.com/pomerium/datasource v0.18.2-0.20221108160055-c6134b5ed524/go.mod h1:7fGbUYJnU8RcxZJvUvhukOIBv1G7LWDAHMfDxAf5+Y0=
+github.com/pomerium/envoy-custom v1.32.4-0.20250114182541-6f6d2147bea6 h1:QLVgpx23jcbgR9qJzIicJ+uXGjQXO0GAy55SCo0Jd9o=
+github.com/pomerium/envoy-custom v1.32.4-0.20250114182541-6f6d2147bea6/go.mod h1:afbaKE6YfshVUOrYc6XWUWfZcXencWmi1jTc00ki0Oo=
 github.com/pomerium/protoutil v0.0.0-20240813175624-47b7ac43ff46 h1:NRTg8JOXCxcIA1lAgD74iYud0rbshbWOB3Ou4+Huil8=
 github.com/pomerium/protoutil v0.0.0-20240813175624-47b7ac43ff46/go.mod h1:QqZmx6ZgPxz18va7kqoT4t/0yJtP7YFIDiT/W2n2fZ4=
 github.com/pomerium/webauthn v0.0.0-20240603205124-0428df511172 h1:TqoPqRgXSHpn+tEJq6H72iCS5pv66j3rPprThUEZg0E=
@@ -598,7 +569,6 @@ github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
 github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
 github.com/quic-go/quic-go v0.48.2 h1:wsKXZPeGWpMpCGSWqOcqpW2wZYic/8T3aqiOID0/KWE=
 github.com/quic-go/quic-go v0.48.2/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs=
-github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -644,7 +614,6 @@ github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+
 github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
 github.com/sryoya/protorand v0.0.0-20240429201223-e7440656b2a4 h1:/jKH9ivHOUkahZs3zPfJfOmkXDFB6OdsHZ4W8gyDb/c=
 github.com/sryoya/protorand v0.0.0-20240429201223-e7440656b2a4/go.mod h1:9a23nlv6vzBeVlQq6JQCjljZ6sfzsB6aha1m5Ly1W2Y=
-github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
@@ -677,17 +646,12 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
-github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
-github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0=
-github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw=
 github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU=
 github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY=
 github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY=
 github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE=
 github.com/tniswong/go.rfcx v0.0.0-20181019234604-07783c52761f h1:C43EMGXFtvYf/zunHR6ivZV7Z6ytg73t0GXwYyicXMQ=
 github.com/tniswong/go.rfcx v0.0.0-20181019234604-07783c52761f/go.mod h1:N+sR0vLSCTtI6o06PMWsjMB4TVqqDttKNq4iC9wvxVY=
-github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U=
-github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
 github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/valyala/gozstd v1.20.1 h1:xPnnnvjmaDDitMFfDxmQ4vpx0+3CdTg2o3lALvXTU/g=
 github.com/valyala/gozstd v1.20.1/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
@@ -730,30 +694,40 @@ go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJyS
 go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
 go.opentelemetry.io/contrib/detectors/gcp v1.31.0 h1:G1JQOreVrfhRkner+l4mrGxmfqYCAuy76asTDAo0xsA=
 go.opentelemetry.io/contrib/detectors/gcp v1.31.0/go.mod h1:tzQL6E1l+iV44YFTkcAeNQqzXUiekSYP9jjJjXwEd00=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.57.0 h1:qtFISDHKolvIxzSs0gIaiPUPR0Cucb0F2coHC7ZLdps=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.57.0/go.mod h1:Y+Pop1Q6hCOnETWTW4NROK/q1hv50hM7yDaUTjG8lp8=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q=
+go.opentelemetry.io/contrib/propagators/autoprop v0.57.0 h1:bNPJOdT5154XxzeFmrh8R+PXnV4t3TZEczy8gHEpcpg=
+go.opentelemetry.io/contrib/propagators/autoprop v0.57.0/go.mod h1:Tb0j0mK+QatKdCxCKPN7CSzc7kx/q34/KaohJx/N96s=
+go.opentelemetry.io/contrib/propagators/aws v1.32.0 h1:NELzr8bW7a7aHVZj5gaep1PfkvoSCGx+1qNGZx/uhhU=
+go.opentelemetry.io/contrib/propagators/aws v1.32.0/go.mod h1:XKMrzHNka3eOA+nGEcNKYVL9s77TAhkwQEynYuaRFnQ=
+go.opentelemetry.io/contrib/propagators/b3 v1.32.0 h1:MazJBz2Zf6HTN/nK/s3Ru1qme+VhWU5hm83QxEP+dvw=
+go.opentelemetry.io/contrib/propagators/b3 v1.32.0/go.mod h1:B0s70QHYPrJwPOwD1o3V/R8vETNOG9N3qZf4LDYvA30=
+go.opentelemetry.io/contrib/propagators/jaeger v1.32.0 h1:K/fOyTMD6GELKTIJBaJ9k3ppF2Njt8MeUGBOwfaWXXA=
+go.opentelemetry.io/contrib/propagators/jaeger v1.32.0/go.mod h1:ISE6hda//MTWvtngG7p4et3OCngsrTVfl7c6DjN17f8=
+go.opentelemetry.io/contrib/propagators/ot v1.32.0 h1:Poy02A4wOZubHyd2hpHPDgZW+rn6EIq0vCwTZJ6Lmu8=
+go.opentelemetry.io/contrib/propagators/ot v1.32.0/go.mod h1:cbhaURV+VR3NIMarzDYZU1RDEkXG1fNd1WMP1XCcGkY=
 go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw=
 go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I=
-go.opentelemetry.io/otel/bridge/opencensus v1.33.0 h1:sGcK0Wif7sPG3GZG9z8b+tpRZiUHwv27WgmsaZ1wgzM=
-go.opentelemetry.io/otel/bridge/opencensus v1.33.0/go.mod h1:LXJy68HiJRu+2yJmVnbDn/F9JS9Kxfsj5WpA5t5NfRY=
-go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.33.0 h1:7F29RDmnlqk6B5d+sUqemt8TBfDqxryYW5gX6L74RFA=
-go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.33.0/go.mod h1:ZiGDq7xwDMKmWDrN1XsXAj0iC7hns+2DhxBFSncNHSE=
+go.opentelemetry.io/otel/bridge/opencensus v1.32.0 h1:OVbbFgPG60UolI8ZUs+Z75NnKiO0C9QltXBrqUDImS0=
+go.opentelemetry.io/otel/bridge/opencensus v1.32.0/go.mod h1:J5SEiJNu6zzqpcA6+AVpxUKzxNocUMsefgHRpS8zdW8=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0 h1:j7ZSD+5yn+lo3sGV69nW04rRR0jhYnBwjuX3r0HvnK0=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0/go.mod h1:WXbYJTUaZXAbYd8lbgGuvih0yuCfOFC5RJoYnoLcGz8=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 h1:j9+03ymgYhPKmeXGk5Zu+cIZOlVzd9Zv7QIiyItjFBU=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0/go.mod h1:Y5+XiUG4Emn1hTfciPzGPJaSI+RpDts6BnCIir0SLqk=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0 h1:cMyu9O88joYEaI47CnQkxO1XZdpoTF9fEnW2duIddhw=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0/go.mod h1:6Am3rn7P9TVVeXYG+wtcGE7IE1tsQ+bP3AuWcKt/gOI=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0 h1:WDdP9acbMYjbKIyJUhTvtzj601sVJOqgWdUxSdR/Ysc=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I=
 go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ=
 go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M=
 go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM=
 go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM=
-go.opentelemetry.io/otel/sdk/metric v1.33.0 h1:Gs5VK9/WUJhNXZgn8MR6ITatvAmKeIuCtNbsP3JkNqU=
-go.opentelemetry.io/otel/sdk/metric v1.33.0/go.mod h1:dL5ykHZmm1B1nVRk9dDjChwDmt81MjVp3gLkQRwKf/Q=
+go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU=
+go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ=
 go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s=
 go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck=
 go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg=
@@ -811,7 +785,6 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
 golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
@@ -852,7 +825,6 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
@@ -935,7 +907,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -946,7 +917,6 @@ golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
@@ -959,7 +929,6 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
@@ -1014,7 +983,6 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
 golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=
@@ -1082,7 +1050,6 @@ google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.
 google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
@@ -1111,10 +1078,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk=
-google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
-gopkg.in/DataDog/dd-trace-go.v1 v1.22.0 h1:gpWsqqkwUldNZXGJqT69NU9MdEDhLboK1C4nMgR0MWw=
-gopkg.in/DataDog/dd-trace-go.v1 v1.22.0/go.mod h1:DVp8HmDh8PuTu2Z0fVVlBsyWaC++fzwVCaGWylTe3tg=
+google.golang.org/protobuf v1.36.2 h1:R8FeyR1/eLmkutZOM5CWghmo5itiG9z0ktFlTVLuTmU=
+google.golang.org/protobuf v1.36.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internal/atomicutil/value.go

@@ -33,3 +33,14 @@ func (v *Value[T]) Load() T {
 func (v *Value[T]) Store(val T) {
 	v.value.Store(val)
 }
+
+// Swap swaps the value atomically.
+func (v *Value[T]) Swap(val T) T {
+	old, _ := v.value.Swap(val).(T)
+	return old
+}
+
+// Swap swaps the value atomically.
+func (v *Value[T]) CompareAndSwap(old, new T) bool {
+	return v.value.CompareAndSwap(old, new)
+}

internal/authenticateflow/authenticateflow.go

@@ -4,11 +4,17 @@
 package authenticateflow
 
 import (
+	"context"
 	"fmt"
 	"time"
 
+	oteltrace "go.opentelemetry.io/otel/trace"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/stats"
+	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/types/known/structpb"
 
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/pkg/grpc"
 	"github.com/pomerium/pomerium/pkg/grpc/user"
 	"github.com/pomerium/pomerium/pkg/identity"
@@ -33,3 +39,23 @@ func populateUserFromClaims(u *user.User, claims map[string]any) {
 		u.Claims[k] = vs
 	}
 }
+
+var outboundDatabrokerTraceClientOpts = []trace.ClientStatsHandlerOption{
+	trace.WithStatsInterceptor(ignoreNotFoundErrors),
+}
+
+func ignoreNotFoundErrors(ctx context.Context, rs stats.RPCStats) stats.RPCStats {
+	if end, ok := rs.(*stats.End); ok && end.IsClient() {
+		if status.Code(end.Error) == codes.NotFound {
+			oteltrace.SpanFromContext(ctx).AddEvent("status code: NotFound")
+			return &stats.End{
+				Client:    end.Client,
+				BeginTime: end.BeginTime,
+				EndTime:   end.EndTime,
+				Trailer:   end.Trailer,
+				Error:     nil,
+			}
+		}
+	}
+	return rs
+}

internal/authenticateflow/stateful.go

@@ -9,7 +9,11 @@ import (
 	"net/url"
 	"time"
 
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
+	"go.opentelemetry.io/otel"
+	oteltrace "go.opentelemetry.io/otel/trace"
 	"golang.org/x/oauth2"
+	googlegrpc "google.golang.org/grpc"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"github.com/pomerium/pomerium/config"
@@ -19,6 +23,7 @@ import (
 	"github.com/pomerium/pomerium/internal/httputil"
 	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/internal/sessions"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/urlutil"
 	"github.com/pomerium/pomerium/pkg/cryptutil"
 	"github.com/pomerium/pomerium/pkg/grpc"
@@ -56,7 +61,7 @@ type Stateful struct {
 
 // NewStateful initializes the authentication flow for the given configuration
 // and session store.
-func NewStateful(ctx context.Context, cfg *config.Config, sessionStore sessions.SessionStore) (*Stateful, error) {
+func NewStateful(ctx context.Context, tracerProvider oteltrace.TracerProvider, cfg *config.Config, sessionStore sessions.SessionStore) (*Stateful, error) {
 	s := &Stateful{
 		sessionDuration: cfg.Options.CookieExpire,
 		sessionStore:    sessionStore,
@@ -94,7 +99,10 @@ func NewStateful(ctx context.Context, cfg *config.Config, sessionStore sessions.
 			InstallationID: cfg.Options.InstallationID,
 			ServiceName:    cfg.Options.Services,
 			SignedJWTKey:   s.sharedKey,
-		})
+		}, googlegrpc.WithStatsHandler(trace.NewClientStatsHandler(
+			otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(tracerProvider)),
+			outboundDatabrokerTraceClientOpts...,
+		)))
 	if err != nil {
 		return nil, err
 	}
@@ -316,7 +324,7 @@ func (s *Stateful) LogAuthenticateEvent(*http.Request) {}
 // AuthenticateSignInURL returns a URL to redirect the user to the authenticate
 // domain.
 func (s *Stateful) AuthenticateSignInURL(
-	_ context.Context, queryParams url.Values, redirectURL *url.URL, idpID string,
+	ctx context.Context, queryParams url.Values, redirectURL *url.URL, idpID string,
 ) (string, error) {
 	signinURL := s.authenticateURL.ResolveReference(&url.URL{
 		Path: "/.pomerium/sign_in",
@@ -327,6 +335,7 @@ func (s *Stateful) AuthenticateSignInURL(
 	}
 	queryParams.Set(urlutil.QueryRedirectURI, redirectURL.String())
 	queryParams.Set(urlutil.QueryIdentityProviderID, idpID)
+	otel.GetTextMapPropagator().Inject(ctx, trace.PomeriumURLQueryCarrier(queryParams))
 	signinURL.RawQuery = queryParams.Encode()
 	redirectTo := urlutil.NewSignedURL(s.sharedKey, signinURL).String()
 

internal/authenticateflow/stateful_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/go-jose/go-jose/v3/jwt"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace"
 	"go.uber.org/mock/gomock"
 	"golang.org/x/oauth2"
 	"google.golang.org/grpc"
@@ -69,7 +70,7 @@ func TestStatefulSignIn(t *testing.T) {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
 			sessionStore := &mstore.Store{SaveError: tt.saveError}
-			flow, err := NewStateful(context.Background(), &config.Config{Options: opts}, sessionStore)
+			flow, err := NewStateful(context.Background(), trace.NewNoopTracerProvider(), &config.Config{Options: opts}, sessionStore)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -123,12 +124,12 @@ func TestStatefulAuthenticateSignInURL(t *testing.T) {
 	opts.AuthenticateURLString = "https://authenticate.example.com"
 	key := cryptutil.NewKey()
 	opts.SharedKey = base64.StdEncoding.EncodeToString(key)
-	flow, err := NewStateful(context.Background(), &config.Config{Options: opts}, nil)
+	flow, err := NewStateful(context.Background(), trace.NewNoopTracerProvider(), &config.Config{Options: opts}, nil)
 	require.NoError(t, err)
 
 	t.Run("NilQueryParams", func(t *testing.T) {
 		redirectURL := &url.URL{Scheme: "https", Host: "example.com"}
-		u, err := flow.AuthenticateSignInURL(nil, nil, redirectURL, "fake-idp-id")
+		u, err := flow.AuthenticateSignInURL(context.Background(), nil, redirectURL, "fake-idp-id")
 		assert.NoError(t, err)
 		parsed, _ := url.Parse(u)
 		assert.NoError(t, urlutil.NewSignedURL(key, parsed).Validate())
@@ -143,7 +144,7 @@ func TestStatefulAuthenticateSignInURL(t *testing.T) {
 		redirectURL := &url.URL{Scheme: "https", Host: "example.com"}
 		q := url.Values{}
 		q.Set("foo", "bar")
-		u, err := flow.AuthenticateSignInURL(nil, q, redirectURL, "fake-idp-id")
+		u, err := flow.AuthenticateSignInURL(context.Background(), q, redirectURL, "fake-idp-id")
 		assert.NoError(t, err)
 		parsed, _ := url.Parse(u)
 		assert.NoError(t, urlutil.NewSignedURL(key, parsed).Validate())
@@ -238,7 +239,7 @@ func TestStatefulCallback(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			flow, err := NewStateful(context.Background(), &config.Config{Options: opts}, tt.sessionStore)
+			flow, err := NewStateful(context.Background(), trace.NewNoopTracerProvider(), &config.Config{Options: opts}, tt.sessionStore)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -289,7 +290,7 @@ func TestStatefulCallback(t *testing.T) {
 
 func TestStatefulRevokeSession(t *testing.T) {
 	opts := config.NewDefaultOptions()
-	flow, err := NewStateful(context.Background(), &config.Config{Options: opts}, nil)
+	flow, err := NewStateful(context.Background(), trace.NewNoopTracerProvider(), &config.Config{Options: opts}, nil)
 	require.NoError(t, err)
 
 	ctrl := gomock.NewController(t)
@@ -367,7 +368,7 @@ func TestPersistSession(t *testing.T) {
 
 	opts := config.NewDefaultOptions()
 	opts.CookieExpire = 4 * time.Hour
-	flow, err := NewStateful(context.Background(), &config.Config{Options: opts}, nil)
+	flow, err := NewStateful(context.Background(), trace.NewNoopTracerProvider(), &config.Config{Options: opts}, nil)
 	require.NoError(t, err)
 
 	ctrl := gomock.NewController(t)

internal/authenticateflow/stateless.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/go-jose/go-jose/v3"
 	"golang.org/x/oauth2"
+	googlegrpc "google.golang.org/grpc"
 	"google.golang.org/protobuf/encoding/protojson"
 
 	"github.com/pomerium/pomerium/authenticate/events"
@@ -20,6 +21,7 @@ import (
 	"github.com/pomerium/pomerium/internal/httputil"
 	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/internal/sessions"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/urlutil"
 	"github.com/pomerium/pomerium/pkg/cryptutil"
 	"github.com/pomerium/pomerium/pkg/grpc"
@@ -29,6 +31,9 @@ import (
 	"github.com/pomerium/pomerium/pkg/grpc/user"
 	"github.com/pomerium/pomerium/pkg/hpke"
 	"github.com/pomerium/pomerium/pkg/identity"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
+	"go.opentelemetry.io/otel"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
 // Stateless implements the stateless authentication flow. In this flow, the
@@ -56,18 +61,21 @@ type Stateless struct {
 
 	dataBrokerClient databroker.DataBrokerServiceClient
 
-	getIdentityProvider func(options *config.Options, idpID string) (identity.Authenticator, error)
+	getIdentityProvider func(ctx context.Context, tracerProvider oteltrace.TracerProvider, options *config.Options, idpID string) (identity.Authenticator, error)
 	profileTrimFn       func(*identitypb.Profile)
 	authEventFn         events.AuthEventFn
+
+	tracerProvider oteltrace.TracerProvider
 }
 
 // NewStateless initializes the authentication flow for the given
 // configuration, session store, and additional options.
 func NewStateless(
 	ctx context.Context,
+	tracerProvider oteltrace.TracerProvider,
 	cfg *config.Config,
 	sessionStore sessions.SessionStore,
-	getIdentityProvider func(options *config.Options, idpID string) (identity.Authenticator, error),
+	getIdentityProvider func(ctx context.Context, tracerProvider oteltrace.TracerProvider, options *config.Options, idpID string) (identity.Authenticator, error),
 	profileTrimFn func(*identitypb.Profile),
 	authEventFn events.AuthEventFn,
 ) (*Stateless, error) {
@@ -77,6 +85,7 @@ func NewStateless(
 		getIdentityProvider: getIdentityProvider,
 		profileTrimFn:       profileTrimFn,
 		authEventFn:         authEventFn,
+		tracerProvider:      tracerProvider,
 	}
 
 	var err error
@@ -137,7 +146,10 @@ func NewStateless(
 		InstallationID: cfg.Options.InstallationID,
 		ServiceName:    cfg.Options.Services,
 		SignedJWTKey:   sharedKey,
-	})
+	}, googlegrpc.WithStatsHandler(trace.NewClientStatsHandler(
+		otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(tracerProvider)),
+		outboundDatabrokerTraceClientOpts...,
+	)))
 	if err != nil {
 		return nil, err
 	}
@@ -154,7 +166,7 @@ func (s *Stateless) VerifySession(ctx context.Context, r *http.Request, _ *sessi
 		return fmt.Errorf("identity profile load error: %w", err)
 	}
 
-	authenticator, err := s.getIdentityProvider(s.options, profile.GetProviderId())
+	authenticator, err := s.getIdentityProvider(ctx, s.tracerProvider, s.options, profile.GetProviderId())
 	if err != nil {
 		return fmt.Errorf("couldn't get identity provider: %w", err)
 	}
@@ -355,6 +367,7 @@ func (s *Stateless) AuthenticateSignInURL(
 	for k, v := range queryParams {
 		q[k] = v
 	}
+	otel.GetTextMapPropagator().Inject(ctx, trace.PomeriumURLQueryCarrier(q))
 	authenticateURLWithParams.RawQuery = q.Encode()
 
 	return urlutil.SignInURL(

internal/benchmarks/latency_bench_test.go

@@ -48,6 +48,7 @@ func TestRequestLatency(t *testing.T) {
 	for i := range numRoutes {
 		routes[i] = up.Route().
 			From(env.SubdomainURL(fmt.Sprintf("from-%d", i))).
+			// Policy(func(p *config.Policy) { p.AllowPublicUnauthenticatedAccess = true })
 			PPL(fmt.Sprintf(`{"allow":{"and":["email":{"is":"user%d@example.com"}]}}`, i))
 	}
 	env.AddUpstream(up)

internal/controlplane/events.go

@@ -7,6 +7,8 @@ import (
 
 	"github.com/cenkalti/backoff/v4"
 	"github.com/google/uuid"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
+	googlegrpc "google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/proto"
@@ -76,7 +78,7 @@ func (srv *Server) getDataBrokerClient(ctx context.Context) (databrokerpb.DataBr
 		InstallationID: cfg.Options.InstallationID,
 		ServiceName:    cfg.Options.Services,
 		SignedJWTKey:   sharedKey,
-	})
+	}, googlegrpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(srv.tracerProvider))))
 	if err != nil {
 		return nil, fmt.Errorf("controlplane: error creating databroker connection: %w", err)
 	}

internal/controlplane/grpc_accesslog.go

@@ -7,6 +7,8 @@ import (
 	envoy_data_accesslog_v3 "github.com/envoyproxy/go-control-plane/envoy/data/accesslog/v3"
 	envoy_service_accesslog_v3 "github.com/envoyproxy/go-control-plane/envoy/service/accesslog/v3"
 	"github.com/rs/zerolog"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 
 	"github.com/pomerium/pomerium/internal/log"
 )
@@ -21,7 +23,11 @@ func (srv *Server) StreamAccessLogs(stream envoy_service_accesslog_v3.AccessLogS
 	for {
 		msg, err := stream.Recv()
 		if err != nil {
-			log.Ctx(stream.Context()).Error().Err(err).Msg("access log stream error, disconnecting")
+			if status.Code(err) == codes.Canceled {
+				log.Ctx(stream.Context()).Debug().Err(err).Msg("access log stream canceled")
+			} else {
+				log.Ctx(stream.Context()).Error().Err(err).Msg("access log stream error, disconnecting")
+			}
 			return err
 		}
 

internal/controlplane/http.go

@@ -2,6 +2,7 @@
 package controlplane
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"time"
@@ -9,18 +10,21 @@ import (
 	"github.com/CAFxX/httpcompression"
 	"github.com/gorilla/mux"
 	"github.com/rs/zerolog"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 
 	"github.com/pomerium/pomerium/config"
 	"github.com/pomerium/pomerium/internal/handlers"
 	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/internal/middleware"
 	"github.com/pomerium/pomerium/internal/telemetry"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/urlutil"
 	hpke_handlers "github.com/pomerium/pomerium/pkg/hpke/handlers"
 	"github.com/pomerium/pomerium/pkg/telemetry/requestid"
 )
 
-func (srv *Server) addHTTPMiddleware(root *mux.Router, logger *zerolog.Logger, _ *config.Config) {
+func (srv *Server) addHTTPMiddleware(ctx context.Context, root *mux.Router, _ *config.Config) {
+	logger := log.Ctx(ctx)
 	compressor, err := httpcompression.DefaultAdapter()
 	if err != nil {
 		panic(err)
@@ -69,9 +73,11 @@ func (srv *Server) mountCommonEndpoints(root *mux.Router, cfg *config.Config) er
 
 	root.HandleFunc("/healthz", handlers.HealthCheck)
 	root.HandleFunc("/ping", handlers.HealthCheck)
-	root.Handle("/.well-known/pomerium", handlers.WellKnownPomerium(authenticateURL))
-	root.Handle("/.well-known/pomerium/", handlers.WellKnownPomerium(authenticateURL))
-	root.Path("/.well-known/pomerium/jwks.json").Methods(http.MethodGet).Handler(handlers.JWKSHandler(signingKey))
-	root.Path(urlutil.HPKEPublicKeyPath).Methods(http.MethodGet).Handler(hpke_handlers.HPKEPublicKeyHandler(hpkePublicKey))
+
+	traceHandler := trace.NewHTTPMiddleware(otelhttp.WithTracerProvider(srv.tracerProvider))
+	root.Handle("/.well-known/pomerium", traceHandler(handlers.WellKnownPomerium(authenticateURL)))
+	root.Handle("/.well-known/pomerium/", traceHandler(handlers.WellKnownPomerium(authenticateURL)))
+	root.Path("/.well-known/pomerium/jwks.json").Methods(http.MethodGet).Handler(traceHandler(handlers.JWKSHandler(signingKey)))
+	root.Path(urlutil.HPKEPublicKeyPath).Methods(http.MethodGet).Handler(traceHandler(hpke_handlers.HPKEPublicKeyHandler(hpkePublicKey)))
 	return nil
 }

internal/controlplane/server.go

@@ -11,6 +11,8 @@ import (
 	envoy_service_discovery_v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
 	"github.com/gorilla/mux"
 	"github.com/rs/zerolog"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
+	coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/health/grpc_health_v1"
@@ -25,7 +27,6 @@ import (
 	"github.com/pomerium/pomerium/internal/events"
 	"github.com/pomerium/pomerium/internal/httputil/reproxy"
 	"github.com/pomerium/pomerium/internal/log"
-	"github.com/pomerium/pomerium/internal/telemetry"
 	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/urlutil"
 	"github.com/pomerium/pomerium/internal/version"
@@ -34,6 +35,7 @@ import (
 	"github.com/pomerium/pomerium/pkg/grpcutil"
 	"github.com/pomerium/pomerium/pkg/httputil"
 	"github.com/pomerium/pomerium/pkg/telemetry/requestid"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
 // A Service can be mounted on the control plane.
@@ -43,6 +45,7 @@ type Service interface {
 
 // A Server is the control-plane gRPC and HTTP servers.
 type Server struct {
+	coltracepb.UnimplementedTraceServiceServer
 	GRPCListener    net.Listener
 	GRPCServer      *grpc.Server
 	HTTPListener    net.Listener
@@ -66,6 +69,9 @@ type Server struct {
 	proxySvc        Service
 
 	haveSetCapacity map[string]bool
+
+	tracerProvider oteltrace.TracerProvider
+	tracer         oteltrace.Tracer
 }
 
 // NewServer creates a new Server. Listener ports are chosen by the OS.
@@ -76,7 +82,10 @@ func NewServer(
 	eventsMgr *events.Manager,
 	fileMgr *filemgr.Manager,
 ) (*Server, error) {
+	tracerProvider := trace.NewTracerProvider(ctx, "Control Plane")
 	srv := &Server{
+		tracerProvider:  tracerProvider,
+		tracer:          tracerProvider.Tracer(trace.PomeriumCoreTracer),
 		metricsMgr:      metricsMgr,
 		EventsMgr:       eventsMgr,
 		filemgr:         fileMgr,
@@ -105,7 +114,7 @@ func NewServer(
 		),
 	)
 	srv.GRPCServer = grpc.NewServer(
-		grpc.StatsHandler(telemetry.NewGRPCServerStatsHandler(cfg.Options.Services)),
+		grpc.StatsHandler(otelgrpc.NewServerHandler(otelgrpc.WithTracerProvider(tracerProvider))),
 		grpc.ChainUnaryInterceptor(
 			log.UnaryServerInterceptor(log.Ctx(ctx)),
 			requestid.UnaryServerInterceptor(),
@@ -177,7 +186,9 @@ func NewServer(
 
 	srv.xdsmgr = xdsmgr.NewManager(res)
 	envoy_service_discovery_v3.RegisterAggregatedDiscoveryServiceServer(srv.GRPCServer, srv.xdsmgr)
-
+	if exp := trace.ExporterServerFromContext(ctx); exp != nil {
+		coltracepb.RegisterTraceServiceServer(srv.GRPCServer, exp)
+	}
 	return srv, nil
 }
 
@@ -241,7 +252,7 @@ func (srv *Server) Run(ctx context.Context) error {
 
 // OnConfigChange updates the pomerium config options.
 func (srv *Server) OnConfigChange(ctx context.Context, cfg *config.Config) error {
-	ctx, span := trace.StartSpan(ctx, "controlplane.Server.OnConfigChange")
+	ctx, span := srv.tracer.Start(ctx, "controlplane.Server.OnConfigChange")
 	defer span.End()
 
 	select {
@@ -265,7 +276,7 @@ func (srv *Server) EnableProxy(ctx context.Context, svc Service) error {
 }
 
 func (srv *Server) update(ctx context.Context, cfg *config.Config) error {
-	ctx, span := trace.StartSpan(ctx, "controlplane.Server.update")
+	ctx, span := srv.tracer.Start(ctx, "controlplane.Server.update")
 	defer span.End()
 
 	if err := srv.updateRouter(ctx, cfg); err != nil {
@@ -283,7 +294,7 @@ func (srv *Server) update(ctx context.Context, cfg *config.Config) error {
 
 func (srv *Server) updateRouter(ctx context.Context, cfg *config.Config) error {
 	httpRouter := mux.NewRouter()
-	srv.addHTTPMiddleware(httpRouter, log.Ctx(ctx), cfg)
+	srv.addHTTPMiddleware(ctx, httpRouter, cfg)
 	if err := srv.mountCommonEndpoints(httpRouter, cfg); err != nil {
 		return err
 	}

internal/controlplane/xds.go

@@ -9,7 +9,6 @@ import (
 	"golang.org/x/sync/errgroup"
 
 	"github.com/pomerium/pomerium/internal/log"
-	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/pkg/cryptutil"
 	"github.com/pomerium/pomerium/pkg/protoutil"
 )
@@ -21,7 +20,7 @@ const (
 )
 
 func (srv *Server) buildDiscoveryResources(ctx context.Context) (map[string][]*envoy_service_discovery_v3.Resource, error) {
-	ctx, span := trace.StartSpan(ctx, "controlplane.Server.buildDiscoveryResources")
+	ctx, span := srv.tracer.Start(ctx, "controlplane.Server.buildDiscoveryResources")
 	defer span.End()
 
 	cfg := srv.currentConfig.Load()

internal/databroker/config_source.go

@@ -22,6 +22,9 @@ import (
 	"github.com/pomerium/pomerium/pkg/grpc/databroker"
 	"github.com/pomerium/pomerium/pkg/grpcutil"
 	"github.com/pomerium/pomerium/pkg/health"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
+	oteltrace "go.opentelemetry.io/otel/trace"
+	googlegrpc "google.golang.org/grpc"
 )
 
 // ConfigSource provides a new Config source that decorates an underlying config with
@@ -35,6 +38,7 @@ type ConfigSource struct {
 	updaterHash            uint64
 	cancel                 func()
 	enableValidation       bool
+	tracerProvider         oteltrace.TracerProvider
 
 	config.ChangeDispatcher
 }
@@ -50,11 +54,13 @@ type EnableConfigValidation bool
 // NewConfigSource creates a new ConfigSource.
 func NewConfigSource(
 	ctx context.Context,
+	tracerProvider oteltrace.TracerProvider,
 	underlying config.Source,
 	enableValidation EnableConfigValidation,
 	listeners ...config.ChangeListener,
 ) *ConfigSource {
 	src := &ConfigSource{
+		tracerProvider:         tracerProvider,
 		enableValidation:       bool(enableValidation),
 		dbConfigs:              map[string]dbConfig{},
 		outboundGRPCConnection: new(grpc.CachedOutboundGRPClientConn),
@@ -85,7 +91,7 @@ func (src *ConfigSource) GetConfig() *config.Config {
 type firstTime bool
 
 func (src *ConfigSource) rebuild(ctx context.Context, firstTime firstTime) {
-	_, span := trace.StartSpan(ctx, "databroker.config_source.rebuild")
+	_, span := trace.Continue(ctx, "databroker.config_source.rebuild")
 	defer span.End()
 
 	now := time.Now()
@@ -259,7 +265,8 @@ func (src *ConfigSource) runUpdater(ctx context.Context, cfg *config.Config) {
 
 	ctx, src.cancel = context.WithCancel(ctx)
 
-	cc, err := src.outboundGRPCConnection.Get(ctx, connectionOptions)
+	cc, err := src.outboundGRPCConnection.Get(ctx, connectionOptions,
+		googlegrpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(src.tracerProvider))))
 	if err != nil {
 		log.Ctx(ctx).Error().Err(err).Msg("databroker: failed to create gRPC connection to data broker")
 		return

internal/databroker/config_source_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace"
 	"google.golang.org/grpc"
 	"google.golang.org/protobuf/proto"
 
@@ -41,7 +42,7 @@ func TestConfigSource(t *testing.T) {
 	defer func() { _ = li.Close() }()
 	_, outboundPort, _ := net.SplitHostPort(li.Addr().String())
 
-	dataBrokerServer := New(ctx)
+	dataBrokerServer := New(ctx, trace.NewNoopTracerProvider())
 	srv := grpc.NewServer()
 	databroker.RegisterDataBrokerServiceServer(srv, dataBrokerServer)
 	go func() { _ = srv.Serve(li) }()
@@ -65,7 +66,7 @@ func TestConfigSource(t *testing.T) {
 		OutboundPort: outboundPort,
 		Options:      base,
 	})
-	src := NewConfigSource(ctx, baseSource, EnableConfigValidation(true), func(_ context.Context, cfg *config.Config) {
+	src := NewConfigSource(ctx, trace.NewNoopTracerProvider(), baseSource, EnableConfigValidation(true), func(_ context.Context, cfg *config.Config) {
 		cfgs <- cfg
 	})
 	cfgs <- src.GetConfig()

internal/databroker/registry.go

@@ -9,7 +9,6 @@ import (
 	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/internal/registry"
 	"github.com/pomerium/pomerium/internal/registry/inmemory"
-	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	registrypb "github.com/pomerium/pomerium/pkg/grpc/registry"
 	"github.com/pomerium/pomerium/pkg/storage"
 )
@@ -25,7 +24,7 @@ func (stream registryWatchServer) Context() context.Context {
 
 // Report calls the registry Report method.
 func (srv *Server) Report(ctx context.Context, req *registrypb.RegisterRequest) (*registrypb.RegisterResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.Report")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Report")
 	defer span.End()
 
 	r, err := srv.getRegistry(ctx)
@@ -38,7 +37,7 @@ func (srv *Server) Report(ctx context.Context, req *registrypb.RegisterRequest)
 
 // List calls the registry List method.
 func (srv *Server) List(ctx context.Context, req *registrypb.ListRequest) (*registrypb.ServiceList, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.List")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.List")
 	defer span.End()
 
 	r, err := srv.getRegistry(ctx)
@@ -52,7 +51,7 @@ func (srv *Server) List(ctx context.Context, req *registrypb.ListRequest) (*regi
 // Watch calls the registry Watch method.
 func (srv *Server) Watch(req *registrypb.ListRequest, stream registrypb.Registry_WatchServer) error {
 	ctx := stream.Context()
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.Watch")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Watch")
 	defer span.End()
 
 	r, err := srv.getRegistry(ctx)

internal/databroker/server.go

@@ -22,22 +22,28 @@ import (
 	"github.com/pomerium/pomerium/pkg/storage"
 	"github.com/pomerium/pomerium/pkg/storage/inmemory"
 	"github.com/pomerium/pomerium/pkg/storage/postgres"
+	oteltrace "go.opentelemetry.io/otel/trace"
 )
 
 // Server implements the databroker service using an in memory database.
 type Server struct {
 	cfg *serverConfig
 
-	mu         sync.RWMutex
-	backend    storage.Backend
-	backendCtx context.Context
-	registry   registry.Interface
+	mu             sync.RWMutex
+	backend        storage.Backend
+	backendCtx     context.Context
+	registry       registry.Interface
+	tracerProvider oteltrace.TracerProvider
+	tracer         oteltrace.Tracer
 }
 
 // New creates a new server.
-func New(ctx context.Context, options ...ServerOption) *Server {
+func New(ctx context.Context, tracerProvider oteltrace.TracerProvider, options ...ServerOption) *Server {
+	tracer := tracerProvider.Tracer(trace.PomeriumCoreTracer)
 	srv := &Server{
-		backendCtx: ctx,
+		backendCtx:     ctx,
+		tracerProvider: tracerProvider,
+		tracer:         tracer,
 	}
 	srv.UpdateConfig(ctx, options...)
 	return srv
@@ -74,7 +80,7 @@ func (srv *Server) UpdateConfig(ctx context.Context, options ...ServerOption) {
 
 // AcquireLease acquires a lease.
 func (srv *Server) AcquireLease(ctx context.Context, req *databroker.AcquireLeaseRequest) (*databroker.AcquireLeaseResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.AcquireLease")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.AcquireLease")
 	defer span.End()
 	log.Ctx(ctx).Debug().
 		Str("name", req.GetName()).
@@ -101,7 +107,7 @@ func (srv *Server) AcquireLease(ctx context.Context, req *databroker.AcquireLeas
 
 // Get gets a record from the in-memory list.
 func (srv *Server) Get(ctx context.Context, req *databroker.GetRequest) (*databroker.GetResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.Get")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Get")
 	defer span.End()
 	log.Ctx(ctx).Debug().
 		Str("type", req.GetType()).
@@ -128,7 +134,7 @@ func (srv *Server) Get(ctx context.Context, req *databroker.GetRequest) (*databr
 
 // ListTypes lists all the record types.
 func (srv *Server) ListTypes(ctx context.Context, _ *emptypb.Empty) (*databroker.ListTypesResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.ListTypes")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.ListTypes")
 	defer span.End()
 	log.Ctx(ctx).Debug().Msg("list types")
 
@@ -145,7 +151,7 @@ func (srv *Server) ListTypes(ctx context.Context, _ *emptypb.Empty) (*databroker
 
 // Query queries for records.
 func (srv *Server) Query(ctx context.Context, req *databroker.QueryRequest) (*databroker.QueryResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.Query")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Query")
 	defer span.End()
 	log.Ctx(ctx).Debug().
 		Str("type", req.GetType()).
@@ -198,7 +204,7 @@ func (srv *Server) Query(ctx context.Context, req *databroker.QueryRequest) (*da
 
 // Put updates an existing record or adds a new one.
 func (srv *Server) Put(ctx context.Context, req *databroker.PutRequest) (*databroker.PutResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.Put")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Put")
 	defer span.End()
 
 	records := req.GetRecords()
@@ -237,7 +243,7 @@ func (srv *Server) Put(ctx context.Context, req *databroker.PutRequest) (*databr
 
 // Patch updates specific fields of an existing record.
 func (srv *Server) Patch(ctx context.Context, req *databroker.PatchRequest) (*databroker.PatchResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.Patch")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Patch")
 	defer span.End()
 
 	records := req.GetRecords()
@@ -276,9 +282,9 @@ func (srv *Server) Patch(ctx context.Context, req *databroker.PatchRequest) (*da
 
 // ReleaseLease releases a lease.
 func (srv *Server) ReleaseLease(ctx context.Context, req *databroker.ReleaseLeaseRequest) (*emptypb.Empty, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.ReleaseLease")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.ReleaseLease")
 	defer span.End()
-	log.Ctx(ctx).Debug().
+	log.Ctx(ctx).Trace().
 		Str("name", req.GetName()).
 		Str("id", req.GetId()).
 		Msg("release lease")
@@ -298,9 +304,9 @@ func (srv *Server) ReleaseLease(ctx context.Context, req *databroker.ReleaseLeas
 
 // RenewLease releases a lease.
 func (srv *Server) RenewLease(ctx context.Context, req *databroker.RenewLeaseRequest) (*emptypb.Empty, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.RenewLease")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.RenewLease")
 	defer span.End()
-	log.Ctx(ctx).Debug().
+	log.Ctx(ctx).Trace().
 		Str("name", req.GetName()).
 		Str("id", req.GetId()).
 		Dur("duration", req.GetDuration().AsDuration()).
@@ -323,7 +329,7 @@ func (srv *Server) RenewLease(ctx context.Context, req *databroker.RenewLeaseReq
 
 // SetOptions sets options for a type in the databroker.
 func (srv *Server) SetOptions(ctx context.Context, req *databroker.SetOptionsRequest) (*databroker.SetOptionsResponse, error) {
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.SetOptions")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.SetOptions")
 	defer span.End()
 
 	backend, err := srv.getBackend(ctx)
@@ -346,7 +352,7 @@ func (srv *Server) SetOptions(ctx context.Context, req *databroker.SetOptionsReq
 // Sync streams updates for the given record type.
 func (srv *Server) Sync(req *databroker.SyncRequest, stream databroker.DataBrokerService_SyncServer) error {
 	ctx := stream.Context()
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.Sync")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.Sync")
 	defer span.End()
 
 	ctx, cancel := context.WithCancel(ctx)
@@ -384,7 +390,7 @@ func (srv *Server) Sync(req *databroker.SyncRequest, stream databroker.DataBroke
 // SyncLatest returns the latest value of every record in the databroker as a stream of records.
 func (srv *Server) SyncLatest(req *databroker.SyncLatestRequest, stream databroker.DataBrokerService_SyncLatestServer) error {
 	ctx := stream.Context()
-	ctx, span := trace.StartSpan(ctx, "databroker.grpc.SyncLatest")
+	ctx, span := srv.tracer.Start(ctx, "databroker.grpc.SyncLatest")
 	defer span.End()
 
 	ctx, cancel := context.WithCancel(ctx)

internal/databroker/server_test.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace/noop"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
@@ -49,10 +50,9 @@ func (h testSyncerHandler) UpdateRecords(ctx context.Context, serverVersion uint
 }
 
 func newServer(cfg *serverConfig) *Server {
-	return &Server{
-		cfg:        cfg,
-		backendCtx: context.Background(),
-	}
+	srv := New(context.Background(), noop.NewTracerProvider())
+	srv.cfg = cfg
+	return srv
 }
 
 func TestServer_Get(t *testing.T) {

internal/middleware/middleware.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 
 	"github.com/pomerium/pomerium/internal/httputil"
-	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/urlutil"
 )
 
@@ -14,12 +13,10 @@ import (
 func SetHeaders(headers map[string]string) func(next http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			ctx, span := trace.StartSpan(r.Context(), "middleware.SetHeaders")
-			defer span.End()
 			for key, val := range headers {
 				w.Header().Set(key, val)
 			}
-			next.ServeHTTP(w, r.WithContext(ctx))
+			next.ServeHTTP(w, r)
 		})
 	}
 }
@@ -29,12 +26,10 @@ func SetHeaders(headers map[string]string) func(next http.Handler) http.Handler
 func ValidateSignature(sharedKey []byte) func(next http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return httputil.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
-			ctx, span := trace.StartSpan(r.Context(), "middleware.ValidateSignature")
-			defer span.End()
 			if err := ValidateRequestURL(r, sharedKey); err != nil {
 				return httputil.NewError(http.StatusBadRequest, err)
 			}
-			next.ServeHTTP(w, r.WithContext(ctx))
+			next.ServeHTTP(w, r)
 			return nil
 		})
 	}

internal/registry/reporter.go

@@ -15,18 +15,23 @@ import (
 	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/pkg/grpc"
 	pb "github.com/pomerium/pomerium/pkg/grpc/registry"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
+	oteltrace "go.opentelemetry.io/otel/trace"
+	googlegrpc "google.golang.org/grpc"
 )
 
 // Reporter periodically submits a list of services available on this instance to the service registry
 type Reporter struct {
 	cancel                 func()
 	outboundGRPCConnection *grpc.CachedOutboundGRPClientConn
+	tracerProvider         oteltrace.TracerProvider
 }
 
 // NewReporter creates a new Reporter.
-func NewReporter() *Reporter {
+func NewReporter(tracerProvider oteltrace.TracerProvider) *Reporter {
 	return &Reporter{
 		outboundGRPCConnection: new(grpc.CachedOutboundGRPClientConn),
+		tracerProvider:         tracerProvider,
 	}
 }
 
@@ -52,7 +57,7 @@ func (r *Reporter) OnConfigChange(ctx context.Context, cfg *config.Config) {
 		InstallationID: cfg.Options.InstallationID,
 		ServiceName:    cfg.Options.Services,
 		SignedJWTKey:   sharedKey,
-	})
+	}, googlegrpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(r.tracerProvider))))
 	if err != nil {
 		log.Ctx(ctx).Error().Err(err).Msg("connecting to registry")
 		return

internal/retry/retry.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/rs/zerolog"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/pomerium/pomerium/internal/log"
 )
@@ -47,7 +48,10 @@ restart:
 	backoff:
 		for {
 			interval := backoff.NextBackOff()
-			log.Ctx(ctx).Info().Msgf("backing off for %s...", interval.String())
+			span := trace.SpanFromContext(ctx)
+			msg := fmt.Sprintf("backing off for %s...", interval.String())
+			span.AddEvent(msg)
+			log.Ctx(ctx).Info().Msg(msg)
 			timer := time.NewTimer(interval)
 			s := makeSelect(ctx, watches, name, timer.C, fn)
 			next, err := s.Exec(ctx)

internal/telemetry/grpc.go

@@ -1,100 +0,0 @@
-package telemetry
-
-import (
-	"context"
-	"strings"
-
-	"go.opencensus.io/plugin/ocgrpc"
-	"go.opencensus.io/plugin/ochttp/propagation/b3"
-	"go.opencensus.io/trace"
-	"go.opencensus.io/trace/propagation"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/metadata"
-	grpcstats "google.golang.org/grpc/stats"
-
-	"github.com/pomerium/pomerium/internal/telemetry/metrics"
-)
-
-const (
-	grpcTraceBinHeader = "grpc-trace-bin"
-	b3TraceIDHeader    = "x-b3-traceid"
-	b3SpanIDHeader     = "x-b3-spanid"
-)
-
-type tagRPCHandler interface {
-	TagRPC(context.Context, *grpcstats.RPCTagInfo) context.Context
-}
-
-// GRPCServerStatsHandler provides a grpc stats.Handler for metrics and tracing for a pomerium service
-type GRPCServerStatsHandler struct {
-	service        string
-	metricsHandler tagRPCHandler
-	grpcstats.Handler
-}
-
-// TagRPC implements grpc.stats.Handler and adds metrics and tracing metadata to the context of a given RPC
-func (h *GRPCServerStatsHandler) TagRPC(ctx context.Context, tagInfo *grpcstats.RPCTagInfo) context.Context {
-	// the opencensus trace handler only supports grpc-trace-bin, so we use that code and support b3 too
-
-	md, _ := metadata.FromIncomingContext(ctx)
-	name := strings.TrimPrefix(tagInfo.FullMethodName, "/")
-	name = strings.Replace(name, "/", ".", -1)
-
-	var parent trace.SpanContext
-	hasParent := false
-	if traceBin := md[grpcTraceBinHeader]; len(traceBin) > 0 {
-		parent, hasParent = propagation.FromBinary([]byte(traceBin[0]))
-	}
-
-	if hdr := md[b3TraceIDHeader]; len(hdr) > 0 {
-		if tid, ok := b3.ParseTraceID(hdr[0]); ok {
-			parent.TraceID = tid
-			hasParent = true
-		}
-	}
-	if hdr := md[b3SpanIDHeader]; len(hdr) > 0 {
-		if sid, ok := b3.ParseSpanID(hdr[0]); ok {
-			parent.SpanID = sid
-			hasParent = true
-		}
-	}
-
-	if hasParent {
-		ctx, _ = trace.StartSpanWithRemoteParent(ctx, name, parent,
-			trace.WithSpanKind(trace.SpanKindServer))
-	} else {
-		ctx, _ = trace.StartSpan(ctx, name,
-			trace.WithSpanKind(trace.SpanKindServer))
-	}
-
-	// ocgrpc's TagRPC must be called to attach the context rpcDataKey correctly
-	// https://github.com/census-instrumentation/opencensus-go/blob/bf52d9df8bb2d44cad934587ab946794456cf3c8/plugin/ocgrpc/server_stats_handler.go#L45
-	metricCtx := h.metricsHandler.TagRPC(h.Handler.TagRPC(ctx, tagInfo), tagInfo)
-	return metricCtx
-}
-
-// NewGRPCServerStatsHandler creates a new GRPCServerStatsHandler for a pomerium service
-func NewGRPCServerStatsHandler(service string) grpcstats.Handler {
-	return &GRPCServerStatsHandler{
-		service:        ServiceName(service),
-		Handler:        &ocgrpc.ServerHandler{},
-		metricsHandler: metrics.NewGRPCServerMetricsHandler(ServiceName(service)),
-	}
-}
-
-// GRPCClientStatsHandler provides DialOptions for grpc clients to instrument network calls with
-// both metrics and tracing
-type GRPCClientStatsHandler struct {
-	UnaryInterceptor grpc.UnaryClientInterceptor
-	// TODO: we should have a streaming interceptor too
-	grpcstats.Handler
-}
-
-// NewGRPCClientStatsHandler returns a new GRPCClientStatsHandler used to create
-// telemetry related client DialOptions
-func NewGRPCClientStatsHandler(service string) *GRPCClientStatsHandler {
-	return &GRPCClientStatsHandler{
-		Handler:          &ocgrpc.ClientHandler{},
-		UnaryInterceptor: metrics.GRPCClientInterceptor(ServiceName(service)),
-	}
-}

internal/telemetry/grpc_test.go

@@ -1,47 +0,0 @@
-package telemetry
-
-import (
-	"context"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"go.opencensus.io/plugin/ocgrpc"
-	"go.opencensus.io/plugin/ochttp/propagation/b3"
-	"go.opencensus.io/trace"
-	"google.golang.org/grpc/metadata"
-	grpcstats "google.golang.org/grpc/stats"
-)
-
-type mockTagHandler struct {
-	called bool
-}
-
-type mockCtxTag string
-
-func (m *mockTagHandler) TagRPC(ctx context.Context, _ *grpcstats.RPCTagInfo) context.Context {
-	m.called = true
-	return context.WithValue(ctx, mockCtxTag("added"), "true")
-}
-
-func Test_GRPCServerStatsHandler(t *testing.T) {
-	metricsHandler := &mockTagHandler{}
-	h := &GRPCServerStatsHandler{
-		metricsHandler: metricsHandler,
-		Handler:        &ocgrpc.ServerHandler{},
-	}
-
-	ctx := context.WithValue(context.Background(), mockCtxTag("original"), "true")
-	ctx = metadata.NewIncomingContext(ctx, metadata.MD{
-		b3TraceIDHeader: {"9de3f6756f315fef"},
-		b3SpanIDHeader:  {"b4f83d3096b6bf9c"},
-	})
-	ctx = h.TagRPC(ctx, &grpcstats.RPCTagInfo{})
-
-	assert.True(t, metricsHandler.called)
-	assert.Equal(t, ctx.Value(mockCtxTag("added")), "true")
-	assert.Equal(t, ctx.Value(mockCtxTag("original")), "true")
-
-	span := trace.FromContext(ctx)
-	expectedTraceID, _ := b3.ParseTraceID("9de3f6756f315fef")
-	assert.Equal(t, expectedTraceID, span.SpanContext().TraceID)
-}

internal/telemetry/trace/carriers.go

@@ -0,0 +1,28 @@
+package trace
+
+import (
+	"net/url"
+
+	"go.opentelemetry.io/otel/propagation"
+)
+
+type PomeriumURLQueryCarrier url.Values
+
+// Get implements propagation.TextMapCarrier.
+func (q PomeriumURLQueryCarrier) Get(key string) string {
+	return url.Values(q).Get("pomerium_" + key)
+}
+
+// Set implements propagation.TextMapCarrier.
+func (q PomeriumURLQueryCarrier) Set(key string, value string) {
+	url.Values(q).Set("pomerium_"+key, value)
+}
+
+// Keys implements propagation.TextMapCarrier.
+func (q PomeriumURLQueryCarrier) Keys() []string {
+	// this function is never called in otel, and the way it would be
+	// implemented in this instance is unclear.
+	panic("unimplemented")
+}
+
+var _ propagation.TextMapCarrier = PomeriumURLQueryCarrier{}

internal/telemetry/trace/carriers_test.go

@@ -0,0 +1,30 @@
+package trace_test
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPomeriumURLQueryCarrier(t *testing.T) {
+	t.Parallel()
+	values := url.Values{}
+	carrier := trace.PomeriumURLQueryCarrier(values)
+	assert.Empty(t, carrier.Get("foo"))
+	carrier.Set("foo", "bar")
+	assert.Equal(t, url.Values{
+		"pomerium_foo": []string{"bar"},
+	}, values)
+	assert.Equal(t, "bar", carrier.Get("foo"))
+	carrier.Set("foo", "bar2")
+	assert.Equal(t, url.Values{
+		"pomerium_foo": []string{"bar2"},
+	}, values)
+	assert.Equal(t, "bar2", carrier.Get("foo"))
+
+	assert.Panics(t, func() {
+		carrier.Keys()
+	})
+}

internal/telemetry/trace/client.go

@@ -0,0 +1,259 @@
+package trace
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	v1 "go.opentelemetry.io/proto/otlp/trace/v1"
+)
+
+var (
+	ErrNoClient      = errors.New("no client")
+	ErrClientStopped = errors.New("client is stopped")
+)
+
+type SyncClient interface {
+	otlptrace.Client
+
+	Update(ctx context.Context, newClient otlptrace.Client) error
+}
+
+func NewSyncClient(client otlptrace.Client) SyncClient {
+	return &syncClient{
+		client: client,
+	}
+}
+
+type syncClient struct {
+	mu               sync.Mutex
+	client           otlptrace.Client
+	waitForNewClient chan struct{}
+}
+
+var _ SyncClient = (*syncClient)(nil)
+
+// Start implements otlptrace.Client.
+func (ac *syncClient) Start(ctx context.Context) error {
+	ac.mu.Lock()
+	defer ac.mu.Unlock()
+	if ac.waitForNewClient != nil {
+		panic("bug: Start called during Stop or Update")
+	}
+	if ac.client == nil {
+		return ErrNoClient
+	}
+	return ac.client.Start(ctx)
+}
+
+// Stop implements otlptrace.Client.
+func (ac *syncClient) Stop(ctx context.Context) error {
+	ac.mu.Lock()
+	defer ac.mu.Unlock()
+	if ac.waitForNewClient != nil {
+		panic("bug: Stop called concurrently")
+	}
+	return ac.resetLocked(ctx, nil)
+}
+
+func (ac *syncClient) resetLocked(ctx context.Context, newClient otlptrace.Client) error {
+	if ac.client == nil {
+		return ErrNoClient
+	}
+	ac.waitForNewClient = make(chan struct{})
+	ac.mu.Unlock()
+
+	err := ac.client.Stop(ctx)
+
+	ac.mu.Lock()
+	close(ac.waitForNewClient)
+	ac.waitForNewClient = nil
+	ac.client = newClient
+	return err
+}
+
+// UploadTraces implements otlptrace.Client.
+func (ac *syncClient) UploadTraces(ctx context.Context, protoSpans []*v1.ResourceSpans) error {
+	ac.mu.Lock()
+	if ac.waitForNewClient != nil {
+		wait := ac.waitForNewClient
+		ac.mu.Unlock()
+		select {
+		case <-wait:
+			ac.mu.Lock()
+		case <-ctx.Done():
+			return context.Cause(ctx)
+		}
+	} else if ac.client == nil {
+		ac.mu.Unlock()
+		return ErrNoClient
+	}
+	client := ac.client
+	ac.mu.Unlock()
+	if client == nil {
+		return ErrClientStopped
+	}
+	return client.UploadTraces(ctx, protoSpans)
+}
+
+func (ac *syncClient) Update(ctx context.Context, newClient otlptrace.Client) error {
+	if newClient != nil {
+		if err := newClient.Start(ctx); err != nil {
+			return fmt.Errorf("error starting new client: %w", err)
+		}
+	}
+	ac.mu.Lock()
+	defer ac.mu.Unlock()
+	if ac.waitForNewClient != nil {
+		panic("bug: Update called during Stop")
+	}
+	if newClient == ac.client {
+		return nil
+	}
+	return ac.resetLocked(ctx, newClient)
+}
+
+// NewRemoteClientFromEnv creates an otlp trace client using the well-known
+// environment variables defined in the [OpenTelemetry documentation].
+//
+// [OpenTelemetry documentation]: https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/
+func NewRemoteClientFromEnv() otlptrace.Client {
+	if os.Getenv("OTEL_SDK_DISABLED") == "true" {
+		return NoopClient{}
+	}
+
+	exporter, ok := os.LookupEnv("OTEL_TRACES_EXPORTER")
+	if !ok {
+		exporter = "none"
+	}
+
+	switch strings.ToLower(strings.TrimSpace(exporter)) {
+	case "none", "noop", "":
+		return NoopClient{}
+	case "otlp":
+		var protocol string
+		if v, ok := os.LookupEnv("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"); ok {
+			protocol = v
+		} else if v, ok := os.LookupEnv("OTEL_EXPORTER_OTLP_PROTOCOL"); ok {
+			protocol = v
+		} else {
+			// try to guess the expected protocol from the port number
+			var endpoint string
+			var specific bool
+			if v, ok := os.LookupEnv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"); ok {
+				endpoint = v
+				specific = true
+			} else if v, ok := os.LookupEnv("OTEL_EXPORTER_OTLP_ENDPOINT"); ok {
+				endpoint = v
+			}
+			protocol = BestEffortProtocolFromOTLPEndpoint(endpoint, specific)
+		}
+		switch strings.ToLower(strings.TrimSpace(protocol)) {
+		case "grpc":
+			return otlptracegrpc.NewClient()
+		case "http/protobuf", "":
+			return otlptracehttp.NewClient()
+		default:
+			otel.Handle(fmt.Errorf(`unknown otlp trace exporter protocol %q, expected "grpc" or "http/protobuf"`, protocol))
+			return NoopClient{}
+		}
+	default:
+		otel.Handle(fmt.Errorf(`unknown otlp trace exporter %q, expected "otlp" or "none"`, exporter))
+		return NoopClient{}
+	}
+}
+
+func BestEffortProtocolFromOTLPEndpoint(endpoint string, specificEnv bool) string {
+	if endpoint == "" {
+		return ""
+	}
+	u, err := url.Parse(endpoint)
+	if err != nil {
+		return ""
+	}
+	switch u.Port() {
+	case "4318":
+		return "http/protobuf"
+	case "4317":
+		return "grpc"
+	default:
+		// For http, if the signal-specific form of the endpoint env variable
+		// (e.g. $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT) is used, the /v1/<signal>
+		//                           ^^^^^^
+		// path must be present. Otherwise, the path must _not_ be present,
+		// because the sdk will add it.
+		// This doesn't apply to grpc endpoints, so assume grpc if there is a
+		// conflict here.
+		hasPath := len(strings.Trim(u.Path, "/")) > 0
+		switch {
+		case hasPath && specificEnv:
+			return "http/protobuf"
+		case !hasPath && specificEnv:
+			return "grpc"
+		case hasPath && !specificEnv:
+			// would be invalid for http, so assume it's grpc on a subpath
+			return "grpc"
+		case !hasPath && !specificEnv:
+			// could be either, but default to http
+			return "http/protobuf"
+		}
+		panic("unreachable")
+	}
+}
+
+type NoopClient struct{}
+
+// Start implements otlptrace.Client.
+func (n NoopClient) Start(context.Context) error {
+	return nil
+}
+
+// Stop implements otlptrace.Client.
+func (n NoopClient) Stop(context.Context) error {
+	return nil
+}
+
+// UploadTraces implements otlptrace.Client.
+func (n NoopClient) UploadTraces(context.Context, []*v1.ResourceSpans) error {
+	return nil
+}
+
+func IsDisabledViaEnvironment() bool {
+	if os.Getenv("OTEL_SDK_DISABLED") == "true" {
+		return true
+	}
+	exporter, ok := os.LookupEnv("OTEL_TRACES_EXPORTER")
+	if !ok {
+		return false
+	}
+	switch strings.ToLower(strings.TrimSpace(exporter)) {
+	case "none, noop":
+		return true
+	default:
+		return false
+	}
+}
+
+func IsEnabledViaEnvironment() bool {
+	if os.Getenv("OTEL_SDK_DISABLED") == "true" {
+		return false
+	}
+	exporter, ok := os.LookupEnv("OTEL_TRACES_EXPORTER")
+	if !ok {
+		return false
+	}
+	switch strings.ToLower(strings.TrimSpace(exporter)) {
+	case "none, noop", "":
+		return false
+	default:
+		return true
+	}
+}

internal/telemetry/trace/client_test.go

@@ -0,0 +1,454 @@
+package trace_test
+
+import (
+	"context"
+	"fmt"
+	"runtime"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/pomerium/pomerium/internal/log"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
+	"github.com/pomerium/pomerium/internal/testenv"
+	"github.com/pomerium/pomerium/internal/testenv/scenarios"
+	"github.com/pomerium/pomerium/internal/testenv/snippets"
+	. "github.com/pomerium/pomerium/internal/testutil/tracetest" //nolint:revive
+	"github.com/pomerium/pomerium/internal/testutil/tracetest/mock_otlptrace"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel"
+	tracev1 "go.opentelemetry.io/proto/otlp/trace/v1"
+	"go.uber.org/mock/gomock"
+)
+
+func TestSyncClient(t *testing.T) {
+	t.Run("No client", func(t *testing.T) {
+		sc := trace.NewSyncClient(nil)
+		assert.ErrorIs(t, sc.Start(context.Background()), trace.ErrNoClient)
+		assert.ErrorIs(t, sc.UploadTraces(context.Background(), nil), trace.ErrNoClient)
+		assert.ErrorIs(t, sc.Stop(context.Background()), trace.ErrNoClient)
+	})
+
+	t.Run("Valid client", func(t *testing.T) {
+		ctrl := gomock.NewController(t)
+		mockClient := mock_otlptrace.NewMockClient(ctrl)
+		start := mockClient.EXPECT().
+			Start(gomock.Any()).
+			Return(nil)
+		upload := mockClient.EXPECT().
+			UploadTraces(gomock.Any(), gomock.Any()).
+			Return(nil).
+			After(start)
+		mockClient.EXPECT().
+			Stop(gomock.Any()).
+			Return(nil).
+			After(upload)
+		sc := trace.NewSyncClient(mockClient)
+		assert.NoError(t, sc.Start(context.Background()))
+		assert.NoError(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{}))
+		assert.NoError(t, sc.Stop(context.Background()))
+	})
+	t.Run("Update", func(t *testing.T) {
+		ctrl := gomock.NewController(t)
+		mockClient1 := mock_otlptrace.NewMockClient(ctrl)
+		mockClient2 := mock_otlptrace.NewMockClient(ctrl)
+
+		start1 := mockClient1.EXPECT().
+			Start(gomock.Any()).
+			Return(nil)
+		upload1 := mockClient1.EXPECT().
+			UploadTraces(gomock.Any(), gomock.Any()).
+			Return(nil).
+			After(start1)
+		start2 := mockClient2.EXPECT().
+			Start(gomock.Any()).
+			Return(nil).
+			After(upload1)
+		stop1 := mockClient1.EXPECT().
+			Stop(gomock.Any()).
+			Return(nil).
+			After(start2)
+		upload2 := mockClient2.EXPECT().
+			UploadTraces(gomock.Any(), gomock.Any()).
+			Return(nil).
+			After(stop1)
+		mockClient2.EXPECT().
+			Stop(gomock.Any()).
+			Return(nil).
+			After(upload2)
+		sc := trace.NewSyncClient(mockClient1)
+		assert.NoError(t, sc.Start(context.Background()))
+		assert.NoError(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{}))
+		assert.NoError(t, sc.Update(context.Background(), mockClient2))
+		assert.NoError(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{}))
+		assert.NoError(t, sc.Stop(context.Background()))
+	})
+
+	spinWait := func(counter *atomic.Int32, until int32) error {
+		startTime := time.Now()
+		for counter.Load() != until {
+			if time.Since(startTime) > 1*time.Second {
+				return fmt.Errorf("timed out waiting for counter to equal %d", until)
+			}
+		}
+		return nil
+	}
+
+	t.Run("Concurrent UploadTraces", func(t *testing.T) {
+		ctrl := gomock.NewController(t)
+		mockClient1 := mock_otlptrace.NewMockClient(ctrl)
+		count := atomic.Int32{}
+		unlock := make(chan struct{})
+		concurrency := min(runtime.NumCPU(), 4)
+		mockClient1.EXPECT().
+			UploadTraces(gomock.Any(), gomock.Any()).
+			DoAndReturn(func(context.Context, []*tracev1.ResourceSpans) error {
+				count.Add(1)
+				defer count.Add(-1)
+				<-unlock
+				return nil
+			}).
+			Times(concurrency)
+		sc := trace.NewSyncClient(mockClient1)
+		start := make(chan struct{})
+		for range concurrency {
+			go func() {
+				runtime.LockOSThread()
+				defer runtime.UnlockOSThread()
+				<-start
+				require.NoError(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{}))
+			}()
+		}
+
+		runtime.LockOSThread()
+		defer runtime.UnlockOSThread()
+		close(start)
+		assert.NoError(t, spinWait(&count, int32(concurrency)))
+	})
+
+	t.Run("Concurrent Update/UploadTraces", func(t *testing.T) {
+		runtime.LockOSThread()
+		defer runtime.UnlockOSThread()
+
+		ctrl := gomock.NewController(t)
+		mockClient1 := mock_otlptrace.NewMockClient(ctrl)
+		mockClient2 := mock_otlptrace.NewMockClient(ctrl)
+		uploadTracesCount1 := atomic.Int32{}
+		uploadTracesCount2 := atomic.Int32{}
+		unlock1 := make(chan struct{})
+		unlock2 := make(chan struct{})
+		waitForStop := make(chan struct{})
+		concurrency := min(runtime.NumCPU(), 4)
+
+		// start 1 -> upload 1 -> start 2 -> stop 1 -> upload 2 -> stop 2
+		fStart1 := mockClient1.EXPECT().
+			Start(gomock.Any()).
+			Return(nil)
+		fUpload1 := mockClient1.EXPECT().
+			UploadTraces(gomock.Any(), gomock.Any()).
+			DoAndReturn(func(context.Context, []*tracev1.ResourceSpans) error {
+				// called from non-test threads
+				uploadTracesCount1.Add(1)
+				defer uploadTracesCount1.Add(-1)
+				<-unlock1
+				return nil
+			}).
+			Times(concurrency).
+			After(fStart1)
+		fStart2 := mockClient2.EXPECT().
+			Start(gomock.Any()).
+			Return(nil).
+			After(fUpload1)
+		fStop1 := mockClient1.EXPECT().
+			Stop(gomock.Any()).
+			DoAndReturn(func(context.Context) error {
+				// called from test thread
+				close(unlock1)
+				assert.NoError(t, spinWait(&uploadTracesCount1, 0))
+				return nil
+			}).
+			After(fStart2)
+		fUpload2 := mockClient2.EXPECT().
+			UploadTraces(gomock.Any(), gomock.Any()).
+			DoAndReturn(func(context.Context, []*tracev1.ResourceSpans) error {
+				// called from non-test threads
+				uploadTracesCount2.Add(1)
+				defer uploadTracesCount2.Add(-1)
+				<-unlock2
+				return nil
+			}).
+			Times(concurrency).
+			After(fStop1)
+		mockClient2.EXPECT().
+			Stop(gomock.Any()).
+			DoAndReturn(func(context.Context) error {
+				// called from test thread
+				close(unlock2)
+				assert.NoError(t, spinWait(&uploadTracesCount2, 0))
+				close(waitForStop)
+				// no way around sleeping here - we have to give the other threads time
+				// to call UploadTraces and block waiting on waitForNewClient to be
+				// closed, which happens after this function returns
+				time.Sleep(10 * time.Millisecond)
+				return nil
+			}).
+			After(fUpload2)
+		sc := trace.NewSyncClient(mockClient1)
+		require.NoError(t, sc.Start(context.Background()))
+
+		for range concurrency {
+			go func() {
+				require.NoError(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{}))
+			}()
+		}
+		require.NoError(t, spinWait(&uploadTracesCount1, int32(concurrency)))
+		// at this point, all calls to UploadTraces for client1 are blocked
+
+		for range concurrency {
+			go func() {
+				<-unlock1 // wait for client1.Stop
+				// after this, calls to UploadTraces will block waiting for the
+				// new client, instead of using the old one we're about to close
+				require.NoError(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{}))
+			}()
+		}
+		require.NoError(t, sc.Update(context.Background(), mockClient2))
+		require.NoError(t, spinWait(&uploadTracesCount2, int32(concurrency)))
+		// at this point, all calls to UploadTraces for client2 are blocked.
+
+		// while SyncClient is waiting for the underlying client to stop during
+		// sc.Stop(), *new* calls to sc.UploadTraces will wait for it to stop, then
+		// error with trace.ErrClientStopped, but the previous calls blocked in
+		// client2 will complete without error.
+		for range concurrency {
+			go func() {
+				<-waitForStop
+				assert.ErrorIs(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{}), trace.ErrClientStopped)
+			}()
+		}
+		assert.NoError(t, sc.Stop(context.Background()))
+
+		// sanity checks
+		assert.ErrorIs(t, sc.UploadTraces(context.Background(), []*tracev1.ResourceSpans{}), trace.ErrNoClient)
+		assert.ErrorIs(t, sc.Start(context.Background()), trace.ErrNoClient)
+		assert.ErrorIs(t, sc.Stop(context.Background()), trace.ErrNoClient)
+		assert.NoError(t, sc.Update(context.Background(), nil))
+	})
+}
+
+type errHandler struct {
+	err error
+}
+
+var _ otel.ErrorHandler = (*errHandler)(nil)
+
+func (h *errHandler) Handle(err error) {
+	h.err = err
+}
+
+func TestNewRemoteClientFromEnv(t *testing.T) {
+	env := testenv.New(t, testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags))
+
+	receiver := scenarios.NewOTLPTraceReceiver()
+	env.Add(receiver)
+
+	grpcEndpoint := receiver.GRPCEndpointURL()
+	httpEndpoint := receiver.HTTPEndpointURL()
+
+	env.Start()
+	snippets.WaitStartupComplete(env)
+
+	for _, tc := range []struct {
+		name          string
+		env           map[string]string
+		newClientErr  string
+		uploadErr     bool
+		expectNoSpans bool
+	}{
+		{
+			name: "GRPC endpoint, auto protocol",
+			env: map[string]string{
+				"OTEL_TRACES_EXPORTER":               "otlp",
+				"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": grpcEndpoint.Value(),
+			},
+		},
+		{
+			name: "GRPC endpoint, alternate env, auto protocol",
+			env: map[string]string{
+				"OTEL_TRACES_EXPORTER":        "otlp",
+				"OTEL_EXPORTER_OTLP_ENDPOINT": grpcEndpoint.Value(),
+			},
+			uploadErr: true,
+		},
+		{
+			name: "HTTP endpoint, auto protocol",
+			env: map[string]string{
+				"OTEL_TRACES_EXPORTER":               "otlp",
+				"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": httpEndpoint.Value(),
+			},
+		},
+		{
+			name: "HTTP endpoint, alternate env, auto protocol",
+			env: map[string]string{
+				"OTEL_TRACES_EXPORTER":        "otlp",
+				"OTEL_EXPORTER_OTLP_ENDPOINT": strings.TrimSuffix(httpEndpoint.Value(), "/v1/traces"), // path is added automatically by the sdk here
+			},
+		},
+		{
+			name: "GRPC endpoint, explicit protocol",
+			env: map[string]string{
+				"OTEL_TRACES_EXPORTER":               "otlp",
+				"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": grpcEndpoint.Value(),
+				"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "grpc",
+			},
+		},
+		{
+			name: "HTTP endpoint, explicit protocol",
+			env: map[string]string{
+				"OTEL_TRACES_EXPORTER":               "otlp",
+				"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": httpEndpoint.Value(),
+				"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "http/protobuf",
+			},
+		},
+		{
+			name: "exporter unset",
+			env: map[string]string{
+				"OTEL_TRACES_EXPORTER":               "",
+				"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": httpEndpoint.Value(),
+				"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "http/protobuf",
+			},
+			expectNoSpans: true,
+		},
+		{
+			name: "exporter noop",
+			env: map[string]string{
+				"OTEL_TRACES_EXPORTER":               "noop",
+				"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": httpEndpoint.Value(),
+				"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "http/protobuf",
+			},
+			expectNoSpans: true,
+		},
+		{
+			name: "exporter none",
+			env: map[string]string{
+				"OTEL_TRACES_EXPORTER":               "none",
+				"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": httpEndpoint.Value(),
+				"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "http/protobuf",
+			},
+			expectNoSpans: true,
+		},
+		{
+			name: "invalid exporter",
+			env: map[string]string{
+				"OTEL_TRACES_EXPORTER": "invalid",
+			},
+			newClientErr: `unknown otlp trace exporter "invalid", expected "otlp" or "none"`,
+		},
+		{
+			name: "invalid protocol",
+			env: map[string]string{
+				"OTEL_TRACES_EXPORTER":               "otlp",
+				"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": grpcEndpoint.Value(),
+				"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "invalid",
+			},
+			newClientErr: `unknown otlp trace exporter protocol "invalid", expected "grpc" or "http/protobuf"`,
+		},
+		{
+			name: "valid configuration, but sdk disabled",
+			env: map[string]string{
+				"OTEL_TRACES_EXPORTER":               "otlp",
+				"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": grpcEndpoint.Value(),
+				"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "grpc",
+				"OTEL_SDK_DISABLED":                  "true",
+			},
+			expectNoSpans: true,
+		},
+		{
+			name: "valid configuration, wrong value for sdk disabled env",
+			env: map[string]string{
+				"OTEL_TRACES_EXPORTER":               "otlp",
+				"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": grpcEndpoint.Value(),
+				"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "grpc",
+				"OTEL_SDK_DISABLED":                  "1", // only "true" works according to the spec
+			},
+		},
+		{
+			name: "endpoint variable precedence",
+			env: map[string]string{
+				"OTEL_TRACES_EXPORTER":               "otlp",
+				"OTEL_EXPORTER_OTLP_ENDPOINT":        "invalid",
+				"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": grpcEndpoint.Value(), // should take precedence
+				"OTEL_EXPORTER_OTLP_PROTOCOL":        "grpc",
+			},
+		},
+		{
+			name: "protocol variable precedence",
+			env: map[string]string{
+				"OTEL_TRACES_EXPORTER":               "otlp",
+				"OTEL_EXPORTER_OTLP_PROTOCOL":        "invalid",
+				"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "grpc", // should take precedence
+				"OTEL_EXPORTER_OTLP_ENDPOINT":        grpcEndpoint.Value(),
+			},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			for k, v := range tc.env {
+				t.Setenv(k, v)
+			}
+			handler := &errHandler{}
+			oldErrHandler := otel.GetErrorHandler()
+			otel.SetErrorHandler(handler)
+			t.Cleanup(func() { otel.SetErrorHandler(oldErrHandler) })
+
+			remoteClient := trace.NewRemoteClientFromEnv()
+			ctx := trace.Options{
+				RemoteClient: remoteClient,
+			}.NewContext(log.Ctx(env.Context()).WithContext(context.Background()))
+
+			if tc.newClientErr != "" {
+				assert.ErrorContains(t, handler.err, tc.newClientErr)
+				return
+			}
+
+			tp := trace.NewTracerProvider(ctx, t.Name())
+
+			_, span := tp.Tracer(trace.PomeriumCoreTracer).Start(ctx, "test span")
+			span.End()
+
+			if tc.uploadErr {
+				assert.Error(t, trace.ForceFlush(ctx))
+				assert.NoError(t, trace.ShutdownContext(ctx))
+				return
+			}
+			assert.NoError(t, trace.ShutdownContext(ctx))
+
+			results := NewTraceResults(receiver.FlushResourceSpans())
+			if tc.expectNoSpans {
+				results.MatchTraces(t, MatchOptions{Exact: true})
+			} else {
+				results.MatchTraces(t, MatchOptions{
+					Exact: true,
+				}, Match{Name: t.Name() + ": test span", TraceCount: 1, Services: []string{t.Name()}})
+			}
+		})
+	}
+}
+
+func TestBestEffortProtocolFromOTLPEndpoint(t *testing.T) {
+	t.Run("Well-known port numbers", func(t *testing.T) {
+		assert.Equal(t, "grpc", trace.BestEffortProtocolFromOTLPEndpoint("http://127.0.0.1:4317", true))
+		assert.Equal(t, "http/protobuf", trace.BestEffortProtocolFromOTLPEndpoint("http://127.0.0.1:4318", true))
+	})
+	t.Run("path presence", func(t *testing.T) {
+		assert.Equal(t, "http/protobuf", trace.BestEffortProtocolFromOTLPEndpoint("http://127.0.0.1:12345", false))
+		assert.Equal(t, "grpc", trace.BestEffortProtocolFromOTLPEndpoint("http://127.0.0.1:12345", true))
+		assert.Equal(t, "grpc", trace.BestEffortProtocolFromOTLPEndpoint("http://127.0.0.1:12345/v1/traces", false))
+		assert.Equal(t, "http/protobuf", trace.BestEffortProtocolFromOTLPEndpoint("http://127.0.0.1:12345/v1/traces", true))
+	})
+	t.Run("invalid inputs", func(t *testing.T) {
+		assert.Equal(t, "", trace.BestEffortProtocolFromOTLPEndpoint("", false))
+		assert.Equal(t, "", trace.BestEffortProtocolFromOTLPEndpoint("http://\x7f", false))
+	})
+}

internal/telemetry/trace/datadog.go

@@ -1,34 +0,0 @@
-package trace
-
-import (
-	datadog "github.com/DataDog/opencensus-go-exporter-datadog"
-	octrace "go.opencensus.io/trace"
-)
-
-type datadogProvider struct {
-	exporter *datadog.Exporter
-}
-
-func (provider *datadogProvider) Register(opts *TracingOptions) error {
-	dOpts := datadog.Options{
-		Service:   opts.Service,
-		TraceAddr: opts.DatadogAddress,
-	}
-	dex, err := datadog.NewExporter(dOpts)
-	if err != nil {
-		return err
-	}
-	octrace.RegisterExporter(dex)
-	provider.exporter = dex
-	return nil
-}
-
-func (provider *datadogProvider) Unregister() error {
-	if provider.exporter == nil {
-		return nil
-	}
-	octrace.UnregisterExporter(provider.exporter)
-	provider.exporter.Stop()
-	provider.exporter = nil
-	return nil
-}

internal/telemetry/trace/debug.go

@@ -0,0 +1,524 @@
+package trace
+
+import (
+	"context"
+	"encoding/binary"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"runtime"
+	"slices"
+	"strings"
+	"sync"
+	"time"
+
+	"go.opentelemetry.io/otel/attribute"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	oteltrace "go.opentelemetry.io/otel/trace"
+	coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/protobuf/encoding/protojson"
+)
+
+type DebugFlags uint32
+
+const (
+	// If set, adds the "caller" attribute to each trace with the source location
+	// where the trace was started.
+	TrackSpanCallers = (1 << iota)
+
+	// If set, keeps track of all span references and will attempt to wait for
+	// all traces to complete when shutting down a trace context.
+	// Use with caution, this will cause increasing memory usage over time.
+	TrackSpanReferences = (1 << iota)
+
+	// If set, keeps track of all observed spans, including span context and
+	// all attributes.
+	// Use with caution, this will cause significantly increasing memory usage
+	// over time.
+	TrackAllSpans = (1 << iota) | TrackSpanCallers
+
+	// If set, will log all trace IDs and their span counts on close.
+	//
+	// Enables [TrackAllSpans]
+	LogTraceIDs = (1 << iota) | TrackAllSpans
+
+	// If set, will log all spans observed by the exporter on close. These spans
+	// may belong to incomplete traces.
+	//
+	// Enables [TrackAllSpans]
+	LogAllSpans = (1 << iota) | TrackAllSpans
+
+	// If set, will log all exported spans when a warning is issued on close
+	// (requires warning flags to also be set)
+	//
+	// Enables [TrackAllSpans]
+	LogAllSpansOnWarn = (1 << iota) | TrackAllSpans
+
+	// If set, will log all trace ID mappings when a warning is issued on close.
+	// (requires warning flags to also be set)
+	LogTraceIDsOnWarn = (1 << iota)
+
+	// If set, will print a warning to stderr on close if there are any incomplete
+	// traces (traces with no observed root spans)
+	WarnOnIncompleteTraces = (1 << iota)
+
+	// If set, will print a warning to stderr on close if there are any incomplete
+	// spans (spans started, but not ended)
+	WarnOnIncompleteSpans = (1 << iota)
+
+	// If set, will print a warning to stderr on close if there are any spans
+	// which reference unknown parent spans.
+	//
+	// Enables [TrackSpanReferences]
+	WarnOnUnresolvedReferences = (1 << iota) | TrackSpanReferences
+
+	// If set, configures Envoy to flush every span individually, disabling its
+	// internal buffer.
+	EnvoyFlushEverySpan = (1 << iota)
+)
+
+func (df DebugFlags) Check(flags DebugFlags) bool {
+	return (df & flags) == flags
+}
+
+var (
+	ErrIncompleteSpans    = errors.New("exporter shut down with incomplete spans")
+	ErrMissingParentSpans = errors.New("exporter shut down with missing parent spans")
+)
+
+// WaitForSpans will block up to the given max duration and wait for all
+// in-flight spans from tracers created with the given context to end. This
+// function can be called more than once, and is safe to call from multiple
+// goroutines in parallel.
+//
+// This requires the [TrackSpanReferences] debug flag to have been set with
+// [Options.NewContext]. Otherwise, this function is a no-op and will return
+// immediately.
+//
+// If this function blocks for more than 10 seconds, it will print a warning
+// to stderr containing a list of span IDs it is waiting for, and the IDs of
+// their parents (if known). Additionally, if the [TrackAllSpans] debug flag
+// is set, details about parent spans will be displayed, including call site
+// and trace ID.
+func WaitForSpans(ctx context.Context, maxDuration time.Duration) error {
+	if sys := systemContextFromContext(ctx); sys != nil && sys.observer != nil {
+		done := make(chan struct{})
+		go func() {
+			defer close(done)
+			sys.observer.wait(10 * time.Second)
+		}()
+		select {
+		case <-done:
+			return nil
+		case <-time.After(maxDuration):
+			return ErrMissingParentSpans
+		}
+	}
+	return nil
+}
+
+func DebugFlagsFromContext(ctx context.Context) DebugFlags {
+	if sys := systemContextFromContext(ctx); sys != nil {
+		return sys.options.DebugFlags
+	}
+	return 0
+}
+
+type stackTraceProcessor struct{}
+
+// ForceFlush implements trace.SpanProcessor.
+func (s *stackTraceProcessor) ForceFlush(context.Context) error {
+	return nil
+}
+
+// OnEnd implements trace.SpanProcessor.
+func (*stackTraceProcessor) OnEnd(sdktrace.ReadOnlySpan) {
+}
+
+// OnStart implements trace.SpanProcessor.
+func (*stackTraceProcessor) OnStart(_ context.Context, s sdktrace.ReadWriteSpan) {
+	_, file, line, _ := runtime.Caller(2)
+	s.SetAttributes(attribute.String("caller", fmt.Sprintf("%s:%d", file, line)))
+}
+
+// Shutdown implements trace.SpanProcessor.
+func (s *stackTraceProcessor) Shutdown(context.Context) error {
+	return nil
+}
+
+var debugMessageWriter io.Writer
+
+func startMsg(title string) *strings.Builder {
+	msg := &strings.Builder{}
+	msg.WriteString("\n==================================================\n")
+	msg.WriteString(title)
+	return msg
+}
+
+func endMsg(msg *strings.Builder) {
+	msg.WriteString("==================================================\n")
+	w := debugMessageWriter
+	if w == nil {
+		w = os.Stderr
+	}
+	fmt.Fprint(w, msg.String())
+}
+
+type DebugEvent struct {
+	Timestamp time.Time                             `json:"timestamp"`
+	Request   *coltracepb.ExportTraceServiceRequest `json:"request"`
+}
+
+func (e DebugEvent) MarshalJSON() ([]byte, error) {
+	type debugEvent struct {
+		Timestamp time.Time       `json:"timestamp"`
+		Request   json.RawMessage `json:"request"`
+	}
+	reqData, _ := protojson.Marshal(e.Request)
+	return json.Marshal(debugEvent{
+		Timestamp: e.Timestamp,
+		Request:   reqData,
+	})
+}
+
+func (e *DebugEvent) UnmarshalJSON(b []byte) error {
+	type debugEvent struct {
+		Timestamp time.Time       `json:"timestamp"`
+		Request   json.RawMessage `json:"request"`
+	}
+	var ev debugEvent
+	if err := json.Unmarshal(b, &ev); err != nil {
+		return err
+	}
+	e.Timestamp = ev.Timestamp
+	var msg coltracepb.ExportTraceServiceRequest
+	if err := protojson.Unmarshal(ev.Request, &msg); err != nil {
+		return err
+	}
+	e.Request = &msg
+	return nil
+}
+
+const shardCount = 64
+
+type (
+	shardedSet   [shardCount]map[oteltrace.SpanID]struct{}
+	shardedLocks [shardCount]sync.Mutex
+)
+
+func (s *shardedSet) Range(f func(key oteltrace.SpanID)) {
+	for i := range shardCount {
+		for k := range s[i] {
+			f(k)
+		}
+	}
+}
+
+func (s *shardedLocks) LockAll() {
+	for i := range shardCount {
+		s[i].Lock()
+	}
+}
+
+func (s *shardedLocks) UnlockAll() {
+	for i := range shardCount {
+		s[i].Unlock()
+	}
+}
+
+type spanTracker struct {
+	inflightSpansMu shardedLocks
+	inflightSpans   shardedSet
+	allSpans        sync.Map
+	debugFlags      DebugFlags
+	observer        *spanObserver
+	shutdownOnce    sync.Once
+}
+
+func newSpanTracker(observer *spanObserver, debugFlags DebugFlags) *spanTracker {
+	st := &spanTracker{
+		observer:   observer,
+		debugFlags: debugFlags,
+	}
+	for i := range len(st.inflightSpans) {
+		st.inflightSpans[i] = make(map[oteltrace.SpanID]struct{})
+	}
+	return st
+}
+
+type spanInfo struct {
+	Name        string
+	SpanContext oteltrace.SpanContext
+	Parent      oteltrace.SpanContext
+	caller      string
+	startTime   time.Time
+}
+
+// ForceFlush implements trace.SpanProcessor.
+func (t *spanTracker) ForceFlush(context.Context) error {
+	return nil
+}
+
+// OnEnd implements trace.SpanProcessor.
+func (t *spanTracker) OnEnd(s sdktrace.ReadOnlySpan) {
+	id := s.SpanContext().SpanID()
+	bucket := binary.BigEndian.Uint64(id[:]) % shardCount
+	t.inflightSpansMu[bucket].Lock()
+	defer t.inflightSpansMu[bucket].Unlock()
+	delete(t.inflightSpans[bucket], id)
+}
+
+// OnStart implements trace.SpanProcessor.
+func (t *spanTracker) OnStart(_ context.Context, s sdktrace.ReadWriteSpan) {
+	id := s.SpanContext().SpanID()
+	bucket := binary.BigEndian.Uint64(id[:]) % shardCount
+	t.inflightSpansMu[bucket].Lock()
+	defer t.inflightSpansMu[bucket].Unlock()
+	t.inflightSpans[bucket][id] = struct{}{}
+
+	if t.debugFlags.Check(TrackSpanReferences) {
+		if s.Parent().IsValid() {
+			t.observer.ObserveReference(s.Parent().SpanID(), id)
+		}
+		t.observer.Observe(id)
+	}
+
+	if t.debugFlags.Check(TrackAllSpans) {
+		var caller string
+		for _, attr := range s.Attributes() {
+			if attr.Key == "caller" {
+				caller = attr.Value.AsString()
+				break
+			}
+		}
+		t.allSpans.Store(id, &spanInfo{
+			Name:        s.Name(),
+			SpanContext: s.SpanContext(),
+			Parent:      s.Parent(),
+			caller:      caller,
+			startTime:   s.StartTime(),
+		})
+	}
+}
+
+// Shutdown implements trace.SpanProcessor.
+func (t *spanTracker) Shutdown(_ context.Context) error {
+	if t.debugFlags == 0 {
+		return nil
+	}
+	didWarn := false
+	t.shutdownOnce.Do(func() {
+		if t.debugFlags.Check(WarnOnUnresolvedReferences) {
+			var unknownParentIDs []string
+			for id, via := range t.observer.referencedIDs {
+				if via.IsValid() {
+					if t.debugFlags.Check(TrackAllSpans) {
+						if viaSpan, ok := t.allSpans.Load(via); ok {
+							unknownParentIDs = append(unknownParentIDs, fmt.Sprintf("%s via %s (%s)", id, via, viaSpan.(*spanInfo).Name))
+						} else {
+							unknownParentIDs = append(unknownParentIDs, fmt.Sprintf("%s via %s", id, via))
+						}
+					}
+				}
+			}
+			if len(unknownParentIDs) > 0 {
+				didWarn = true
+				msg := startMsg("WARNING: parent spans referenced but never seen:\n")
+				for _, str := range unknownParentIDs {
+					msg.WriteString(str)
+					msg.WriteString("\n")
+				}
+				endMsg(msg)
+			}
+		}
+		if t.debugFlags.Check(WarnOnIncompleteSpans) {
+			if t.debugFlags.Check(TrackAllSpans) {
+				incompleteSpans := []*spanInfo{}
+				t.inflightSpansMu.LockAll()
+				t.inflightSpans.Range(func(key oteltrace.SpanID) {
+					if info, ok := t.allSpans.Load(key); ok {
+						incompleteSpans = append(incompleteSpans, info.(*spanInfo))
+					}
+				})
+				t.inflightSpansMu.UnlockAll()
+				if len(incompleteSpans) > 0 {
+					didWarn = true
+					msg := startMsg("WARNING: spans not ended:\n")
+					longestName := 0
+					for _, span := range incompleteSpans {
+						longestName = max(longestName, len(span.Name)+2)
+					}
+					for _, span := range incompleteSpans {
+						var startedAt string
+						if span.caller != "" {
+							startedAt = " | started at: " + span.caller
+						}
+						fmt.Fprintf(msg, "%-*s (trace: %s | span: %s | parent: %s%s)\n", longestName, "'"+span.Name+"'",
+							span.SpanContext.TraceID(), span.SpanContext.SpanID(), span.Parent.SpanID(), startedAt)
+					}
+					endMsg(msg)
+				}
+			} else {
+				incompleteSpans := []oteltrace.SpanID{}
+				t.inflightSpansMu.LockAll()
+				t.inflightSpans.Range(func(key oteltrace.SpanID) {
+					incompleteSpans = append(incompleteSpans, key)
+				})
+				t.inflightSpansMu.UnlockAll()
+				if len(incompleteSpans) > 0 {
+					didWarn = true
+					msg := startMsg("WARNING: spans not ended:\n")
+					for _, span := range incompleteSpans {
+						fmt.Fprintf(msg, "%s\n", span)
+					}
+					msg.WriteString("Note: set TrackAllSpans flag for more info\n")
+					endMsg(msg)
+				}
+			}
+		}
+
+		if t.debugFlags.Check(LogAllSpans) || (t.debugFlags.Check(LogAllSpansOnWarn) && didWarn) {
+			allSpans := []*spanInfo{}
+			t.allSpans.Range(func(_, value any) bool {
+				allSpans = append(allSpans, value.(*spanInfo))
+				return true
+			})
+			slices.SortFunc(allSpans, func(a, b *spanInfo) int {
+				return a.startTime.Compare(b.startTime)
+			})
+			msg := startMsg("All observed spans:\n")
+			longestName := 0
+			for _, span := range allSpans {
+				longestName = max(longestName, len(span.Name)+2)
+			}
+			for _, span := range allSpans {
+				var startedAt string
+				if span.caller != "" {
+					startedAt = " | started at: " + span.caller
+				}
+				fmt.Fprintf(msg, "%-*s (trace: %s | span: %s | parent: %s%s)\n", longestName, "'"+span.Name+"'",
+					span.SpanContext.TraceID(), span.SpanContext.SpanID(), span.Parent.SpanID(), startedAt)
+			}
+			endMsg(msg)
+		}
+
+		if t.debugFlags.Check(LogTraceIDs) || (didWarn && t.debugFlags.Check(LogTraceIDsOnWarn)) {
+			msg := startMsg("Known trace ids:\n")
+			traceIDs := map[oteltrace.TraceID]int{}
+			t.allSpans.Range(func(_, value any) bool {
+				v := value.(*spanInfo)
+				traceIDs[v.SpanContext.TraceID()]++
+				return true
+			})
+			for id, n := range traceIDs {
+				fmt.Fprintf(msg, "%s (%d spans)\n", id.String(), n)
+			}
+			endMsg(msg)
+		}
+	})
+	if didWarn {
+		return ErrIncompleteSpans
+	}
+	return nil
+}
+
+func newSpanObserver() *spanObserver {
+	return &spanObserver{
+		referencedIDs: map[oteltrace.SpanID]oteltrace.SpanID{},
+		cond:          sync.NewCond(&sync.Mutex{}),
+	}
+}
+
+type spanObserver struct {
+	cond          *sync.Cond
+	referencedIDs map[oteltrace.SpanID]oteltrace.SpanID
+	unobservedIDs int
+}
+
+func (obs *spanObserver) ObserveReference(id oteltrace.SpanID, via oteltrace.SpanID) {
+	obs.cond.L.Lock()
+	defer obs.cond.L.Unlock()
+	if _, referenced := obs.referencedIDs[id]; !referenced {
+		obs.referencedIDs[id] = via // referenced, but not observed
+		// It is possible for new unobserved references to come in while waiting,
+		// but incrementing the counter wouldn't satisfy the condition so we don't
+		// need to signal the waiters
+		obs.unobservedIDs++
+	}
+}
+
+func (obs *spanObserver) Observe(id oteltrace.SpanID) {
+	obs.cond.L.Lock()
+	defer obs.cond.L.Unlock()
+	if observed, referenced := obs.referencedIDs[id]; !referenced || observed.IsValid() { // NB: subtle condition
+		obs.referencedIDs[id] = zeroSpanID
+		if referenced {
+			obs.unobservedIDs--
+			obs.cond.Broadcast()
+		}
+	}
+}
+
+func (obs *spanObserver) wait(warnAfter time.Duration) {
+	done := make(chan struct{})
+	defer close(done)
+	go func() {
+		select {
+		case <-done:
+			return
+		case <-time.After(warnAfter):
+			obs.debugWarnWaiting()
+		}
+	}()
+
+	obs.cond.L.Lock()
+	for obs.unobservedIDs > 0 {
+		obs.cond.Wait()
+	}
+	obs.cond.L.Unlock()
+}
+
+func (obs *spanObserver) debugWarnWaiting() {
+	obs.cond.L.Lock()
+	msg := startMsg(fmt.Sprintf("Waiting on %d unobserved spans:\n", obs.unobservedIDs))
+	for id, via := range obs.referencedIDs {
+		if via.IsValid() {
+			fmt.Fprintf(msg, "%s via %s\n", id, via)
+		}
+	}
+	endMsg(msg)
+	obs.cond.L.Unlock()
+}
+
+func (srv *ExporterServer) observeExport(ctx context.Context, req *coltracepb.ExportTraceServiceRequest) {
+	isLocal := len(metadata.ValueFromIncomingContext(ctx, localExporterMetadataKey)) != 0
+	if isLocal {
+		return
+	}
+	for _, res := range req.ResourceSpans {
+		for _, scope := range res.ScopeSpans {
+			for _, span := range scope.Spans {
+				id, ok := ToSpanID(span.SpanId)
+				if !ok {
+					continue
+				}
+				srv.observer.Observe(id)
+				for _, attr := range span.Attributes {
+					if attr.Key != "pomerium.external-parent-span" {
+						continue
+					}
+					if bytes, err := hex.DecodeString(attr.Value.GetStringValue()); err == nil {
+						if id, ok := ToSpanID(bytes); ok {
+							srv.observer.Observe(id)
+						}
+					}
+					break
+				}
+			}
+		}
+	}
+}

internal/telemetry/trace/debug_test.go

@@ -0,0 +1,288 @@
+package trace_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"runtime"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
+	. "github.com/pomerium/pomerium/internal/testutil/tracetest" //nolint:revive
+	"github.com/stretchr/testify/assert"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	oteltrace "go.opentelemetry.io/otel/trace"
+)
+
+func TestSpanObserver(t *testing.T) {
+	t.Run("observe single reference", func(t *testing.T) {
+		obs := trace.NewSpanObserver()
+		assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs())
+
+		obs.ObserveReference(Span(1).ID(), Span(2).ID())
+		assert.Equal(t, []oteltrace.SpanID{Span(1).ID()}, obs.XUnobservedIDs())
+		obs.Observe(Span(1).ID())
+		assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs())
+	})
+	t.Run("observe multiple references", func(t *testing.T) {
+		obs := trace.NewSpanObserver()
+
+		obs.ObserveReference(Span(1).ID(), Span(2).ID())
+		obs.ObserveReference(Span(1).ID(), Span(3).ID())
+		obs.ObserveReference(Span(1).ID(), Span(4).ID())
+		assert.Equal(t, []oteltrace.SpanID{Span(1).ID()}, obs.XUnobservedIDs())
+		obs.Observe(Span(1).ID())
+		assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs())
+	})
+	t.Run("observe before reference", func(t *testing.T) {
+		obs := trace.NewSpanObserver()
+
+		obs.Observe(Span(1).ID())
+		assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs())
+		obs.ObserveReference(Span(1).ID(), Span(2).ID())
+		assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs())
+	})
+
+	t.Run("wait", func(t *testing.T) {
+		obs := trace.NewSpanObserver()
+		obs.ObserveReference(Span(1).ID(), Span(2).ID())
+		obs.Observe(Span(2).ID())
+		obs.ObserveReference(Span(3).ID(), Span(4).ID())
+		obs.Observe(Span(4).ID())
+		obs.ObserveReference(Span(5).ID(), Span(6).ID())
+		obs.Observe(Span(6).ID())
+		waitOkToExit := atomic.Bool{}
+		waitExited := atomic.Bool{}
+		go func() {
+			defer waitExited.Store(true)
+			obs.XWait()
+			assert.True(t, waitOkToExit.Load(), "wait exited early")
+		}()
+
+		time.Sleep(10 * time.Millisecond)
+		assert.False(t, waitExited.Load())
+
+		obs.Observe(Span(1).ID())
+		time.Sleep(10 * time.Millisecond)
+		assert.False(t, waitExited.Load())
+
+		obs.Observe(Span(3).ID())
+		time.Sleep(10 * time.Millisecond)
+		assert.False(t, waitExited.Load())
+
+		waitOkToExit.Store(true)
+		obs.Observe(Span(5).ID())
+		assert.Eventually(t, waitExited.Load, 10*time.Millisecond, 1*time.Millisecond)
+	})
+
+	t.Run("new references observed during wait", func(t *testing.T) {
+		obs := trace.NewSpanObserver()
+		obs.ObserveReference(Span(1).ID(), Span(2).ID())
+		obs.Observe(Span(2).ID())
+		obs.ObserveReference(Span(3).ID(), Span(4).ID())
+		obs.Observe(Span(4).ID())
+		obs.ObserveReference(Span(5).ID(), Span(6).ID())
+		obs.Observe(Span(6).ID())
+		waitOkToExit := atomic.Bool{}
+		waitExited := atomic.Bool{}
+		go func() {
+			defer waitExited.Store(true)
+			obs.XWait()
+			assert.True(t, waitOkToExit.Load(), "wait exited early")
+		}()
+
+		assert.Equal(t, []oteltrace.SpanID{Span(1).ID(), Span(3).ID(), Span(5).ID()}, obs.XUnobservedIDs())
+		time.Sleep(10 * time.Millisecond)
+		assert.False(t, waitExited.Load())
+
+		obs.Observe(Span(1).ID())
+		assert.Equal(t, []oteltrace.SpanID{Span(3).ID(), Span(5).ID()}, obs.XUnobservedIDs())
+		time.Sleep(10 * time.Millisecond)
+		assert.False(t, waitExited.Load())
+
+		obs.Observe(Span(3).ID())
+		assert.Equal(t, []oteltrace.SpanID{Span(5).ID()}, obs.XUnobservedIDs())
+		time.Sleep(10 * time.Millisecond)
+		assert.False(t, waitExited.Load())
+
+		// observe a new reference
+		obs.ObserveReference(Span(7).ID(), Span(8).ID())
+		obs.Observe(Span(8).ID())
+		assert.Equal(t, []oteltrace.SpanID{Span(5).ID(), Span(7).ID()}, obs.XUnobservedIDs())
+		time.Sleep(10 * time.Millisecond)
+		assert.False(t, waitExited.Load())
+
+		obs.Observe(Span(5).ID())
+		assert.Equal(t, []oteltrace.SpanID{Span(7).ID()}, obs.XUnobservedIDs())
+		time.Sleep(10 * time.Millisecond)
+		assert.False(t, waitExited.Load())
+
+		waitOkToExit.Store(true)
+		obs.Observe(Span(7).ID())
+		assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs())
+		assert.Eventually(t, waitExited.Load, 10*time.Millisecond, 1*time.Millisecond)
+	})
+
+	t.Run("multiple waiters", func(t *testing.T) {
+		t.Parallel()
+		obs := trace.NewSpanObserver()
+		obs.ObserveReference(Span(1).ID(), Span(2).ID())
+		obs.Observe(Span(2).ID())
+
+		waitersExited := atomic.Int32{}
+		for range 10 {
+			go func() {
+				defer waitersExited.Add(1)
+				obs.XWait()
+			}()
+		}
+
+		assert.Equal(t, []oteltrace.SpanID{Span(1).ID()}, obs.XUnobservedIDs())
+		time.Sleep(10 * time.Millisecond)
+		assert.Equal(t, int32(0), waitersExited.Load())
+
+		obs.Observe(Span(1).ID())
+
+		startTime := time.Now()
+		for waitersExited.Load() != 10 {
+			if time.Since(startTime) > 1*time.Millisecond {
+				t.Fatal("timed out")
+			}
+			runtime.Gosched()
+		}
+	})
+}
+
+func TestSpanTracker(t *testing.T) {
+	t.Run("no debug flags", func(t *testing.T) {
+		t.Parallel()
+		obs := trace.NewSpanObserver()
+		tracker := trace.NewSpanTracker(obs, 0)
+		tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracker))
+		tracer := tp.Tracer("test")
+		assert.Equal(t, []oteltrace.SpanID{}, tracker.XInflightSpans())
+		_, span1 := tracer.Start(context.Background(), "span 1")
+		assert.Equal(t, []oteltrace.SpanID{span1.SpanContext().SpanID()}, tracker.XInflightSpans())
+		assert.Equal(t, []oteltrace.SpanID{}, obs.XObservedIDs())
+		span1.End()
+		assert.Equal(t, []oteltrace.SpanID{}, tracker.XInflightSpans())
+		assert.Equal(t, []oteltrace.SpanID{}, obs.XObservedIDs())
+	})
+	t.Run("with TrackSpanReferences debug flag", func(t *testing.T) {
+		t.Parallel()
+		obs := trace.NewSpanObserver()
+		tracker := trace.NewSpanTracker(obs, trace.TrackSpanReferences)
+		tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracker))
+		tracer := tp.Tracer("test")
+		assert.Equal(t, []oteltrace.SpanID{}, tracker.XInflightSpans())
+		_, span1 := tracer.Start(context.Background(), "span 1")
+		assert.Equal(t, []oteltrace.SpanID{span1.SpanContext().SpanID()}, tracker.XInflightSpans())
+		assert.Equal(t, []oteltrace.SpanID{span1.SpanContext().SpanID()}, obs.XObservedIDs())
+		span1.End()
+		assert.Equal(t, []oteltrace.SpanID{}, tracker.XInflightSpans())
+		assert.Equal(t, []oteltrace.SpanID{span1.SpanContext().SpanID()}, obs.XObservedIDs())
+	})
+}
+
+func TestSpanTrackerWarnings(t *testing.T) {
+	t.Run("WarnOnIncompleteSpans", func(t *testing.T) {
+		var buf bytes.Buffer
+		trace.SetDebugMessageWriterForTest(t, &buf)
+
+		obs := trace.NewSpanObserver()
+		tracker := trace.NewSpanTracker(obs, trace.WarnOnIncompleteSpans)
+		tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracker))
+		tracer := tp.Tracer("test")
+		_, span1 := tracer.Start(context.Background(), "span 1")
+
+		assert.ErrorIs(t, tp.Shutdown(context.Background()), trace.ErrIncompleteSpans)
+
+		assert.Equal(t, fmt.Sprintf(`
+==================================================
+WARNING: spans not ended:
+%s
+Note: set TrackAllSpans flag for more info
+==================================================
+`, span1.SpanContext().SpanID()), buf.String())
+	})
+
+	t.Run("WarnOnIncompleteSpans with TrackAllSpans", func(t *testing.T) {
+		var buf bytes.Buffer
+		trace.SetDebugMessageWriterForTest(t, &buf)
+
+		obs := trace.NewSpanObserver()
+		tracker := trace.NewSpanTracker(obs, trace.WarnOnIncompleteSpans|trace.TrackAllSpans)
+		tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracker))
+		tracer := tp.Tracer("test")
+		_, span1 := tracer.Start(context.Background(), "span 1")
+
+		assert.ErrorIs(t, tp.Shutdown(context.Background()), trace.ErrIncompleteSpans)
+
+		assert.Equal(t, fmt.Sprintf(`
+==================================================
+WARNING: spans not ended:
+'span 1' (trace: %s | span: %s | parent: 0000000000000000)
+==================================================
+`, span1.SpanContext().TraceID(), span1.SpanContext().SpanID()), buf.String())
+	})
+
+	t.Run("WarnOnIncompleteSpans with TrackAllSpans and stackTraceProcessor", func(t *testing.T) {
+		var buf bytes.Buffer
+		trace.SetDebugMessageWriterForTest(t, &buf)
+
+		obs := trace.NewSpanObserver()
+		tracker := trace.NewSpanTracker(obs, trace.WarnOnIncompleteSpans|trace.TrackAllSpans)
+		tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(&trace.XStackTraceProcessor{}), sdktrace.WithSpanProcessor(tracker))
+		tracer := tp.Tracer("test")
+		_, span1 := tracer.Start(context.Background(), "span 1")
+		_, file, line, _ := runtime.Caller(0)
+		line--
+
+		assert.ErrorIs(t, tp.Shutdown(context.Background()), trace.ErrIncompleteSpans)
+
+		assert.Equal(t, fmt.Sprintf(`
+==================================================
+WARNING: spans not ended:
+'span 1' (trace: %s | span: %s | parent: 0000000000000000 | started at: %s:%d)
+==================================================
+`, span1.SpanContext().TraceID(), span1.SpanContext().SpanID(), file, line), buf.String())
+	})
+
+	t.Run("LogAllSpansOnWarn", func(t *testing.T) {
+		var buf bytes.Buffer
+		trace.SetDebugMessageWriterForTest(t, &buf)
+
+		obs := trace.NewSpanObserver()
+		tracker := trace.NewSpanTracker(obs, trace.WarnOnIncompleteSpans|trace.TrackAllSpans|trace.LogAllSpansOnWarn)
+		tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(&trace.XStackTraceProcessor{}), sdktrace.WithSpanProcessor(tracker))
+		tracer := tp.Tracer("test")
+		_, span1 := tracer.Start(context.Background(), "span 1")
+		time.Sleep(10 * time.Millisecond)
+		span1.End()
+		time.Sleep(10 * time.Millisecond)
+		_, span2 := tracer.Start(context.Background(), "span 2")
+		_, file, line, _ := runtime.Caller(0)
+		line--
+
+		tp.Shutdown(context.Background())
+
+		assert.Equal(t,
+			fmt.Sprintf(`
+==================================================
+WARNING: spans not ended:
+'span 2' (trace: %[1]s | span: %[2]s | parent: 0000000000000000 | started at: %[3]s:%[4]d)
+==================================================
+
+==================================================
+All observed spans:
+'span 1' (trace: %[5]s | span: %[6]s | parent: 0000000000000000 | started at: %[3]s:%[7]d)
+'span 2' (trace: %[1]s | span: %[2]s | parent: 0000000000000000 | started at: %[3]s:%[4]d)
+==================================================
+`,
+				span2.SpanContext().TraceID(), span2.SpanContext().SpanID(), file, line,
+				span1.SpanContext().TraceID(), span1.SpanContext().SpanID(), line-4,
+			), buf.String())
+	})
+}

internal/telemetry/trace/doc.go

@@ -1,2 +0,0 @@
-// Package trace contains support for OpenCensus distributed tracing.
-package trace

internal/telemetry/trace/global.go

@@ -0,0 +1,46 @@
+package trace
+
+import (
+	"context"
+
+	"go.opentelemetry.io/contrib/propagators/autoprop"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/trace"
+	"go.opentelemetry.io/otel/trace/embedded"
+)
+
+// PomeriumCoreTracer should be used for all tracers created in pomerium core.
+const PomeriumCoreTracer = "pomerium.io/core"
+
+func init() {
+	otel.SetTextMapPropagator(autoprop.NewTextMapPropagator())
+}
+
+// UseGlobalPanicTracer sets the global tracer provider to one whose tracers
+// panic when starting spans. This can be used to locate errant usages of the
+// global tracer, and is enabled automatically in some tests. It is otherwise
+// not used by default, since pomerium is used as a library in some places that
+// might use the global tracer provider.
+func UseGlobalPanicTracer() {
+	otel.SetTracerProvider(panicTracerProvider{})
+}
+
+type panicTracerProvider struct {
+	embedded.TracerProvider
+}
+
+// Tracer implements trace.TracerProvider.
+func (w panicTracerProvider) Tracer(string, ...trace.TracerOption) trace.Tracer {
+	return panicTracer{}
+}
+
+type panicTracer struct {
+	embedded.Tracer
+}
+
+var _ trace.Tracer = panicTracer{}
+
+// Start implements trace.Tracer.
+func (p panicTracer) Start(context.Context, string, ...trace.SpanStartOption) (context.Context, trace.Span) {
+	panic("global tracer used")
+}

internal/telemetry/trace/global_test.go

@@ -0,0 +1,22 @@
+package trace_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
+	"github.com/stretchr/testify/assert"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/trace/noop"
+)
+
+func TestUseGlobalPanicTracer(t *testing.T) {
+	t.Cleanup(func() {
+		otel.SetTracerProvider(noop.NewTracerProvider())
+	})
+	trace.UseGlobalPanicTracer()
+	tracer := otel.GetTracerProvider().Tracer("test")
+	assert.Panics(t, func() {
+		tracer.Start(context.Background(), "span")
+	})
+}

internal/telemetry/trace/jaeger.go

@@ -1,37 +0,0 @@
-package trace
-
-import (
-	"contrib.go.opencensus.io/exporter/jaeger"
-	octrace "go.opencensus.io/trace"
-)
-
-type jaegerProvider struct {
-	exporter *jaeger.Exporter
-}
-
-func (provider *jaegerProvider) Register(opts *TracingOptions) error {
-	jOpts := jaeger.Options{
-		ServiceName:   opts.Service,
-		AgentEndpoint: opts.JaegerAgentEndpoint,
-	}
-	if opts.JaegerCollectorEndpoint != nil {
-		jOpts.CollectorEndpoint = opts.JaegerCollectorEndpoint.String()
-	}
-	jex, err := jaeger.NewExporter(jOpts)
-	if err != nil {
-		return err
-	}
-	octrace.RegisterExporter(jex)
-	provider.exporter = jex
-	return nil
-}
-
-func (provider *jaegerProvider) Unregister() error {
-	if provider.exporter == nil {
-		return nil
-	}
-	octrace.UnregisterExporter(provider.exporter)
-	provider.exporter.Flush()
-	provider.exporter = nil
-	return nil
-}

internal/telemetry/trace/main_test.go

@@ -0,0 +1,13 @@
+package trace_test
+
+import (
+	"os"
+	"testing"
+
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
+)
+
+func TestMain(m *testing.M) {
+	trace.UseGlobalPanicTracer()
+	os.Exit(m.Run())
+}

internal/telemetry/trace/middleware.go

@@ -0,0 +1,96 @@
+package trace
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"reflect"
+
+	"github.com/gorilla/mux"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"google.golang.org/grpc/stats"
+)
+
+func NewHTTPMiddleware(opts ...otelhttp.Option) mux.MiddlewareFunc {
+	return otelhttp.NewMiddleware("Server: %s %s", append(opts, otelhttp.WithSpanNameFormatter(func(operation string, r *http.Request) string {
+		routeStr := ""
+		route := mux.CurrentRoute(r)
+		if route != nil {
+			var err error
+			routeStr, err = route.GetPathTemplate()
+			if err != nil {
+				routeStr, err = route.GetPathRegexp()
+				if err != nil {
+					routeStr = ""
+				}
+			}
+		}
+		return fmt.Sprintf(operation, r.Method, routeStr)
+	}))...)
+}
+
+type clientStatsHandlerWrapper struct {
+	ClientStatsHandlerOptions
+	base stats.Handler
+}
+
+type ClientStatsHandlerOptions struct {
+	statsInterceptor func(ctx context.Context, rs stats.RPCStats) stats.RPCStats
+}
+
+type ClientStatsHandlerOption func(*ClientStatsHandlerOptions)
+
+func (o *ClientStatsHandlerOptions) apply(opts ...ClientStatsHandlerOption) {
+	for _, op := range opts {
+		op(o)
+	}
+}
+
+// WithStatsInterceptor calls the given function to modify the rpc stats before
+// passing it to the stats handler during HandleRPC events.
+//
+// The interceptor MUST NOT modify the RPCStats it is given. It should instead
+// return a copy of the underlying object with the same type, with any
+// modifications made to the copy.
+func WithStatsInterceptor(statsInterceptor func(ctx context.Context, rs stats.RPCStats) stats.RPCStats) ClientStatsHandlerOption {
+	return func(o *ClientStatsHandlerOptions) {
+		o.statsInterceptor = statsInterceptor
+	}
+}
+
+func NewClientStatsHandler(base stats.Handler, opts ...ClientStatsHandlerOption) stats.Handler {
+	options := ClientStatsHandlerOptions{}
+	options.apply(opts...)
+	return &clientStatsHandlerWrapper{
+		ClientStatsHandlerOptions: options,
+		base:                      base,
+	}
+}
+
+// HandleConn implements stats.Handler.
+func (w *clientStatsHandlerWrapper) HandleConn(ctx context.Context, stats stats.ConnStats) {
+	w.base.HandleConn(ctx, stats)
+}
+
+// HandleRPC implements stats.Handler.
+func (w *clientStatsHandlerWrapper) HandleRPC(ctx context.Context, stats stats.RPCStats) {
+	if w.statsInterceptor != nil {
+		modified := w.statsInterceptor(ctx, stats)
+		if reflect.TypeOf(stats) != reflect.TypeOf(modified) {
+			panic("bug: stats interceptor returned a message of a different type")
+		}
+		w.base.HandleRPC(ctx, modified)
+	} else {
+		w.base.HandleRPC(ctx, stats)
+	}
+}
+
+// TagConn implements stats.Handler.
+func (w *clientStatsHandlerWrapper) TagConn(ctx context.Context, info *stats.ConnTagInfo) context.Context {
+	return w.base.TagConn(ctx, info)
+}
+
+// TagRPC implements stats.Handler.
+func (w *clientStatsHandlerWrapper) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context {
+	return w.base.TagRPC(ctx, info)
+}

internal/telemetry/trace/middleware_test.go

@@ -0,0 +1,210 @@
+package trace_test
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
+	"github.com/stretchr/testify/assert"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	oteltrace "go.opentelemetry.io/otel/trace"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/stats"
+)
+
+func TestHTTPMiddleware(t *testing.T) {
+	router := mux.NewRouter()
+	tp := sdktrace.NewTracerProvider()
+	router.Use(trace.NewHTTPMiddleware(
+		otelhttp.WithTracerProvider(tp),
+	))
+	router.Path("/foo").HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
+		span := oteltrace.SpanFromContext(r.Context())
+		assert.Equal(t, "Server: GET /foo", span.(interface{ Name() string }).Name())
+	}).Methods(http.MethodGet)
+	w := httptest.NewRecorder()
+	ctx, span := tp.Tracer("test").Start(context.Background(), "test")
+	router.ServeHTTP(w, httptest.NewRequestWithContext(ctx, http.MethodGet, "/foo", nil))
+	span.End()
+}
+
+type mockHandler struct {
+	handleConn func(ctx context.Context, stats stats.ConnStats)
+	handleRPC  func(ctx context.Context, stats stats.RPCStats)
+	tagConn    func(ctx context.Context, info *stats.ConnTagInfo) context.Context
+	tagRPC     func(ctx context.Context, info *stats.RPCTagInfo) context.Context
+}
+
+// HandleConn implements stats.Handler.
+func (m *mockHandler) HandleConn(ctx context.Context, stats stats.ConnStats) {
+	m.handleConn(ctx, stats)
+}
+
+// HandleRPC implements stats.Handler.
+func (m *mockHandler) HandleRPC(ctx context.Context, stats stats.RPCStats) {
+	m.handleRPC(ctx, stats)
+}
+
+// TagConn implements stats.Handler.
+func (m *mockHandler) TagConn(ctx context.Context, info *stats.ConnTagInfo) context.Context {
+	return m.tagConn(ctx, info)
+}
+
+// TagRPC implements stats.Handler.
+func (m *mockHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context {
+	return m.tagRPC(ctx, info)
+}
+
+var _ stats.Handler = (*mockHandler)(nil)
+
+func TestStatsInterceptor(t *testing.T) {
+	var outBegin *stats.Begin
+	var outEnd *stats.End
+	base := &mockHandler{
+		handleRPC: func(_ context.Context, rs stats.RPCStats) {
+			switch rs := rs.(type) {
+			case *stats.Begin:
+				outBegin = rs
+			case *stats.End:
+				outEnd = rs
+			}
+		},
+	}
+	interceptor := func(_ context.Context, rs stats.RPCStats) stats.RPCStats {
+		switch rs := rs.(type) {
+		case *stats.Begin:
+			return &stats.Begin{
+				Client:                    rs.Client,
+				BeginTime:                 rs.BeginTime.Add(-1 * time.Minute),
+				FailFast:                  rs.FailFast,
+				IsClientStream:            rs.IsClientStream,
+				IsServerStream:            rs.IsServerStream,
+				IsTransparentRetryAttempt: rs.IsTransparentRetryAttempt,
+			}
+		case *stats.End:
+			return &stats.End{
+				Client:    rs.Client,
+				BeginTime: rs.BeginTime,
+				EndTime:   rs.EndTime,
+				Trailer:   rs.Trailer,
+				Error:     errors.New("modified"),
+			}
+		}
+		return rs
+	}
+	handler := trace.NewClientStatsHandler(
+		base,
+		trace.WithStatsInterceptor(interceptor),
+	)
+	inBegin := &stats.Begin{
+		Client:                    true,
+		BeginTime:                 time.Now(),
+		FailFast:                  true,
+		IsClientStream:            true,
+		IsServerStream:            false,
+		IsTransparentRetryAttempt: false,
+	}
+	handler.HandleRPC(context.Background(), inBegin)
+	assert.NotNil(t, outBegin)
+	assert.NotSame(t, inBegin, outBegin)
+	assert.Equal(t, inBegin.BeginTime.Add(-1*time.Minute), outBegin.BeginTime)
+	assert.Equal(t, inBegin.Client, outBegin.Client)
+	assert.Equal(t, inBegin.FailFast, outBegin.FailFast)
+	assert.Equal(t, inBegin.IsClientStream, outBegin.IsClientStream)
+	assert.Equal(t, inBegin.IsServerStream, outBegin.IsServerStream)
+	assert.Equal(t, inBegin.IsTransparentRetryAttempt, outBegin.IsTransparentRetryAttempt)
+
+	inEnd := &stats.End{
+		Client:    true,
+		BeginTime: time.Now(),
+		EndTime:   time.Now().Add(1 * time.Minute),
+		Trailer:   metadata.Pairs("a", "b", "c", "d"),
+		Error:     errors.New("input"),
+	}
+	handler.HandleRPC(context.Background(), inEnd)
+	assert.NotNil(t, outEnd)
+	assert.NotSame(t, inEnd, outEnd)
+	assert.Equal(t, inEnd.Client, outEnd.Client)
+	assert.Equal(t, inEnd.BeginTime, outEnd.BeginTime)
+	assert.Equal(t, inEnd.EndTime, outEnd.EndTime)
+	assert.Equal(t, inEnd.Trailer, outEnd.Trailer)
+	assert.Equal(t, "input", inEnd.Error.Error())
+	assert.Equal(t, "modified", outEnd.Error.Error())
+}
+
+func TestStatsInterceptor_Nil(t *testing.T) {
+	var outCtx context.Context
+	var outConnStats stats.ConnStats
+	var outRPCStats stats.RPCStats
+	var outConnTagInfo *stats.ConnTagInfo
+	var outRPCTagInfo *stats.RPCTagInfo
+	base := &mockHandler{
+		handleConn: func(ctx context.Context, stats stats.ConnStats) {
+			outCtx = ctx
+			outConnStats = stats
+		},
+		handleRPC: func(ctx context.Context, stats stats.RPCStats) {
+			outCtx = ctx
+			outRPCStats = stats
+		},
+		tagConn: func(ctx context.Context, info *stats.ConnTagInfo) context.Context {
+			outCtx = ctx
+			outConnTagInfo = info
+			return ctx
+		},
+		tagRPC: func(ctx context.Context, info *stats.RPCTagInfo) context.Context {
+			outCtx = ctx
+			outRPCTagInfo = info
+			return ctx
+		},
+	}
+	handler := trace.NewClientStatsHandler(
+		base,
+		trace.WithStatsInterceptor(nil),
+	)
+
+	inCtx := context.Background()
+	inConnStats := &stats.ConnBegin{}
+	inRPCStats := &stats.Begin{}
+	inConnTagInfo := &stats.ConnTagInfo{}
+	inRPCTagInfo := &stats.RPCTagInfo{}
+
+	handler.HandleConn(inCtx, inConnStats)
+	assert.Equal(t, inCtx, outCtx)
+	assert.Same(t, inConnStats, outConnStats)
+
+	handler.HandleRPC(inCtx, inRPCStats)
+	assert.Equal(t, inCtx, outCtx)
+	assert.Same(t, inRPCStats, outRPCStats)
+
+	handler.TagConn(inCtx, inConnTagInfo)
+	assert.Equal(t, inCtx, outCtx)
+	assert.Same(t, inConnTagInfo, outConnTagInfo)
+
+	handler.TagRPC(inCtx, inRPCTagInfo)
+	assert.Equal(t, inCtx, outCtx)
+	assert.Same(t, inRPCTagInfo, outRPCTagInfo)
+}
+
+func TestStatsInterceptor_Bug(t *testing.T) {
+	handler := trace.NewClientStatsHandler(
+		&mockHandler{
+			handleRPC: func(_ context.Context, _ stats.RPCStats) {
+				t.Error("should not be reached")
+			},
+		},
+		trace.WithStatsInterceptor(func(_ context.Context, rs stats.RPCStats) stats.RPCStats {
+			_ = rs.(*stats.Begin)
+			return &stats.End{}
+		}),
+	)
+	assert.PanicsWithValue(t, "bug: stats interceptor returned a message of a different type", func() {
+		handler.HandleRPC(context.Background(), &stats.Begin{})
+	})
+}

internal/telemetry/trace/server.go

@@ -0,0 +1,99 @@
+package trace
+
+import (
+	"context"
+	"errors"
+	"net"
+	"time"
+
+	"github.com/pomerium/pomerium/internal/log"
+	coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/test/bufconn"
+
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+)
+
+const localExporterMetadataKey = "x-local-exporter"
+
+// Export implements ptraceotlp.GRPCServer.
+func (srv *ExporterServer) Export(ctx context.Context, req *coltracepb.ExportTraceServiceRequest) (*coltracepb.ExportTraceServiceResponse, error) {
+	if srv.observer != nil {
+		srv.observeExport(ctx, req)
+	}
+	if err := srv.remoteClient.UploadTraces(ctx, req.GetResourceSpans()); err != nil {
+		log.Ctx(ctx).Err(err).Msg("error uploading traces")
+		return nil, err
+	}
+	return &coltracepb.ExportTraceServiceResponse{}, nil
+}
+
+type ExporterServer struct {
+	coltracepb.UnimplementedTraceServiceServer
+	server       *grpc.Server
+	observer     *spanObserver
+	remoteClient otlptrace.Client
+	cc           *grpc.ClientConn
+}
+
+func NewServer(ctx context.Context) *ExporterServer {
+	sys := systemContextFromContext(ctx)
+	ex := &ExporterServer{
+		remoteClient: sys.options.RemoteClient,
+		observer:     sys.observer,
+		server:       grpc.NewServer(grpc.Creds(insecure.NewCredentials())),
+	}
+	coltracepb.RegisterTraceServiceServer(ex.server, ex)
+	return ex
+}
+
+func (srv *ExporterServer) Start(ctx context.Context) {
+	lis := bufconn.Listen(4096)
+	go func() {
+		if err := srv.remoteClient.Start(ctx); err != nil {
+			panic(err)
+		}
+		_ = srv.server.Serve(lis)
+	}()
+	cc, err := grpc.NewClient("passthrough://ignore",
+		grpc.WithContextDialer(func(context.Context, string) (net.Conn, error) {
+			return lis.Dial()
+		}), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		panic(err)
+	}
+	srv.cc = cc
+}
+
+func (srv *ExporterServer) NewClient() otlptrace.Client {
+	return otlptracegrpc.NewClient(
+		otlptracegrpc.WithGRPCConn(srv.cc),
+		otlptracegrpc.WithTimeout(1*time.Minute),
+		otlptracegrpc.WithHeaders(map[string]string{
+			localExporterMetadataKey: "1",
+		}),
+	)
+}
+
+func (srv *ExporterServer) Shutdown(ctx context.Context) error {
+	stopped := make(chan struct{})
+	go func() {
+		srv.server.GracefulStop()
+		close(stopped)
+	}()
+	select {
+	case <-stopped:
+	case <-ctx.Done():
+		return context.Cause(ctx)
+	}
+	var errs []error
+	if err := WaitForSpans(ctx, 30*time.Second); err != nil {
+		errs = append(errs, err)
+	}
+	if err := srv.remoteClient.Stop(ctx); err != nil {
+		errs = append(errs, err)
+	}
+	return errors.Join(errs...)
+}

internal/telemetry/trace/trace.go

@@ -2,87 +2,214 @@ package trace
 
 import (
 	"context"
+	"errors"
 	"fmt"
-	"net/url"
+	"runtime"
+	"sync"
+	"sync/atomic"
 
-	octrace "go.opencensus.io/trace"
-
-	"github.com/pomerium/pomerium/internal/log"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
+	"go.opentelemetry.io/otel/trace"
+	"go.opentelemetry.io/otel/trace/noop"
+	coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
 )
 
-const (
-	// DatadogTracingProviderName is the name of the tracing provider Datadog.
-	DatadogTracingProviderName = "datadog"
-	// JaegerTracingProviderName is the name of the tracing provider Jaeger.
-	JaegerTracingProviderName = "jaeger"
-	// ZipkinTracingProviderName is the name of the tracing provider Zipkin.
-	ZipkinTracingProviderName = "zipkin"
-)
-
-// Provider is a trace provider.
-type Provider interface {
-	Register(options *TracingOptions) error
-	Unregister() error
+type Options struct {
+	DebugFlags   DebugFlags
+	RemoteClient otlptrace.Client
 }
 
-// TracingOptions contains the configurations settings for a http server.
-type TracingOptions struct {
-	// Shared
-	Provider string
-	Service  string
-	Debug    bool
-
-	// Datadog
-	DatadogAddress string
-
-	// Jaeger
-
-	// CollectorEndpoint is the full url to the Jaeger HTTP Thrift collector.
-	// For example, http://localhost:14268/api/traces
-	JaegerCollectorEndpoint *url.URL
-	// AgentEndpoint instructs exporter to send spans to jaeger-agent at this address.
-	// For example, localhost:6831.
-	JaegerAgentEndpoint string
-
-	// Zipkin
-
-	// ZipkinEndpoint configures the zipkin collector URI
-	// Example: http://zipkin:9411/api/v2/spans
-	ZipkinEndpoint *url.URL
-
-	// SampleRate is percentage of requests which are sampled
-	SampleRate float64
-}
-
-// Enabled indicates whether tracing is enabled on a given TracingOptions
-func (t *TracingOptions) Enabled() bool {
-	return t.Provider != ""
-}
-
-// GetProvider creates a new trace provider from TracingOptions.
-func GetProvider(opts *TracingOptions) (Provider, error) {
-	var provider Provider
-	switch opts.Provider {
-	case DatadogTracingProviderName:
-		provider = new(datadogProvider)
-	case JaegerTracingProviderName:
-		provider = new(jaegerProvider)
-	case ZipkinTracingProviderName:
-		provider = new(zipkinProvider)
-	default:
-		return nil, fmt.Errorf("telemetry/trace: provider %s unknown", opts.Provider)
+func (op Options) NewContext(parent context.Context) context.Context {
+	if systemContextFromContext(parent) != nil {
+		panic("parent already contains trace system context")
 	}
-	octrace.ApplyConfig(octrace.Config{DefaultSampler: octrace.ProbabilitySampler(opts.SampleRate)})
-
-	log.Debug().Interface("Opts", opts).Msg("telemetry/trace: provider created")
-	return provider, nil
+	if op.RemoteClient == nil {
+		op.RemoteClient = NewRemoteClientFromEnv()
+	}
+	sys := &systemContext{
+		options: op,
+		tpm:     &tracerProviderManager{},
+	}
+	if op.DebugFlags.Check(TrackSpanReferences) {
+		sys.observer = newSpanObserver()
+	}
+	ctx := context.WithValue(parent, systemContextKey, sys)
+	sys.exporterServer = NewServer(ctx)
+	sys.exporterServer.Start(ctx)
+	return ctx
 }
 
-// StartSpan starts a new child span of the current span in the context. If
-// there is no span in the context, creates a new trace and span.
+// NewContext creates a new top-level background context with tracing machinery
+// and configuration that will be used when creating new tracer providers.
 //
-// Returned context contains the newly created span. You can use it to
-// propagate the returned span in process.
-func StartSpan(ctx context.Context, name string, o ...octrace.StartOption) (context.Context, *octrace.Span) {
-	return octrace.StartSpan(ctx, name, o...)
+// Any context created with NewContext should eventually be shut down by calling
+// [ShutdownContext] to ensure all traces are exported.
+//
+// The parent context should be context.Background(), or a background context
+// containing a logger. If any context in the parent's hierarchy was created
+// by NewContext, this will panic.
+func NewContext(parent context.Context) context.Context {
+	return Options{}.NewContext(parent)
+}
+
+// NewTracerProvider creates a new [trace.TracerProvider] with the given service
+// name and options.
+//
+// A context returned by [NewContext] must exist somewhere in the hierarchy of
+// ctx, otherwise a no-op TracerProvider is returned. The configuration embedded
+// within that context will be used to configure its resource attributes and
+// exporter automatically.
+func NewTracerProvider(ctx context.Context, serviceName string, opts ...sdktrace.TracerProviderOption) trace.TracerProvider {
+	sys := systemContextFromContext(ctx)
+	if sys == nil {
+		return noop.NewTracerProvider()
+	}
+	_, file, line, _ := runtime.Caller(1)
+	exp, err := otlptrace.New(ctx, sys.exporterServer.NewClient())
+	if err != nil {
+		panic(err)
+	}
+	r, err := resource.Merge(
+		resource.Default(),
+		resource.NewWithAttributes(
+			semconv.SchemaURL,
+			semconv.ServiceName(serviceName),
+			attribute.String("provider.created_at", fmt.Sprintf("%s:%d", file, line)),
+		),
+	)
+	if err != nil {
+		panic(err)
+	}
+	options := []sdktrace.TracerProviderOption{}
+	if sys.options.DebugFlags.Check(TrackSpanCallers) {
+		options = append(options, sdktrace.WithSpanProcessor(&stackTraceProcessor{}))
+	}
+	if sys.options.DebugFlags.Check(TrackSpanReferences) {
+		tracker := newSpanTracker(sys.observer, sys.options.DebugFlags)
+		options = append(options, sdktrace.WithSpanProcessor(tracker))
+	}
+	options = append(append(options,
+		sdktrace.WithBatcher(exp),
+		sdktrace.WithResource(r),
+	), opts...)
+	tp := sdktrace.NewTracerProvider(options...)
+	sys.tpm.Add(tp)
+	return tp
+}
+
+// Continue starts a new span using the tracer provider of the span in the given
+// context.
+//
+// In most cases, it is better to start spans directly from a specific tracer,
+// obtained via dependency injection or some other mechanism. This function is
+// useful in shared code where the tracer used to start the span is not
+// necessarily the same every time, but can change based on the call site.
+func Continue(ctx context.Context, name string, o ...trace.SpanStartOption) (context.Context, trace.Span) {
+	return trace.SpanFromContext(ctx).
+		TracerProvider().
+		Tracer(PomeriumCoreTracer).
+		Start(ctx, name, o...)
+}
+
+// ShutdownContext will gracefully shut down all tracing resources created with
+// a context returned by [NewContext], including all tracer providers and the
+// underlying exporter and remote client.
+//
+// This should only be called once before exiting, but subsequent calls are
+// a no-op.
+//
+// The provided context does not necessarily need to be the exact context
+// returned by [NewContext]; it can be anywhere in its context hierarchy and
+// this function will have the same effect.
+func ShutdownContext(ctx context.Context) error {
+	sys := systemContextFromContext(ctx)
+	if sys == nil {
+		panic("context was not created with trace.NewContext")
+	}
+
+	if !sys.shutdown.CompareAndSwap(false, true) {
+		return nil
+	}
+
+	var errs []error
+	if err := sys.tpm.ShutdownAll(context.Background()); err != nil {
+		errs = append(errs, fmt.Errorf("error shutting down tracer providers: %w", err))
+	}
+	if err := sys.exporterServer.Shutdown(context.Background()); err != nil {
+		errs = append(errs, fmt.Errorf("error shutting down trace exporter: %w", err))
+	}
+	return errors.Join(errs...)
+}
+
+func ExporterServerFromContext(ctx context.Context) coltracepb.TraceServiceServer {
+	if sys := systemContextFromContext(ctx); sys != nil {
+		return sys.exporterServer
+	}
+	return nil
+}
+
+func RemoteClientFromContext(ctx context.Context) otlptrace.Client {
+	if sys := systemContextFromContext(ctx); sys != nil {
+		return sys.options.RemoteClient
+	}
+	return nil
+}
+
+// ForceFlush immediately exports all spans that have not yet been exported for
+// all tracer providers created using the given context.
+func ForceFlush(ctx context.Context) error {
+	if sys := systemContextFromContext(ctx); sys != nil {
+		var errs []error
+		for _, tp := range sys.tpm.tracerProviders {
+			errs = append(errs, tp.ForceFlush(ctx))
+		}
+		return errors.Join(errs...)
+	}
+	return nil
+}
+
+type systemContextKeyType struct{}
+
+var systemContextKey systemContextKeyType
+
+type systemContext struct {
+	options        Options
+	tpm            *tracerProviderManager
+	observer       *spanObserver
+	exporterServer *ExporterServer
+	shutdown       atomic.Bool
+}
+
+func systemContextFromContext(ctx context.Context) *systemContext {
+	sys, _ := ctx.Value(systemContextKey).(*systemContext)
+	return sys
+}
+
+type tracerProviderManager struct {
+	mu              sync.Mutex
+	tracerProviders []*sdktrace.TracerProvider
+}
+
+func (tpm *tracerProviderManager) ShutdownAll(ctx context.Context) error {
+	tpm.mu.Lock()
+	defer tpm.mu.Unlock()
+	var errs []error
+	for _, tp := range tpm.tracerProviders {
+		errs = append(errs, tp.ForceFlush(ctx))
+	}
+	for _, tp := range tpm.tracerProviders {
+		errs = append(errs, tp.Shutdown(ctx))
+	}
+	clear(tpm.tracerProviders)
+	return errors.Join(errs...)
+}
+
+func (tpm *tracerProviderManager) Add(tp *sdktrace.TracerProvider) {
+	tpm.mu.Lock()
+	defer tpm.mu.Unlock()
+	tpm.tracerProviders = append(tpm.tracerProviders, tp)
 }

internal/telemetry/trace/trace_export_test.go

@@ -0,0 +1,72 @@
+package trace
+
+import (
+	"cmp"
+	"io"
+	"slices"
+	"testing"
+	"time"
+
+	oteltrace "go.opentelemetry.io/otel/trace"
+)
+
+var (
+	NewSpanObserver = newSpanObserver
+	NewSpanTracker  = newSpanTracker
+)
+
+type XStackTraceProcessor = stackTraceProcessor
+
+func (obs *spanObserver) XWait() {
+	obs.wait(5 * time.Second)
+}
+
+func (obs *spanObserver) XUnobservedIDs() []oteltrace.SpanID {
+	obs.cond.L.Lock()
+	defer obs.cond.L.Unlock()
+	ids := []oteltrace.SpanID{}
+	for k, v := range obs.referencedIDs {
+		if v.IsValid() {
+			ids = append(ids, k)
+		}
+	}
+	slices.SortFunc(ids, func(a, b oteltrace.SpanID) int {
+		return cmp.Compare(a.String(), b.String())
+	})
+	return ids
+}
+
+func (obs *spanObserver) XObservedIDs() []oteltrace.SpanID {
+	obs.cond.L.Lock()
+	defer obs.cond.L.Unlock()
+	ids := []oteltrace.SpanID{}
+	for k, v := range obs.referencedIDs {
+		if !v.IsValid() {
+			ids = append(ids, k)
+		}
+	}
+	slices.SortFunc(ids, func(a, b oteltrace.SpanID) int {
+		return cmp.Compare(a.String(), b.String())
+	})
+	return ids
+}
+
+func (t *spanTracker) XInflightSpans() []oteltrace.SpanID {
+	ids := []oteltrace.SpanID{}
+	t.inflightSpansMu.LockAll()
+	t.inflightSpans.Range(func(key oteltrace.SpanID) {
+		ids = append(ids, key)
+	})
+	t.inflightSpansMu.UnlockAll()
+	slices.SortFunc(ids, func(a, b oteltrace.SpanID) int {
+		return cmp.Compare(a.String(), b.String())
+	})
+	return ids
+}
+
+func SetDebugMessageWriterForTest(t testing.TB, w io.Writer) {
+	debugMessageWriter = w
+	t.Cleanup(func() {
+		debugMessageWriter = nil
+	})
+}

internal/telemetry/trace/trace_test.go

@@ -1,27 +0,0 @@
-package trace
-
-import (
-	"net/url"
-	"testing"
-)
-
-func TestGetProvider(t *testing.T) {
-	tests := []struct {
-		name    string
-		opts    *TracingOptions
-		wantErr bool
-	}{
-		{"jaeger", &TracingOptions{JaegerAgentEndpoint: "localhost:6831", Service: "all", Provider: "jaeger"}, false},
-		{"jaeger with debug", &TracingOptions{JaegerAgentEndpoint: "localhost:6831", Service: "all", Provider: "jaeger", Debug: true}, false},
-		{"jaeger no endpoint", &TracingOptions{JaegerAgentEndpoint: "", Service: "all", Provider: "jaeger"}, false},
-		{"unknown provider", &TracingOptions{JaegerAgentEndpoint: "localhost:0", Service: "all", Provider: "Lucius Cornelius Sulla"}, true},
-		{"zipkin with debug", &TracingOptions{ZipkinEndpoint: &url.URL{Host: "localhost"}, Service: "all", Provider: "zipkin", Debug: true}, false},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if _, err := GetProvider(tt.opts); (err != nil) != tt.wantErr {
-				t.Errorf("RegisterTracing() error = %v, wantErr %v", err, tt.wantErr)
-			}
-		})
-	}
-}

internal/telemetry/trace/util.go

@@ -0,0 +1,32 @@
+package trace
+
+import (
+	"unique"
+
+	oteltrace "go.opentelemetry.io/otel/trace"
+)
+
+var (
+	zeroSpanID  oteltrace.SpanID
+	zeroTraceID = unique.Make(oteltrace.TraceID([16]byte{}))
+)
+
+func ToSpanID(bytes []byte) (oteltrace.SpanID, bool) {
+	switch len(bytes) {
+	case 0:
+		return zeroSpanID, true
+	case 8:
+		return oteltrace.SpanID(bytes), true
+	}
+	return zeroSpanID, false
+}
+
+func ToTraceID(bytes []byte) (unique.Handle[oteltrace.TraceID], bool) {
+	switch len(bytes) {
+	case 0:
+		return zeroTraceID, true
+	case 16:
+		return unique.Make(oteltrace.TraceID(bytes)), true
+	}
+	return zeroTraceID, false
+}

internal/telemetry/trace/zipkin.go

@@ -1,49 +0,0 @@
-package trace
-
-import (
-	"fmt"
-	stdlog "log"
-
-	oczipkin "contrib.go.opencensus.io/exporter/zipkin"
-	"github.com/openzipkin/zipkin-go"
-	"github.com/openzipkin/zipkin-go/reporter"
-	zipkinHTTP "github.com/openzipkin/zipkin-go/reporter/http"
-	octrace "go.opencensus.io/trace"
-
-	"github.com/pomerium/pomerium/internal/log"
-)
-
-type zipkinProvider struct {
-	reporter reporter.Reporter
-	exporter *oczipkin.Exporter
-}
-
-func (provider *zipkinProvider) Register(opts *TracingOptions) error {
-	localEndpoint, err := zipkin.NewEndpoint(opts.Service, "")
-	if err != nil {
-		return fmt.Errorf("telemetry/trace: could not create local endpoint: %w", err)
-	}
-
-	logger := log.With().Str("service", "zipkin").Logger()
-	logWriter := &log.StdLogWrapper{Logger: &logger}
-	stdLogger := stdlog.New(logWriter, "", 0)
-
-	provider.reporter = zipkinHTTP.NewReporter(opts.ZipkinEndpoint.String(), zipkinHTTP.Logger(stdLogger))
-	provider.exporter = oczipkin.NewExporter(provider.reporter, localEndpoint)
-	octrace.RegisterExporter(provider.exporter)
-	return nil
-}
-
-func (provider *zipkinProvider) Unregister() error {
-	if provider.exporter != nil {
-		octrace.UnregisterExporter(provider.exporter)
-		provider.exporter = nil
-	}
-
-	var err error
-	if provider.reporter != nil {
-		err = provider.reporter.Close()
-		provider.reporter = nil
-	}
-	return err
-}

internal/testenv/environment.go

@@ -17,7 +17,6 @@ import (
 	"fmt"
 	"io"
 	"math/big"
-	"math/bits"
 	"net"
 	"net/url"
 	"os"
@@ -26,25 +25,34 @@ import (
 	"path/filepath"
 	"runtime"
 	"strconv"
+	"strings"
 	"sync"
+	"sync/atomic"
 	"syscall"
 	"testing"
 	"time"
 
 	"github.com/pomerium/pomerium/config"
 	"github.com/pomerium/pomerium/config/envoyconfig/filemgr"
+	databroker_service "github.com/pomerium/pomerium/databroker"
 	"github.com/pomerium/pomerium/internal/log"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/testenv/envutil"
 	"github.com/pomerium/pomerium/internal/testenv/values"
 	"github.com/pomerium/pomerium/pkg/cmd/pomerium"
 	"github.com/pomerium/pomerium/pkg/envoy"
 	"github.com/pomerium/pomerium/pkg/grpc/databroker"
 	"github.com/pomerium/pomerium/pkg/health"
+	"github.com/pomerium/pomerium/pkg/identity/legacymanager"
+	"github.com/pomerium/pomerium/pkg/identity/manager"
 	"github.com/pomerium/pomerium/pkg/netutil"
 	"github.com/pomerium/pomerium/pkg/slices"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	oteltrace "go.opentelemetry.io/otel/trace"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc/grpclog"
 )
@@ -56,6 +64,7 @@ type Environment interface {
 	// top-level logger scoped to this environment. It will be canceled when
 	// Stop() is called, or during test cleanup.
 	Context() context.Context
+	Tracer() oteltrace.Tracer
 
 	Assert() *assert.Assertions
 	Require() *require.Assertions
@@ -133,10 +142,29 @@ type Environment interface {
 	// the Pomerium server and Envoy.
 	NewLogRecorder(opts ...LogRecorderOption) *LogRecorder
 
+	// GetState returns the current state of the test environment.
+	GetState() EnvironmentState
+
 	// OnStateChanged registers a callback to be invoked when the environment's
-	// state changes to the given state. The callback is invoked in a separate
-	// goroutine.
-	OnStateChanged(state EnvironmentState, callback func())
+	// state changes to the given state. Each callback is invoked in a separate
+	// goroutine, but the test environment will wait for all callbacks to return
+	// before continuing, after triggering the state change.
+	//
+	// Calling the returned stop function will prevent the callback from being
+	// run. Returns true if it stopped the callback from being run, or false if
+	// it already ran or is currently running.
+	//
+	// If the environment is already in the given state, the callback will be run
+	// in a separate goroutine immediately and the returned stop function will
+	// have no effect. A callback run in this way will prevent the state from
+	// advancing until the callback returns.
+	//
+	// State changes are triggered in the following places:
+	// - NotRunning->Starting: in Start(), as the first operation
+	// - Starting->Running: in Start(), just before returning
+	// - Running->Stopping: in Stop(), just before the env context is canceled
+	// - Stopping->Stopped: in Stop(), after all tasks have completed
+	OnStateChanged(state EnvironmentState, callback func()) (stop func() bool)
 }
 
 type Certificate tls.Certificate
@@ -153,10 +181,9 @@ func (c *Certificate) SPKIHash() string {
 
 type EnvironmentState uint32
 
-const NotRunning EnvironmentState = 0
-
 const (
-	Starting EnvironmentState = 1 << iota
+	NotRunning EnvironmentState = iota
+	Starting
 	Running
 	Stopping
 	Stopped
@@ -192,10 +219,13 @@ type environment struct {
 	workspaceFolder string
 	silent          bool
 
-	ctx         context.Context
-	cancel      context.CancelCauseFunc
-	cleanupOnce sync.Once
-	logWriter   *log.MultiWriter
+	ctx            context.Context
+	cancel         context.CancelCauseFunc
+	cleanupOnce    sync.Once
+	logWriter      *log.MultiWriter
+	tracerProvider oteltrace.TracerProvider
+	tracer         oteltrace.Tracer
+	rootSpan       oteltrace.Span
 
 	mods         []WithCaller[Modifier]
 	tasks        []WithCaller[Task]
@@ -204,14 +234,17 @@ type environment struct {
 	stateMu              sync.Mutex
 	state                EnvironmentState
 	stateChangeListeners map[EnvironmentState][]func()
+	stateChangeBlockers  sync.WaitGroup
 
 	src *configSource
 }
 
 type EnvironmentOptions struct {
-	debug          bool
-	pauseOnFailure bool
-	forceSilent    bool
+	debug           bool
+	pauseOnFailure  bool
+	forceSilent     bool
+	traceDebugFlags trace.DebugFlags
+	traceClient     otlptrace.Client
 }
 
 type EnvironmentOption func(*EnvironmentOptions)
@@ -249,28 +282,57 @@ func Silent(silent ...bool) EnvironmentOption {
 	}
 }
 
+const StandardTraceDebugFlags = trace.TrackSpanCallers |
+	trace.WarnOnIncompleteSpans |
+	trace.WarnOnIncompleteTraces |
+	trace.WarnOnUnresolvedReferences |
+	trace.LogTraceIDsOnWarn |
+	trace.LogAllSpansOnWarn
+
+func WithTraceDebugFlags(flags trace.DebugFlags) EnvironmentOption {
+	return func(o *EnvironmentOptions) {
+		o.traceDebugFlags = flags
+	}
+}
+
+func WithTraceClient(traceClient otlptrace.Client) EnvironmentOption {
+	return func(o *EnvironmentOptions) {
+		o.traceClient = traceClient
+	}
+}
+
 var setGrpcLoggerOnce sync.Once
 
+const defaultTraceDebugFlags = trace.TrackSpanCallers | trace.TrackSpanReferences
+
 var (
-	flagDebug          = flag.Bool("env.debug", false, "enables test environment debug logging (equivalent to Debug() option)")
-	flagPauseOnFailure = flag.Bool("env.pause-on-failure", false, "enables pausing the test environment on failure (equivalent to PauseOnFailure() option)")
-	flagSilent         = flag.Bool("env.silent", false, "suppresses all test environment output (equivalent to Silent() option)")
+	flagDebug           = flag.Bool("env.debug", false, "enables test environment debug logging (equivalent to Debug() option)")
+	flagPauseOnFailure  = flag.Bool("env.pause-on-failure", false, "enables pausing the test environment on failure (equivalent to PauseOnFailure() option)")
+	flagSilent          = flag.Bool("env.silent", false, "suppresses all test environment output (equivalent to Silent() option)")
+	flagTraceDebugFlags = flag.String("env.trace-debug-flags", strconv.Itoa(defaultTraceDebugFlags), "trace debug flags (equivalent to TraceDebugFlags() option)")
 )
 
 func New(t testing.TB, opts ...EnvironmentOption) Environment {
-	if runtime.GOOS != "linux" {
-		t.Skip("test environment only supported on linux")
+	addTraceDebugFlags := strings.HasPrefix(*flagTraceDebugFlags, "+")
+	defaultTraceDebugFlags, err := strconv.Atoi(strings.TrimPrefix(*flagTraceDebugFlags, "+"))
+	if err != nil {
+		panic("malformed value for --env.trace-debug-flags: " + err.Error())
 	}
 	options := EnvironmentOptions{
-		debug:          *flagDebug,
-		pauseOnFailure: *flagPauseOnFailure,
-		forceSilent:    *flagSilent,
+		debug:           *flagDebug,
+		pauseOnFailure:  *flagPauseOnFailure,
+		forceSilent:     *flagSilent,
+		traceDebugFlags: trace.DebugFlags(defaultTraceDebugFlags),
 	}
 	options.apply(opts...)
 	if testing.Short() {
 		t.Helper()
 		t.Skip("test environment disabled in short mode")
 	}
+	if addTraceDebugFlags {
+		options.traceDebugFlags |= trace.DebugFlags(defaultTraceDebugFlags)
+	}
+	trace.UseGlobalPanicTracer()
 	databroker.DebugUseFasterBackoff.Store(true)
 	workspaceFolder, err := os.Getwd()
 	require.NoError(t, err)
@@ -305,7 +367,16 @@ func New(t testing.TB, opts ...EnvironmentOption) Environment {
 	})
 	logger := zerolog.New(writer).With().Timestamp().Logger().Level(zerolog.DebugLevel)
 
-	ctx, cancel := context.WithCancelCause(logger.WithContext(context.Background()))
+	ctx := trace.Options{
+		DebugFlags:   options.traceDebugFlags,
+		RemoteClient: options.traceClient,
+	}.NewContext(logger.WithContext(context.Background()))
+	tracerProvider := trace.NewTracerProvider(ctx, "Test Environment")
+	tracer := tracerProvider.Tracer(trace.PomeriumCoreTracer)
+	ctx, span := tracer.Start(ctx, t.Name(), oteltrace.WithNewRoot())
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancelCause(ctx)
 	taskErrGroup, ctx := errgroup.WithContext(ctx)
 
 	e := &environment{
@@ -313,7 +384,7 @@ func New(t testing.TB, opts ...EnvironmentOption) Environment {
 		t:                  t,
 		assert:             assert.New(t),
 		require:            require.New(t),
-		tempDir:            t.TempDir(),
+		tempDir:            tempDir(t),
 		ports: Ports{
 			ProxyHTTP:    values.Deferred[int](),
 			ProxyGRPC:    values.Deferred[int](),
@@ -325,13 +396,18 @@ func New(t testing.TB, opts ...EnvironmentOption) Environment {
 			Debug:        values.Deferred[int](),
 			ALPN:         values.Deferred[int](),
 		},
-		workspaceFolder: workspaceFolder,
-		silent:          silent,
-		ctx:             ctx,
-		cancel:          cancel,
-		logWriter:       writer,
-		taskErrGroup:    taskErrGroup,
+		workspaceFolder:      workspaceFolder,
+		silent:               silent,
+		ctx:                  ctx,
+		cancel:               cancel,
+		tracerProvider:       tracerProvider,
+		tracer:               tracer,
+		logWriter:            writer,
+		taskErrGroup:         taskErrGroup,
+		stateChangeListeners: make(map[EnvironmentState][]func()),
+		rootSpan:             span,
 	}
+
 	_, err = rand.Read(e.sharedSecret[:])
 	require.NoError(t, err)
 	_, err = rand.Read(e.cookieSecret[:])
@@ -362,11 +438,13 @@ func New(t testing.TB, opts ...EnvironmentOption) Environment {
 
 func (e *environment) debugf(format string, args ...any) {
 	e.t.Helper()
+	if e.rootSpan.IsRecording() {
+		e.rootSpan.AddEvent(fmt.Sprintf(format, args...))
+	}
 	if !e.debug {
 		return
 	}
-
-	e.t.Logf("\x1b[34m[debug] "+format+"\x1b[0m", args...)
+	e.t.Logf("\x1b[34mDEBUG ["+e.t.Name()+"] "+format+"\x1b[0m", args...)
 }
 
 type WithCaller[T any] struct {
@@ -394,6 +472,10 @@ func (e *environment) Context() context.Context {
 	return ContextWithEnv(e.ctx, e)
 }
 
+func (e *environment) Tracer() oteltrace.Tracer {
+	return e.tracer
+}
+
 func (e *environment) Assert() *assert.Assertions {
 	return e.assert
 }
@@ -455,9 +537,11 @@ var ErrCauseTestCleanup = errors.New("test cleanup")
 var ErrCauseManualStop = errors.New("Stop() called")
 
 func (e *environment) Start() {
+	_, span := e.tracer.Start(e.Context(), "Start")
+	defer span.End()
 	e.debugf("Start()")
 	e.advanceState(Starting)
-	e.t.Cleanup(e.cleanup)
+	e.t.Cleanup(e.onTestCleanup)
 	e.t.Setenv("TMPDIR", e.TempDir())
 	e.debugf("temp dir: %s", e.TempDir())
 
@@ -524,8 +608,13 @@ func (e *environment) Start() {
 			require.NoError(e.t, cfg.Options.Validate(), "invoking modifier resulted in an invalid configuration:\nadded by: "+mod.Caller)
 		}
 
-		opts := []pomerium.RunOption{
+		opts := []pomerium.Option{
 			pomerium.WithOverrideFileManager(fileMgr),
+			pomerium.WithEnvoyServerOptions(envoy.WithExitGracePeriod(30 * time.Second)),
+			pomerium.WithDataBrokerServerOptions(
+				databroker_service.WithManagerOptions(manager.WithLeaseTTL(1*time.Second)),
+				databroker_service.WithLegacyManagerOptions(legacymanager.WithLeaseTTL(1*time.Second)),
+			),
 		}
 		envoyBinaryPath := filepath.Join(e.workspaceFolder, fmt.Sprintf("pkg/envoy/files/envoy-%s-%s", runtime.GOOS, runtime.GOARCH))
 		if envutil.EnvoyProfilerAvailable(envoyBinaryPath) {
@@ -556,23 +645,29 @@ func (e *environment) Start() {
 			}
 			if len(envVars) > 0 {
 				e.debugf("adding envoy env vars: %v\n", envVars)
-				opts = append(opts, pomerium.WithEnvoyServerOptions(
-					envoy.WithExtraEnvVars(envVars...),
-					envoy.WithExitGracePeriod(10*time.Second), // allow envoy time to flush pprof data to disk
-				))
+				opts = append(opts, pomerium.WithEnvoyServerOptions(envoy.WithExtraEnvVars(envVars...)))
 			}
 		} else {
 			e.debugf("envoy profiling not available")
 		}
 
-		return pomerium.Run(ctx, e.src, opts...)
+		pom := pomerium.New(opts...)
+		e.OnStateChanged(Stopping, func() {
+			if err := pom.Shutdown(ctx); err != nil {
+				log.Ctx(ctx).Err(err).Msg("error shutting down pomerium server")
+			} else {
+				e.debugf("pomerium server shut down without error")
+			}
+		})
+		require.NoError(e.t, pom.Start(ctx, e.tracerProvider, e.src))
+		return pom.Wait()
 	}))
 
 	for i, task := range e.tasks {
-		log.Ctx(e.ctx).Debug().Str("caller", task.Caller).Msgf("starting task %d", i)
+		log.Ctx(e.Context()).Debug().Str("caller", task.Caller).Msgf("starting task %d", i)
 		e.taskErrGroup.Go(func() error {
-			defer log.Ctx(e.ctx).Debug().Str("caller", task.Caller).Msgf("task %d exited", i)
-			return task.Value.Run(e.ctx)
+			defer log.Ctx(e.Context()).Debug().Str("caller", task.Caller).Msgf("task %d exited", i)
+			return task.Value.Run(e.Context())
 		})
 	}
 
@@ -695,14 +790,9 @@ func (e *environment) Stop() {
 		b.StopTimer()
 		defer b.StartTimer()
 	}
+	_, file, line, _ := runtime.Caller(1)
 	e.cleanupOnce.Do(func() {
-		e.debugf("stop: Stop() called manually")
-		e.advanceState(Stopping)
-		e.cancel(ErrCauseManualStop)
-		err := e.taskErrGroup.Wait()
-		e.advanceState(Stopped)
-		e.debugf("stop: done waiting")
-		assert.ErrorIs(e.t, err, ErrCauseManualStop)
+		e.cleanup(fmt.Errorf("%w (caller: %s:%d)", ErrCauseManualStop, file, line))
 	})
 }
 
@@ -714,33 +804,51 @@ func (e *environment) Pause() {
 	e.t.Log("\x1b[31mctrl+c received, continuing\x1b[0m")
 }
 
-func (e *environment) cleanup() {
+func (e *environment) onTestCleanup() {
 	e.cleanupOnce.Do(func() {
-		e.debugf("stop: test cleanup")
-		if e.t.Failed() {
-			if e.pauseOnFailure {
-				e.t.Log("\x1b[31m*** pausing on test failure; continue with ctrl+c ***\x1b[0m")
-				c := make(chan os.Signal, 1)
-				signal.Notify(c, syscall.SIGINT)
-				<-c
-				e.t.Log("\x1b[31mctrl+c received, continuing\x1b[0m")
-				signal.Stop(c)
-			}
-		}
-		e.advanceState(Stopping)
-		e.cancel(ErrCauseTestCleanup)
-		err := e.taskErrGroup.Wait()
-		e.advanceState(Stopped)
-		e.debugf("stop: done waiting")
-		assert.ErrorIs(e.t, err, ErrCauseTestCleanup)
+		e.cleanup(ErrCauseTestCleanup)
 	})
 }
 
+func (e *environment) cleanup(cancelCause error) {
+	e.debugf("stop: %s", cancelCause.Error())
+	if e.t.Failed() {
+		if e.pauseOnFailure {
+			e.t.Log("\x1b[31m*** pausing on test failure; continue with ctrl+c ***\x1b[0m")
+			c := make(chan os.Signal, 1)
+			signal.Notify(c, syscall.SIGINT)
+			<-c
+			e.t.Log("\x1b[31mctrl+c received, continuing\x1b[0m")
+			signal.Stop(c)
+		}
+	}
+	e.advanceState(Stopping)
+	e.cancel(cancelCause)
+	errs := []error{}
+	if err := e.taskErrGroup.Wait(); err != nil {
+		errs = append(errs, fmt.Errorf("error waiting for tasks: %w", err))
+	}
+	e.rootSpan.End()
+	if err := trace.ShutdownContext(e.Context()); err != nil {
+		errs = append(errs, fmt.Errorf("error shutting down trace context: %w", err))
+	}
+	e.advanceState(Stopped)
+	// Wait for any additional callbacks created during stopped callbacks
+	// (for consistency, we consider the stopped state to "end" here)
+	e.stateChangeBlockers.Wait()
+	e.debugf("stop: done")
+	// err can be nil if e.g. the only task is the internal pomerium task, which
+	// returns a nil error if it exits cleanly
+	if err := errors.Join(errs...); err != nil {
+		assert.ErrorIs(e.t, err, cancelCause)
+	}
+}
+
 func (e *environment) Add(m Modifier) {
 	e.t.Helper()
 	caller := getCaller()
 	e.debugf("Add: %T from %s", m, caller)
-	switch e.getState() {
+	switch e.GetState() {
 	case NotRunning:
 		for _, mod := range e.mods {
 			if mod.Value == m {
@@ -757,11 +865,11 @@ func (e *environment) Add(m Modifier) {
 		panic("test bug: cannot call Add() before Start() has returned")
 	case Running:
 		e.debugf("Add: state=Running; calling ModifyConfig")
-		e.src.ModifyConfig(e.ctx, m)
+		e.src.ModifyConfig(e.Context(), m)
 	case Stopped, Stopping:
 		panic("test bug: cannot call Add() after Stop()")
 	default:
-		panic(fmt.Sprintf("unexpected environment state: %s", e.getState()))
+		panic(fmt.Sprintf("unexpected environment state: %s", e.GetState()))
 	}
 }
 
@@ -803,34 +911,63 @@ func (e *environment) advanceState(newState EnvironmentState) {
 	if newState <= e.state {
 		panic(fmt.Sprintf("internal test environment bug: changed state to <= current: newState=%s, current=%s", newState, e.state))
 	}
+	e.stateChangeBlockers.Wait()
 	e.debugf("state %s -> %s", e.state.String(), newState.String())
 	e.state = newState
-	e.debugf("notifying %d listeners of state change", len(e.stateChangeListeners[newState]))
-	for _, listener := range e.stateChangeListeners[newState] {
-		go listener()
+	if len(e.stateChangeListeners[newState]) > 0 {
+		e.debugf("notifying %d listeners of state change", len(e.stateChangeListeners[newState]))
+		var wg sync.WaitGroup
+		for _, listener := range e.stateChangeListeners[newState] {
+			wg.Add(1)
+			go func() {
+				_, span := e.tracer.Start(e.Context(), "State Change Callback")
+				span.SetAttributes(attribute.String("state", newState.String()))
+				defer span.End()
+				defer wg.Done()
+				listener()
+			}()
+		}
+		wg.Wait()
+		e.debugf("done notifying state change listeners")
 	}
 }
 
-func (e *environment) getState() EnvironmentState {
+func (e *environment) GetState() EnvironmentState {
 	e.stateMu.Lock()
 	defer e.stateMu.Unlock()
 	return e.state
 }
 
-func (e *environment) OnStateChanged(state EnvironmentState, callback func()) {
+func (e *environment) OnStateChanged(state EnvironmentState, callback func()) (cancel func() bool) {
 	e.stateMu.Lock()
 	defer e.stateMu.Unlock()
 
-	if e.state&state != 0 {
-		go callback()
-		return
-	}
-
-	// add change listeners for all states, if there are multiple bits set
-	for state > 0 {
-		stateBit := EnvironmentState(bits.TrailingZeros32(uint32(state)))
-		state &= (state - 1)
-		e.stateChangeListeners[stateBit] = append(e.stateChangeListeners[stateBit], callback)
+	_, file, line, _ := runtime.Caller(1)
+	switch {
+	case state < e.state:
+		panic(fmt.Sprintf("test bug: OnStateChanged called with state %s which is < current state (%s)", state, e.sharedSecret))
+	case state == e.state:
+		e.stateChangeBlockers.Add(1)
+		e.debugf("invoking callback for current state (state: %s, caller: %s:%d)", state.String(), file, line)
+		go func() {
+			defer func() {
+				e.stateChangeBlockers.Done()
+			}()
+			callback()
+		}()
+		return func() bool { return false }
+	default:
+		canceled := &atomic.Bool{}
+		e.stateChangeListeners[state] = append(e.stateChangeListeners[state], func() {
+			if canceled.CompareAndSwap(false, true) {
+				e.debugf("invoking state change callback (caller: %s:%d)", file, line)
+				callback()
+			}
+		})
+		return func() bool {
+			e.debugf("stopped state change callback (state: %s, caller: %s:%d)", state.String(), file, line)
+			return canceled.CompareAndSwap(false, true)
+		}
 	}
 }
 

internal/testenv/logs.go

@@ -12,6 +12,7 @@ import (
 	"io"
 	"os"
 	"reflect"
+	"regexp"
 	"sync"
 	"testing"
 	"time"
@@ -128,7 +129,7 @@ func (e *environment) NewLogRecorder(opts ...LogRecorderOption) *LogRecorder {
 	lr := &LogRecorder{
 		LogRecorderOptions: options,
 		t:                  e.t,
-		canceled:           e.ctx.Done(),
+		canceled:           e.Context().Done(),
 		buf:                newBuffer(),
 	}
 	e.logWriter.Add(lr.buf)
@@ -139,7 +140,7 @@ func (e *environment) NewLogRecorder(opts ...LogRecorderOption) *LogRecorder {
 		}
 		e.logWriter.Remove(lr.buf)
 	})
-	context.AfterFunc(e.ctx, lr.removeGlobalWriterOnce)
+	context.AfterFunc(e.Context(), lr.removeGlobalWriterOnce)
 	return lr
 }
 
@@ -356,6 +357,11 @@ func match(expected, actual map[string]any, open bool) (matched bool, score int)
 					return false, score
 				}
 				score++
+			case *regexp.Regexp:
+				if !value.MatchString(actualValue) {
+					return false, score
+				}
+				score++
 			default:
 				return false, score
 			}

internal/testenv/scenarios/mock_idp.go

@@ -45,7 +45,7 @@ type IDP struct {
 func (idp *IDP) Attach(ctx context.Context) {
 	env := testenv.EnvFromContext(ctx)
 
-	router := upstreams.HTTP(nil)
+	router := upstreams.HTTP(nil, upstreams.WithDisplayName("IDP"))
 
 	idp.url = values.Bind2(env.SubdomainURL("mock-idp"), router.Port(), func(urlStr string, port int) string {
 		u, _ := url.Parse(urlStr)

internal/testenv/scenarios/trace_receiver.go

@@ -0,0 +1,207 @@
+package scenarios
+
+import (
+	"compress/gzip"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/pomerium/pomerium/config"
+	"github.com/pomerium/pomerium/internal/testenv"
+	"github.com/pomerium/pomerium/internal/testenv/upstreams"
+	"github.com/pomerium/pomerium/internal/testenv/values"
+	"github.com/pomerium/pomerium/internal/testutil/tracetest"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
+	tracev1 "go.opentelemetry.io/proto/otlp/trace/v1"
+	"google.golang.org/protobuf/proto"
+)
+
+type OTLPTraceReceiver struct {
+	coltracepb.UnimplementedTraceServiceServer
+
+	mu               sync.Mutex
+	receivedRequests []*coltracepb.ExportTraceServiceRequest
+	grpcUpstream     values.MutableValue[upstreams.GRPCUpstream]
+	httpUpstream     values.MutableValue[upstreams.HTTPUpstream]
+}
+
+func NewOTLPTraceReceiver() *OTLPTraceReceiver {
+	return &OTLPTraceReceiver{
+		grpcUpstream: values.Deferred[upstreams.GRPCUpstream](),
+		httpUpstream: values.Deferred[upstreams.HTTPUpstream](),
+	}
+}
+
+// Export implements v1.TraceServiceServer.
+func (rec *OTLPTraceReceiver) Export(_ context.Context, req *coltracepb.ExportTraceServiceRequest) (*coltracepb.ExportTraceServiceResponse, error) {
+	rec.mu.Lock()
+	defer rec.mu.Unlock()
+	rec.receivedRequests = append(rec.receivedRequests, req)
+	return &coltracepb.ExportTraceServiceResponse{}, nil
+}
+
+// Attach implements testenv.Modifier.
+func (rec *OTLPTraceReceiver) Attach(ctx context.Context) {
+	env := testenv.EnvFromContext(ctx)
+
+	// NB: we cannot install tracing middleware into the receiver server, since
+	// it will cause a feedback loop of spans created when exporting other spans
+
+	grpcUpstream := upstreams.GRPC(nil,
+		upstreams.WithDisplayName("OTLP GRPC Receiver"),
+		upstreams.WithDelayedShutdown(),
+		upstreams.WithNoClientTracing(),
+		upstreams.WithNoServerTracing(),
+	)
+	httpUpstream := upstreams.HTTP(nil,
+		upstreams.WithDisplayName("OTLP HTTP Receiver"),
+		upstreams.WithDelayedShutdown(),
+		upstreams.WithNoClientTracing(),
+		upstreams.WithNoServerTracing(),
+	)
+
+	coltracepb.RegisterTraceServiceServer(grpcUpstream, rec)
+	httpUpstream.Handle("/v1/traces", rec.handleV1Traces)
+	env.AddUpstream(grpcUpstream)
+	env.AddUpstream(httpUpstream)
+	rec.grpcUpstream.Resolve(grpcUpstream)
+	rec.httpUpstream.Resolve(httpUpstream)
+}
+
+// Modify implements testenv.Modifier.
+func (rec *OTLPTraceReceiver) Modify(*config.Config) {}
+
+func (rec *OTLPTraceReceiver) handleV1Traces(w http.ResponseWriter, r *http.Request) {
+	if r.Header.Get("Content-Type") != "application/x-protobuf" {
+		w.WriteHeader(http.StatusBadRequest)
+		_, _ = w.Write([]byte("invalid content type"))
+		return
+	}
+	reader := r.Body
+	if r.Header.Get("Content-Encoding") == "gzip" {
+		var err error
+		reader, err = gzip.NewReader(r.Body)
+		if err != nil {
+			w.WriteHeader(http.StatusBadRequest)
+			_, _ = w.Write([]byte(err.Error()))
+			return
+		}
+	}
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		_, _ = w.Write([]byte(err.Error()))
+		return
+	}
+	var req coltracepb.ExportTraceServiceRequest
+	if err := proto.Unmarshal(data, &req); err != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		_, _ = w.Write([]byte(err.Error()))
+		return
+	}
+	resp, err := rec.Export(context.TODO(), &req)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		_, _ = w.Write([]byte(err.Error()))
+		return
+	}
+	respData, err := proto.Marshal(resp)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		_, _ = w.Write([]byte(err.Error()))
+		return
+	}
+	w.Header().Set("Content-Type", "application/x-protobuf")
+	w.WriteHeader(http.StatusOK)
+	_, _ = w.Write(respData)
+}
+
+func (rec *OTLPTraceReceiver) ReceivedRequests() []*coltracepb.ExportTraceServiceRequest {
+	rec.mu.Lock()
+	defer rec.mu.Unlock()
+	return rec.receivedRequests
+}
+
+func (rec *OTLPTraceReceiver) PeekResourceSpans() []*tracev1.ResourceSpans {
+	rec.mu.Lock()
+	defer rec.mu.Unlock()
+
+	return rec.peekResourceSpansLocked()
+}
+
+func (rec *OTLPTraceReceiver) peekResourceSpansLocked() []*tracev1.ResourceSpans {
+	return tracetest.FlattenExportRequests(rec.receivedRequests)
+}
+
+func (rec *OTLPTraceReceiver) FlushResourceSpans() []*tracev1.ResourceSpans {
+	rec.mu.Lock()
+	defer rec.mu.Unlock()
+	spans := rec.peekResourceSpansLocked()
+	rec.receivedRequests = nil
+	return spans
+}
+
+// GRPCEndpointURL returns a url suitable for use with the environment variable
+// $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT or with [otlptracegrpc.WithEndpointURL].
+func (rec *OTLPTraceReceiver) GRPCEndpointURL() values.Value[string] {
+	return values.Chain(rec.grpcUpstream, upstreams.GRPCUpstream.Port, func(port int) string {
+		return fmt.Sprintf("http://127.0.0.1:%d", port)
+	})
+}
+
+// GRPCEndpointURL returns a url suitable for use with the environment variable
+// $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT or with [otlptracehttp.WithEndpointURL].
+func (rec *OTLPTraceReceiver) HTTPEndpointURL() values.Value[string] {
+	return values.Chain(rec.httpUpstream, upstreams.HTTPUpstream.Port, func(port int) string {
+		return fmt.Sprintf("http://127.0.0.1:%d/v1/traces", port)
+	})
+}
+
+func (rec *OTLPTraceReceiver) NewGRPCClient(opts ...otlptracegrpc.Option) otlptrace.Client {
+	return &deferredClient{
+		client: values.Bind(rec.grpcUpstream, func(up upstreams.GRPCUpstream) otlptrace.Client {
+			return otlptracegrpc.NewClient(append(opts,
+				otlptracegrpc.WithGRPCConn(up.DirectConnect()),
+				otlptracegrpc.WithTimeout(1*time.Minute),
+			)...)
+		}),
+	}
+}
+
+func (rec *OTLPTraceReceiver) NewHTTPClient(opts ...otlptracehttp.Option) otlptrace.Client {
+	return &deferredClient{
+		client: values.Chain(rec.httpUpstream, upstreams.HTTPUpstream.Port, func(port int) otlptrace.Client {
+			return otlptracehttp.NewClient(append(opts,
+				otlptracehttp.WithEndpointURL(fmt.Sprintf("http://127.0.0.1:%d/v1/traces", port)),
+				otlptracehttp.WithTimeout(1*time.Minute),
+			)...)
+		}),
+	}
+}
+
+type deferredClient struct {
+	client values.Value[otlptrace.Client]
+}
+
+// Start implements otlptrace.Client.
+func (o *deferredClient) Start(ctx context.Context) error {
+	return o.client.Value().Start(ctx)
+}
+
+// Stop implements otlptrace.Client.
+func (o *deferredClient) Stop(ctx context.Context) error {
+	return o.client.Value().Stop(ctx)
+}
+
+// UploadTraces implements otlptrace.Client.
+func (o *deferredClient) UploadTraces(ctx context.Context, protoSpans []*tracev1.ResourceSpans) error {
+	return o.client.Value().UploadTraces(ctx, protoSpans)
+}
+
+var _ otlptrace.Client = (*deferredClient)(nil)

internal/testenv/selftests/tracing_test.go

@@ -0,0 +1,402 @@
+package selftests_test
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"maps"
+	"net/http"
+	"os"
+	"slices"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/pomerium/pomerium/config"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
+	"github.com/pomerium/pomerium/internal/testenv"
+	"github.com/pomerium/pomerium/internal/testenv/scenarios"
+	"github.com/pomerium/pomerium/internal/testenv/snippets"
+	"github.com/pomerium/pomerium/internal/testenv/upstreams"
+	. "github.com/pomerium/pomerium/internal/testutil/tracetest" //nolint:revive
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
+	oteltrace "go.opentelemetry.io/otel/trace"
+)
+
+func otlpTraceReceiverOrFromEnv(t *testing.T) (modifier testenv.Modifier, newRemoteClient func() otlptrace.Client, getResults func() *TraceResults) {
+	t.Setenv("OTEL_TRACES_EXPORTER", "otlp")
+	tracesEndpoint := os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")
+	if tracesEndpoint == "" {
+		tracesEndpoint = os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
+		if tracesEndpoint == "" {
+			srv := scenarios.NewOTLPTraceReceiver()
+			return srv,
+				func() otlptrace.Client {
+					return srv.NewGRPCClient()
+				},
+				func() *TraceResults {
+					return NewTraceResults(srv.FlushResourceSpans())
+				}
+		}
+	}
+	return testenv.NoopModifier(), trace.NewRemoteClientFromEnv, nil
+}
+
+var allServices = []string{
+	"Test Environment",
+	"Authorize",
+	"Authenticate",
+	"Control Plane",
+	"Data Broker",
+	"Proxy",
+	"Upstream",
+	"IDP",
+	"HTTP Client",
+	"Envoy",
+}
+
+func TestOTLPTracing(t *testing.T) {
+	modifier, newRemoteClient, getResults := otlpTraceReceiverOrFromEnv(t)
+	env := testenv.New(t, testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags), testenv.WithTraceClient(newRemoteClient()))
+	env.Add(modifier)
+
+	up := upstreams.HTTP(nil, upstreams.WithDisplayName("Upstream"))
+	up.Handle("/foo", func(w http.ResponseWriter, _ *http.Request) {
+		w.Write([]byte("OK"))
+	})
+	env.Add(scenarios.NewIDP([]*scenarios.User{
+		{
+			Email:     "foo@example.com",
+			FirstName: "Firstname",
+			LastName:  "Lastname",
+		},
+	}))
+
+	route := up.Route().
+		From(env.SubdomainURL("foo")).
+		PPL(`{"allow":{"and":["email":{"is":"foo@example.com"}]}}`)
+
+	env.AddUpstream(up)
+	env.Start()
+	snippets.WaitStartupComplete(env)
+
+	ctx, span := env.Tracer().Start(env.Context(), "Authenticate", oteltrace.WithNewRoot())
+	resp, err := up.Get(route, upstreams.AuthenticateAs("foo@example.com"), upstreams.Path("/foo"), upstreams.Context(ctx))
+	span.End()
+	require.NoError(t, err)
+	body, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, resp.StatusCode, 200)
+	assert.Equal(t, "OK", string(body))
+
+	env.Stop()
+
+	if getResults != nil {
+		results := getResults()
+		var (
+			testEnvironmentLocalTest    = fmt.Sprintf("Test Environment: %s", t.Name())
+			testEnvironmentAuthenticate = "Test Environment: Authenticate"
+			authenticateOAuth2Client    = "Authenticate: OAuth2 Client: GET /.well-known/jwks.json"
+			idpServerGetUserinfo        = "IDP: Server: GET /oidc/userinfo"
+			idpServerPostToken          = "IDP: Server: POST /oidc/token"
+			controlPlaneEnvoyAccessLogs = "Control Plane: envoy.service.accesslog.v3.AccessLogService/StreamAccessLogs"
+			controlPlaneEnvoyDiscovery  = "Control Plane: envoy.service.discovery.v3.AggregatedDiscoveryService/DeltaAggregatedResources"
+			controlPlaneExport          = "Control Plane: opentelemetry.proto.collector.trace.v1.TraceService/Export"
+		)
+
+		results.MatchTraces(t,
+			MatchOptions{
+				Exact:              true,
+				CheckDetachedSpans: true,
+			},
+			Match{Name: testEnvironmentLocalTest, TraceCount: 1, Services: []string{"Test Environment", "Control Plane", "Data Broker"}},
+			Match{Name: testEnvironmentAuthenticate, TraceCount: 1, Services: allServices},
+			Match{Name: authenticateOAuth2Client, TraceCount: Greater(0)},
+			Match{Name: idpServerGetUserinfo, TraceCount: EqualToMatch(authenticateOAuth2Client)},
+			Match{Name: idpServerPostToken, TraceCount: EqualToMatch(authenticateOAuth2Client)},
+			Match{Name: controlPlaneEnvoyDiscovery, TraceCount: 1},
+			Match{Name: controlPlaneExport, TraceCount: Greater(0)},
+			Match{Name: controlPlaneEnvoyAccessLogs, TraceCount: Any{}},
+		)
+	}
+}
+
+func TestOTLPTracing_TraceCorrelation(t *testing.T) {
+	modifier, newRemoteClient, getResults := otlpTraceReceiverOrFromEnv(t)
+	env := testenv.New(t, testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags), testenv.WithTraceClient(newRemoteClient()))
+	env.Add(modifier)
+
+	up := upstreams.HTTP(nil, upstreams.WithDisplayName("Upstream"), upstreams.WithNoClientTracing())
+	up.Handle("/foo", func(w http.ResponseWriter, _ *http.Request) {
+		w.Write([]byte("OK"))
+	})
+	env.Add(scenarios.NewIDP([]*scenarios.User{
+		{
+			Email:     "foo@example.com",
+			FirstName: "Firstname",
+			LastName:  "Lastname",
+		},
+	}))
+
+	route := up.Route().
+		From(env.SubdomainURL("foo")).
+		PPL(`{"allow":{"and":["email":{"is":"foo@example.com"}]}}`)
+
+	env.AddUpstream(up)
+	env.Start()
+	snippets.WaitStartupComplete(env)
+
+	resp, err := up.Get(route, upstreams.AuthenticateAs("foo@example.com"), upstreams.Path("/foo"), upstreams.Context(context.Background()))
+	require.NoError(t, err)
+	body, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, resp.StatusCode, 200)
+	assert.Equal(t, "OK", string(body))
+
+	env.Stop()
+	if getResults != nil {
+		results := getResults()
+		traces := results.GetTraces()
+		// one unauthenticated (ends in /.pomerium/callback redirect), one authenticated
+		assert.Len(t, traces.ByName[fmt.Sprintf("Envoy: ingress: GET foo.localhost.pomerium.io:%d/foo", env.Ports().ProxyHTTP.Value())].WithoutErrors(), 2)
+	}
+}
+
+type SamplingTestSuite struct {
+	suite.Suite
+	env        testenv.Environment
+	getResults func() *TraceResults
+	route      testenv.Route
+	upstream   upstreams.HTTPUpstream
+
+	sampled    atomic.Int32
+	notSampled atomic.Int32
+}
+
+func (s *SamplingTestSuite) SetupTest() {
+	modifier, newRemoteClient, getResults := otlpTraceReceiverOrFromEnv(s.T())
+	s.getResults = getResults
+	s.env = testenv.New(s.T(),
+		testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags|trace.EnvoyFlushEverySpan),
+		testenv.WithTraceClient(newRemoteClient()),
+	)
+	s.env.Add(modifier)
+
+	s.sampled.Store(0)
+	s.notSampled.Store(0)
+
+	s.env.Add(testenv.ModifierFunc(func(_ context.Context, cfg *config.Config) {
+		half := 0.5
+		cfg.Options.TracingSampleRate = &half
+	}))
+	s.env.Add(scenarios.NewIDP([]*scenarios.User{
+		{
+			Email:     "foo@example.com",
+			FirstName: "Firstname",
+			LastName:  "Lastname",
+		},
+	}))
+
+	s.upstream = upstreams.HTTP(nil, upstreams.WithNoClientTracing(), upstreams.WithDisplayName("Upstream"))
+	s.upstream.Handle("/", s.handleRequest)
+
+	s.route = s.upstream.Route().
+		From(s.env.SubdomainURL("sampling-50pct")).
+		PPL(`{"allow":{"and":["email":{"is":"foo@example.com"}]}}`)
+
+	s.env.AddUpstream(s.upstream)
+	s.env.Start()
+	snippets.WaitStartupComplete(s.env)
+}
+
+func (s *SamplingTestSuite) TearDownTest() {
+	s.env.Stop()
+}
+
+func (s *SamplingTestSuite) handleRequest(w http.ResponseWriter, req *http.Request) {
+	span := oteltrace.SpanFromContext(req.Context())
+	flags := span.SpanContext().TraceFlags()
+	if flags.IsSampled() {
+		s.sampled.Add(1)
+	} else {
+		s.notSampled.Add(1)
+	}
+	w.Write([]byte("OK"))
+}
+
+func (s *SamplingTestSuite) doRequest(ctx context.Context) {
+	resp, err := s.upstream.Get(s.route, upstreams.AuthenticateAs("foo@example.com"), upstreams.Path("/"), upstreams.Context(ctx))
+	s.Require().NoError(err)
+	body, err := io.ReadAll(resp.Body)
+	s.Assert().NoError(err)
+	s.Assert().NoError(resp.Body.Close())
+	s.Assert().Equal(resp.StatusCode, 200)
+	s.Assert().Equal("OK", string(body))
+}
+
+func (s *SamplingTestSuite) TestNoExternalTraceparent() {
+	for {
+		s.doRequest(context.Background())
+		if s.sampled.Load() == 20 {
+			break
+		}
+	}
+
+	s.Assert().NoError(trace.ForceFlush(s.env.Context()))
+	trace.WaitForSpans(s.env.Context(), 10*time.Second)
+
+	s.Assert().Equal(int32(20), s.sampled.Load()) // 10 sampled
+	// Ideally we get ~50% sample rate, but CI will always be unlucky.
+	s.Assert().Greater(s.notSampled.Load(), int32(0))
+
+	if s.getResults != nil {
+		results := s.getResults()
+		traces := results.GetTraces()
+		s.Assert().Len(traces.ByParticipant["Upstream"], 20)
+	}
+}
+
+func (s *SamplingTestSuite) TestExternalTraceparentAlwaysSample() {
+	tracer := trace.NewTracerProvider(s.env.Context(), "Always Sample",
+		sdktrace.WithSampler(sdktrace.AlwaysSample())).Tracer(trace.PomeriumCoreTracer)
+	for range 100 {
+		ctx, span := tracer.Start(context.Background(), "should sample")
+		s.doRequest(ctx)
+		span.End()
+	}
+
+	s.Assert().NoError(trace.ForceFlush(s.env.Context()))
+	trace.WaitForSpans(s.env.Context(), 10*time.Second)
+
+	// if the request already has a traceparent header, they will always be sampled
+	// regardless of the random sample rate we configured
+	s.Assert().Equal(int32(100), s.sampled.Load())
+	s.Assert().Equal(int32(0), s.notSampled.Load())
+
+	if s.getResults != nil {
+		results := s.getResults()
+		traces := results.GetTraces()
+		s.Assert().Len(traces.ByParticipant["Envoy"], 100)
+	}
+}
+
+func (s *SamplingTestSuite) TestExternalTraceparentNeverSample() {
+	tracer := trace.NewTracerProvider(s.env.Context(), "Never Sample", sdktrace.WithSampler(sdktrace.NeverSample())).Tracer(trace.PomeriumCoreTracer)
+	for range 100 {
+		ctx, span := tracer.Start(context.Background(), "should not sample")
+		s.doRequest(ctx)
+		span.End()
+	}
+
+	s.Assert().NoError(trace.ForceFlush(s.env.Context()))
+	trace.WaitForSpans(s.env.Context(), 10*time.Second)
+
+	s.Assert().Equal(int32(0), s.sampled.Load())
+	s.Assert().Equal(int32(100), s.notSampled.Load())
+
+	if s.getResults != nil {
+		results := s.getResults()
+		traces := results.GetTraces()
+		if (len(traces.ByParticipant)) != 0 {
+			// whether or not these show up is timing dependent, but not important
+			possibleTraces := map[string]struct{}{
+				"Test Environment: Start":                                 {},
+				"IDP: Server: POST /oidc/token":                           {},
+				"IDP: Server: GET /oidc/userinfo":                         {},
+				"Authenticate: OAuth2 Client: GET /.well-known/jwks.json": {},
+			}
+			actual := slices.Collect(maps.Keys(traces.ByName))
+			for _, name := range actual {
+				if _, ok := possibleTraces[name]; !ok {
+					s.Fail("unexpected trace: " + name)
+				}
+			}
+		}
+	}
+}
+
+func TestSampling(t *testing.T) {
+	suite.Run(t, &SamplingTestSuite{})
+}
+
+func TestExternalSpans(t *testing.T) {
+	modifier, newRemoteClient, getResults := otlpTraceReceiverOrFromEnv(t)
+
+	// set up external tracer
+	external := otlptrace.NewUnstarted(newRemoteClient())
+	r, err := resource.Merge(
+		resource.Empty(),
+		resource.NewWithAttributes(
+			semconv.SchemaURL,
+			semconv.ServiceName("External"),
+		),
+	)
+	require.NoError(t, err)
+
+	externalTracerProvider := sdktrace.NewTracerProvider(sdktrace.WithBatcher(external), sdktrace.WithResource(r))
+
+	env := testenv.New(t, testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags|trace.EnvoyFlushEverySpan), testenv.WithTraceClient(newRemoteClient()))
+	env.Add(modifier)
+
+	up := upstreams.HTTP(nil, upstreams.WithNoClientTracing())
+	up.Handle("/foo", func(w http.ResponseWriter, _ *http.Request) {
+		w.Write([]byte("OK"))
+	})
+	env.Add(scenarios.NewIDP([]*scenarios.User{
+		{
+			Email:     "foo@example.com",
+			FirstName: "Firstname",
+			LastName:  "Lastname",
+		},
+	}))
+
+	route := up.Route().
+		From(env.SubdomainURL("foo")).
+		PPL(`{"allow":{"and":["email":{"is":"foo@example.com"}]}}`)
+
+	env.AddUpstream(up)
+	env.Start()
+	require.NoError(t, external.Start(env.Context()))
+	snippets.WaitStartupComplete(env)
+
+	ctx, span := externalTracerProvider.Tracer("external").Start(context.Background(), "External Root", oteltrace.WithNewRoot())
+	t.Logf("external span id: %s", span.SpanContext().SpanID().String())
+	resp, err := up.Get(route, upstreams.AuthenticateAs("foo@example.com"), upstreams.Path("/foo"), upstreams.Context(ctx))
+	span.End()
+	require.NoError(t, err)
+	body, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.NoError(t, resp.Body.Close())
+	assert.Equal(t, resp.StatusCode, 200)
+	assert.Equal(t, "OK", string(body))
+
+	assert.NoError(t, externalTracerProvider.ForceFlush(context.Background()))
+	assert.NoError(t, externalTracerProvider.Shutdown(context.Background()))
+	assert.NoError(t, external.Shutdown(ctx))
+	env.Stop()
+
+	if getResults != nil {
+		results := getResults()
+		results.MatchTraces(t, MatchOptions{CheckDetachedSpans: true},
+			Match{Name: "External: External Root", TraceCount: 1, Services: []string{
+				"Authorize",
+				"Authenticate",
+				"Control Plane",
+				"Data Broker",
+				"Proxy",
+				"IDP",
+				"Envoy",
+				"External",
+				"HTTP Upstream",
+			}},
+		)
+	}
+}

internal/testenv/snippets/shutdown.go

@@ -0,0 +1,40 @@
+package snippets
+
+import (
+	"context"
+	"errors"
+	"net/http"
+
+	"github.com/pomerium/pomerium/internal/log"
+	"github.com/pomerium/pomerium/internal/testenv"
+)
+
+func RunWithDelayedShutdown(ctx context.Context, serve func() error, stop func()) func() error {
+	env := testenv.EnvFromContext(ctx)
+
+	stopping := make(chan struct{})
+	serveExited := make(chan error, 1)
+	env.OnStateChanged(testenv.Stopping, func() {
+		close(stopping)
+	})
+	cancel := env.OnStateChanged(testenv.Stopped, func() {
+		stop()
+		if err := <-serveExited; err != nil && !errors.Is(err, http.ErrServerClosed) {
+			log.Ctx(ctx).Err(err).Msg("error stopping server")
+		}
+	})
+	go func() {
+		serveExited <- serve()
+		close(serveExited)
+	}()
+
+	return func() error {
+		select {
+		case <-stopping:
+			return nil
+		case err := <-serveExited:
+			cancel()
+			return err
+		}
+	}
+}

internal/testenv/snippets/wait.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"time"
 
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/testenv"
 	"github.com/pomerium/pomerium/pkg/grpcutil"
 	"google.golang.org/grpc"
@@ -12,6 +13,11 @@ import (
 )
 
 func WaitStartupComplete(env testenv.Environment, timeout ...time.Duration) time.Duration {
+	if env.GetState() == testenv.NotRunning {
+		panic("test bug: WaitStartupComplete called before starting the test environment")
+	}
+	_, span := trace.Continue(env.Context(), "snippets.WaitStartupComplete")
+	defer span.End()
 	start := time.Now()
 	recorder := env.NewLogRecorder()
 	if len(timeout) == 0 {
@@ -31,5 +37,6 @@ func WaitStartupComplete(env testenv.Environment, timeout ...time.Duration) time
 	)
 	env.Require().NoError(err)
 	env.Require().True(cc.WaitForStateChange(ctx, connectivity.Ready))
+	cc.Close()
 	return time.Since(start)
 }

internal/testenv/temp.go

@@ -0,0 +1,9 @@
+//go:build !darwin
+
+package testenv
+
+import "testing"
+
+func tempDir(t testing.TB) string {
+	return t.TempDir()
+}

internal/testenv/temp_darwin.go

@@ -0,0 +1,21 @@
+//go:build darwin
+
+package testenv
+
+import (
+	"os"
+	"testing"
+)
+
+// macos temp directory names are too long
+// https://github.com/golang/go/issues/62614
+func tempDir(t testing.TB) string {
+	dir, err := os.MkdirTemp("", "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		os.RemoveAll(dir)
+	})
+	return dir
+}

internal/testenv/types.go

@@ -84,7 +84,7 @@ type Aggregate struct {
 
 func (d *Aggregate) Add(mod Modifier) {
 	if d.env != nil {
-		if d.env.(*environment).getState() == NotRunning {
+		if d.env.(*environment).GetState() == NotRunning {
 			// If the test environment is running, adding to an aggregate is a no-op.
 			// If the test environment has not been started yet, the aggregate is
 			// being used like in the following example, which is incorrect:
@@ -161,6 +161,20 @@ func ModifierFunc(fn func(ctx context.Context, cfg *config.Config)) Modifier {
 	return &modifierFunc{fn: fn}
 }
 
+func NoopModifier() Modifier {
+	return noopModifier{}
+}
+
+type noopModifier struct{}
+
+// Attach implements Modifier.
+func (n noopModifier) Attach(context.Context) {}
+
+// Modify implements Modifier.
+func (n noopModifier) Modify(*config.Config) {}
+
+var _ Modifier = (noopModifier{})
+
 // Task represents a background task that can be added to an [Environment] to
 // have it run automatically on startup.
 //
@@ -169,10 +183,17 @@ type Task interface {
 	Run(ctx context.Context) error
 }
 
-type TaskFunc func(ctx context.Context) error
+func TaskFunc(fn func(ctx context.Context) error) Task {
+	return &taskFunc{fn: fn}
+}
 
-func (f TaskFunc) Run(ctx context.Context) error {
-	return f(ctx)
+// this is a struct wrapper type instead of a typed func so it can be compared
+type taskFunc struct {
+	fn func(ctx context.Context) error
+}
+
+func (f *taskFunc) Run(ctx context.Context) error {
+	return f.fn(ctx)
 }
 
 // Upstream represents an upstream server. It is both a [Task] and a [Modifier]

internal/testenv/upstreams/grpc.go

@@ -6,28 +6,36 @@ import (
 	"net"
 	"strings"
 
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/testenv"
+	"github.com/pomerium/pomerium/internal/testenv/snippets"
 	"github.com/pomerium/pomerium/internal/testenv/values"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
+	oteltrace "go.opentelemetry.io/otel/trace"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
 )
 
-type Options struct {
+type GRPCUpstreamOptions struct {
+	CommonUpstreamOptions
 	serverOpts []grpc.ServerOption
 }
 
-type Option func(*Options)
-
-func (o *Options) apply(opts ...Option) {
-	for _, op := range opts {
-		op(o)
-	}
+type GRPCUpstreamOption interface {
+	applyGRPC(*GRPCUpstreamOptions)
 }
 
-func ServerOpts(opt ...grpc.ServerOption) Option {
-	return func(o *Options) {
+type GRPCUpstreamOptionFunc func(*GRPCUpstreamOptions)
+
+func (f GRPCUpstreamOptionFunc) applyGRPC(o *GRPCUpstreamOptions) {
+	f(o)
+}
+
+func ServerOpts(opt ...grpc.ServerOption) GRPCUpstreamOption {
+	return GRPCUpstreamOptionFunc(func(o *GRPCUpstreamOptions) {
 		o.serverOpts = append(o.serverOpts, opt...)
-	}
+	})
 }
 
 // GRPCUpstream represents a GRPC server which can be used as the target for
@@ -42,13 +50,18 @@ type GRPCUpstream interface {
 	testenv.Upstream
 	grpc.ServiceRegistrar
 	Dial(r testenv.Route, dialOpts ...grpc.DialOption) *grpc.ClientConn
+
+	// Dials the server directly instead of going through a Pomerium route.
+	DirectConnect(dialOpts ...grpc.DialOption) *grpc.ClientConn
 }
 
 type grpcUpstream struct {
-	Options
+	GRPCUpstreamOptions
 	testenv.Aggregate
-	serverPort values.MutableValue[int]
-	creds      credentials.TransportCredentials
+	serverPort           values.MutableValue[int]
+	creds                credentials.TransportCredentials
+	serverTracerProvider values.MutableValue[oteltrace.TracerProvider]
+	clientTracerProvider values.MutableValue[oteltrace.TracerProvider]
 
 	services []service
 }
@@ -59,13 +72,21 @@ var (
 )
 
 // GRPC creates a new GRPC upstream server.
-func GRPC(creds credentials.TransportCredentials, opts ...Option) GRPCUpstream {
-	options := Options{}
-	options.apply(opts...)
+func GRPC(creds credentials.TransportCredentials, opts ...GRPCUpstreamOption) GRPCUpstream {
+	options := GRPCUpstreamOptions{
+		CommonUpstreamOptions: CommonUpstreamOptions{
+			displayName: "GRPC Upstream",
+		},
+	}
+	for _, op := range opts {
+		op.applyGRPC(&options)
+	}
 	up := &grpcUpstream{
-		Options:    options,
-		creds:      creds,
-		serverPort: values.Deferred[int](),
+		GRPCUpstreamOptions:  options,
+		creds:                creds,
+		serverPort:           values.Deferred[int](),
+		serverTracerProvider: values.Deferred[oteltrace.TracerProvider](),
+		clientTracerProvider: values.Deferred[oteltrace.TracerProvider](),
 	}
 	up.RecordCaller()
 	return up
@@ -109,30 +130,65 @@ func (g *grpcUpstream) Run(ctx context.Context) error {
 		return err
 	}
 	g.serverPort.Resolve(listener.Addr().(*net.TCPAddr).Port)
-	server := grpc.NewServer(append(g.serverOpts, grpc.Creds(g.creds))...)
+	if g.serverTracerProviderOverride != nil {
+		g.serverTracerProvider.Resolve(g.serverTracerProviderOverride)
+	} else {
+		g.serverTracerProvider.Resolve(trace.NewTracerProvider(ctx, g.displayName))
+	}
+	if g.clientTracerProviderOverride != nil {
+		g.clientTracerProvider.Resolve(g.clientTracerProviderOverride)
+	} else {
+		g.clientTracerProvider.Resolve(trace.NewTracerProvider(ctx, "GRPC Client"))
+	}
+	server := grpc.NewServer(append(g.serverOpts,
+		grpc.Creds(g.creds),
+		grpc.StatsHandler(otelgrpc.NewServerHandler(
+			otelgrpc.WithTracerProvider(g.serverTracerProvider.Value()),
+		)),
+	)...)
 	for _, s := range g.services {
 		server.RegisterService(s.desc, s.impl)
 	}
+	if g.delayShutdown {
+		return snippets.RunWithDelayedShutdown(ctx,
+			func() error {
+				return server.Serve(listener)
+			},
+			server.GracefulStop,
+		)()
+	}
 	errC := make(chan error, 1)
 	go func() {
 		errC <- server.Serve(listener)
 	}()
 	select {
 	case <-ctx.Done():
-		server.Stop()
+		server.GracefulStop()
 		return context.Cause(ctx)
 	case err := <-errC:
 		return err
 	}
 }
 
-func (g *grpcUpstream) Dial(r testenv.Route, dialOpts ...grpc.DialOption) *grpc.ClientConn {
-	dialOpts = append(dialOpts,
-		grpc.WithContextDialer(testenv.GRPCContextDialer),
+func (g *grpcUpstream) withDefaultDialOpts(extraDialOpts []grpc.DialOption) []grpc.DialOption {
+	return append(extraDialOpts,
 		grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(g.Env().ServerCAs(), "")),
 		grpc.WithDefaultCallOptions(grpc.WaitForReady(true)),
+		grpc.WithStatsHandler(otelgrpc.NewClientHandler(otelgrpc.WithTracerProvider(g.clientTracerProvider.Value()))),
 	)
-	cc, err := grpc.NewClient(strings.TrimPrefix(r.URL().Value(), "https://"), dialOpts...)
+}
+
+func (g *grpcUpstream) Dial(r testenv.Route, dialOpts ...grpc.DialOption) *grpc.ClientConn {
+	cc, err := grpc.NewClient(strings.TrimPrefix(r.URL().Value(), "https://"), g.withDefaultDialOpts(dialOpts)...)
+	if err != nil {
+		panic(err)
+	}
+	return cc
+}
+
+func (g *grpcUpstream) DirectConnect(dialOpts ...grpc.DialOption) *grpc.ClientConn {
+	cc, err := grpc.NewClient(fmt.Sprintf("127.0.0.1:%d", g.Port().Value()),
+		append(g.withDefaultDialOpts(dialOpts), grpc.WithTransportCredentials(insecure.NewCredentials()))...)
 	if err != nil {
 		panic(err)
 	}

internal/testenv/upstreams/http.go

@@ -21,13 +21,21 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/pomerium/pomerium/integration/forms"
 	"github.com/pomerium/pomerium/internal/retry"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/testenv"
+	"github.com/pomerium/pomerium/internal/testenv/snippets"
 	"github.com/pomerium/pomerium/internal/testenv/values"
-	"github.com/pomerium/pomerium/pkg/telemetry/requestid"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
+	oteltrace "go.opentelemetry.io/otel/trace"
 	"google.golang.org/protobuf/proto"
 )
 
 type RequestOptions struct {
+	requestCtx     context.Context
 	path           string
 	query          url.Values
 	headers        map[string]string
@@ -80,6 +88,12 @@ func Client(c *http.Client) RequestOption {
 	}
 }
 
+func Context(ctx context.Context) RequestOption {
+	return func(o *RequestOptions) {
+		o.requestCtx = ctx
+	}
+}
+
 func WithClientTrace(ct *httptrace.ClientTrace) RequestOption {
 	return func(o *RequestOptions) {
 		o.trace = ct
@@ -111,6 +125,14 @@ func ClientCert[T interface {
 	}
 }
 
+type HTTPUpstreamOptions struct {
+	CommonUpstreamOptions
+}
+
+type HTTPUpstreamOption interface {
+	applyHTTP(*HTTPUpstreamOptions)
+}
+
 // HTTPUpstream represents a HTTP server which can be used as the target for
 // one or more Pomerium routes in a test environment.
 //
@@ -128,13 +150,17 @@ type HTTPUpstream interface {
 }
 
 type httpUpstream struct {
+	HTTPUpstreamOptions
 	testenv.Aggregate
 	serverPort values.MutableValue[int]
 	tlsConfig  values.Value[*tls.Config]
 
 	clientCache sync.Map // map[testenv.Route]*http.Client
 
-	router *mux.Router
+	router               *mux.Router
+	serverTracerProvider values.MutableValue[oteltrace.TracerProvider]
+	clientTracerProvider values.MutableValue[oteltrace.TracerProvider]
+	clientTracer         values.Value[oteltrace.Tracer]
 }
 
 var (
@@ -143,12 +169,26 @@ var (
 )
 
 // HTTP creates a new HTTP upstream server.
-func HTTP(tlsConfig values.Value[*tls.Config]) HTTPUpstream {
-	up := &httpUpstream{
-		serverPort: values.Deferred[int](),
-		router:     mux.NewRouter(),
-		tlsConfig:  tlsConfig,
+func HTTP(tlsConfig values.Value[*tls.Config], opts ...HTTPUpstreamOption) HTTPUpstream {
+	options := HTTPUpstreamOptions{
+		CommonUpstreamOptions: CommonUpstreamOptions{
+			displayName: "HTTP Upstream",
+		},
 	}
+	for _, op := range opts {
+		op.applyHTTP(&options)
+	}
+	up := &httpUpstream{
+		HTTPUpstreamOptions:  options,
+		serverPort:           values.Deferred[int](),
+		router:               mux.NewRouter(),
+		tlsConfig:            tlsConfig,
+		serverTracerProvider: values.Deferred[oteltrace.TracerProvider](),
+		clientTracerProvider: values.Deferred[oteltrace.TracerProvider](),
+	}
+	up.clientTracer = values.Bind(up.clientTracerProvider, func(tp oteltrace.TracerProvider) oteltrace.Tracer {
+		return tp.Tracer(trace.PomeriumCoreTracer)
+	})
 	up.RecordCaller()
 	return up
 }
@@ -185,12 +225,34 @@ func (h *httpUpstream) Run(ctx context.Context) error {
 	if h.tlsConfig != nil {
 		tlsConfig = h.tlsConfig.Value()
 	}
+	if h.serverTracerProviderOverride != nil {
+		h.serverTracerProvider.Resolve(h.serverTracerProviderOverride)
+	} else {
+		h.serverTracerProvider.Resolve(trace.NewTracerProvider(ctx, h.displayName))
+	}
+	if h.clientTracerProviderOverride != nil {
+		h.clientTracerProvider.Resolve(h.clientTracerProviderOverride)
+	} else {
+		h.clientTracerProvider.Resolve(trace.NewTracerProvider(ctx, "HTTP Client"))
+	}
+	h.router.Use(trace.NewHTTPMiddleware(otelhttp.WithTracerProvider(h.serverTracerProvider.Value())))
+
 	server := &http.Server{
 		Handler:   h.router,
 		TLSConfig: tlsConfig,
-		BaseContext: func(net.Listener) context.Context {
-			return ctx
-		},
+		// BaseContext: func(net.Listener) context.Context {
+		// 	return ctx
+		// },
+	}
+	if h.delayShutdown {
+		return snippets.RunWithDelayedShutdown(ctx,
+			func() error {
+				return server.Serve(listener)
+			},
+			func() {
+				_ = server.Shutdown(context.Background())
+			},
+		)()
 	}
 	errC := make(chan error, 1)
 	go func() {
@@ -198,7 +260,7 @@ func (h *httpUpstream) Run(ctx context.Context) error {
 	}()
 	select {
 	case <-ctx.Done():
-		server.Close()
+		_ = server.Shutdown(context.Background())
 		return context.Cause(ctx)
 	case err := <-errC:
 		return err
@@ -217,7 +279,9 @@ func (h *httpUpstream) Post(r testenv.Route, opts ...RequestOption) (*http.Respo
 
 // Do implements HTTPUpstream.
 func (h *httpUpstream) Do(method string, r testenv.Route, opts ...RequestOption) (*http.Response, error) {
-	options := RequestOptions{}
+	options := RequestOptions{
+		requestCtx: h.Env().Context(),
+	}
 	options.apply(opts...)
 	u, err := url.Parse(r.URL().Value())
 	if err != nil {
@@ -229,37 +293,15 @@ func (h *httpUpstream) Do(method string, r testenv.Route, opts ...RequestOption)
 			RawQuery: options.query.Encode(),
 		})
 	}
-	ctx := h.Env().Context()
+	ctx, span := h.clientTracer.Value().Start(options.requestCtx, "httpUpstream.Do", oteltrace.WithAttributes(
+		attribute.String("method", method),
+		attribute.String("url", u.String()),
+	))
 	if options.trace != nil {
 		ctx = httptrace.WithClientTrace(ctx, options.trace)
 	}
-	req, err := http.NewRequestWithContext(ctx, method, u.String(), nil)
-	if err != nil {
-		return nil, err
-	}
-	switch body := options.body.(type) {
-	case string:
-		req.Body = io.NopCloser(strings.NewReader(body))
-	case []byte:
-		req.Body = io.NopCloser(bytes.NewReader(body))
-	case io.Reader:
-		req.Body = io.NopCloser(body)
-	case proto.Message:
-		buf, err := proto.Marshal(body)
-		if err != nil {
-			return nil, err
-		}
-		req.Body = io.NopCloser(bytes.NewReader(buf))
-		req.Header.Set("Content-Type", "application/octet-stream")
-	default:
-		buf, err := json.Marshal(body)
-		if err != nil {
-			panic(fmt.Sprintf("unsupported body type: %T", body))
-		}
-		req.Body = io.NopCloser(bytes.NewReader(buf))
-		req.Header.Set("Content-Type", "application/json")
-	case nil:
-	}
+	options.requestCtx = ctx
+	defer span.End()
 
 	newClient := func() *http.Client {
 		transport := http.DefaultTransport.(*http.Transport).Clone()
@@ -269,7 +311,12 @@ func (h *httpUpstream) Do(method string, r testenv.Route, opts ...RequestOption)
 		}
 		transport.DialTLSContext = nil
 		c := http.Client{
-			Transport: requestid.NewRoundTripper(transport),
+			Transport: otelhttp.NewTransport(transport,
+				otelhttp.WithTracerProvider(h.clientTracerProvider.Value()),
+				otelhttp.WithSpanNameFormatter(func(_ string, r *http.Request) string {
+					return fmt.Sprintf("Client: %s %s", r.Method, r.URL.Path)
+				}),
+			),
 		}
 		c.Jar, _ = cookiejar.New(&cookiejar.Options{})
 		return &c
@@ -281,14 +328,45 @@ func (h *httpUpstream) Do(method string, r testenv.Route, opts ...RequestOption)
 		var cachedClient any
 		var ok bool
 		if cachedClient, ok = h.clientCache.Load(r); !ok {
+			span.AddEvent("creating new http client")
 			cachedClient, _ = h.clientCache.LoadOrStore(r, newClient())
+		} else {
+			span.AddEvent("using cached http client")
 		}
 		client = cachedClient.(*http.Client)
 	}
 
 	var resp *http.Response
+	resendCount := 0
 	if err := retry.Retry(ctx, "http", func(ctx context.Context) error {
-		var err error
+		req, err := http.NewRequestWithContext(ctx, method, u.String(), nil)
+		if err != nil {
+			return retry.NewTerminalError(err)
+		}
+		switch body := options.body.(type) {
+		case string:
+			req.Body = io.NopCloser(strings.NewReader(body))
+		case []byte:
+			req.Body = io.NopCloser(bytes.NewReader(body))
+		case io.Reader:
+			req.Body = io.NopCloser(body)
+		case proto.Message:
+			buf, err := proto.Marshal(body)
+			if err != nil {
+				return retry.NewTerminalError(err)
+			}
+			req.Body = io.NopCloser(bytes.NewReader(buf))
+			req.Header.Set("Content-Type", "application/octet-stream")
+		default:
+			buf, err := json.Marshal(body)
+			if err != nil {
+				panic(fmt.Sprintf("unsupported body type: %T", body))
+			}
+			req.Body = io.NopCloser(bytes.NewReader(buf))
+			req.Header.Set("Content-Type", "application/json")
+		case nil:
+		}
+
 		if options.authenticateAs != "" {
 			resp, err = authenticateFlow(ctx, client, req, options.authenticateAs) //nolint:bodyclose
 		} else {
@@ -296,46 +374,70 @@ func (h *httpUpstream) Do(method string, r testenv.Route, opts ...RequestOption)
 		}
 		// retry on connection refused
 		if err != nil {
+			span.RecordError(err)
 			var opErr *net.OpError
 			if errors.As(err, &opErr) && opErr.Op == "dial" && opErr.Err.Error() == "connect: connection refused" {
+				span.AddEvent("Retrying on dial error")
 				return err
 			}
 			return retry.NewTerminalError(err)
 		}
-		if resp.StatusCode == http.StatusInternalServerError {
+		if resp.StatusCode/100 == 5 {
+			resendCount++
+			_, _ = io.ReadAll(resp.Body)
+			_ = resp.Body.Close()
+			span.SetAttributes(semconv.HTTPRequestResendCount(resendCount))
+			span.AddEvent("Retrying on 5xx error", oteltrace.WithAttributes(
+				attribute.String("status", resp.Status),
+			))
 			return errors.New(http.StatusText(resp.StatusCode))
 		}
+		span.SetStatus(codes.Ok, "request completed successfully")
 		return nil
-	}, retry.WithMaxInterval(100*time.Millisecond)); err != nil {
+	},
+		retry.WithInitialInterval(1*time.Millisecond),
+		retry.WithMaxInterval(100*time.Millisecond),
+	); err != nil {
 		return nil, err
 	}
 	return resp, nil
 }
 
 func authenticateFlow(ctx context.Context, client *http.Client, req *http.Request, email string) (*http.Response, error) {
+	span := oteltrace.SpanFromContext(ctx)
 	var res *http.Response
 	originalHostname := req.URL.Hostname()
 	res, err := client.Do(req)
 	if err != nil {
+		span.RecordError(err)
 		return nil, err
 	}
-
 	location := res.Request.URL
 	if location.Hostname() == originalHostname {
 		// already authenticated
-		return res, err
+		span.SetStatus(codes.Ok, "already authenticated")
+		return res, nil
 	}
-	defer res.Body.Close()
 	fs := forms.Parse(res.Body)
+	_, _ = io.ReadAll(res.Body)
+	_ = res.Body.Close()
 	if len(fs) > 0 {
 		f := fs[0]
 		f.Inputs["email"] = email
 		f.Inputs["token_expiration"] = strconv.Itoa(int((time.Hour * 24).Seconds()))
+		span.AddEvent("submitting form", oteltrace.WithAttributes(attribute.String("location", location.String())))
 		formReq, err := f.NewRequestWithContext(ctx, location)
 		if err != nil {
+			span.RecordError(err)
 			return nil, err
 		}
-		return client.Do(formReq)
+		resp, err := client.Do(formReq)
+		if err != nil {
+			span.RecordError(err)
+			return nil, err
+		}
+		span.SetStatus(codes.Ok, "form submitted successfully")
+		return resp, nil
 	}
 	return nil, fmt.Errorf("test bug: expected IDP login form")
 }

internal/testenv/upstreams/options.go

@@ -0,0 +1,64 @@
+package upstreams
+
+import (
+	oteltrace "go.opentelemetry.io/otel/trace"
+	"go.opentelemetry.io/otel/trace/noop"
+)
+
+type CommonUpstreamOptions struct {
+	displayName                  string
+	clientTracerProviderOverride oteltrace.TracerProvider
+	serverTracerProviderOverride oteltrace.TracerProvider
+	delayShutdown                bool
+}
+
+type CommonUpstreamOption interface {
+	GRPCUpstreamOption
+	HTTPUpstreamOption
+}
+
+type commonUpstreamOption func(o *CommonUpstreamOptions)
+
+// applyGRPC implements CommonUpstreamOption.
+func (c commonUpstreamOption) applyGRPC(o *GRPCUpstreamOptions) { c(&o.CommonUpstreamOptions) }
+
+// applyHTTP implements CommonUpstreamOption.
+func (c commonUpstreamOption) applyHTTP(o *HTTPUpstreamOptions) { c(&o.CommonUpstreamOptions) }
+
+func WithDisplayName(displayName string) CommonUpstreamOption {
+	return commonUpstreamOption(func(o *CommonUpstreamOptions) {
+		o.displayName = displayName
+	})
+}
+
+func WithNoClientTracing() CommonUpstreamOption {
+	return commonUpstreamOption(func(o *CommonUpstreamOptions) {
+		o.clientTracerProviderOverride = noop.NewTracerProvider()
+	})
+}
+
+func WithNoServerTracing() CommonUpstreamOption {
+	return commonUpstreamOption(func(o *CommonUpstreamOptions) {
+		o.serverTracerProviderOverride = noop.NewTracerProvider()
+	})
+}
+
+func WithClientTracerProvider(tp oteltrace.TracerProvider) CommonUpstreamOption {
+	return commonUpstreamOption(func(o *CommonUpstreamOptions) {
+		o.clientTracerProviderOverride = tp
+	})
+}
+
+func WithServerTracerProvider(tp oteltrace.TracerProvider) CommonUpstreamOption {
+	return commonUpstreamOption(func(o *CommonUpstreamOptions) {
+		o.serverTracerProviderOverride = tp
+	})
+}
+
+// WithDelayedShutdown keeps the server alive until the test environment has
+// fully shut down, instead of stopping it during the shutdown sequence.
+func WithDelayedShutdown() CommonUpstreamOption {
+	return commonUpstreamOption(func(o *CommonUpstreamOptions) {
+		o.delayShutdown = true
+	})
+}

internal/testenv/values/value.go

@@ -69,7 +69,7 @@ func (p *value[T]) Resolve(value T) {
 	p.ResolveFunc(func() T { return value })
 }
 
-// Bind creates a new [MutableValue] whose ultimate value depends on the result
+// Bind creates a new [Value] whose ultimate value depends on the result
 // of another [Value] that may not yet be available. When Value() is called on
 // the result, it will cascade and trigger the full chain of initialization
 // functions necessary to produce the final value.
@@ -118,3 +118,49 @@ func (s List[T]) Value() []T {
 	}
 	return values
 }
+
+// Chain is like [Bind], returning a Value[V] whose ultimate value depends on
+// the result of a Value[U] which is obtained from the concrete T given in dt.
+// It is intended to be used when T has a method returning a Value[U], and
+// you want to access U, but don't need T.
+//
+// Example usage:
+//
+//	type Foo interface { Number() Value[int] }
+//	var foo Foo
+//	values.Chain(foo, Foo.Number, func(number int) string { ... })
+func Chain[T any, U any, V any](
+	dt Value[T],
+	callback1 func(value T) Value[U],
+	callback2 func(value U) V,
+) Value[V] {
+	dv := Deferred[V]()
+	dv.ResolveFunc(func() V {
+		du := callback1(dt.Value())
+		return callback2(du.Value())
+	})
+	return dv
+}
+
+// Chain2 is like [Chain], but with two levels of indirection.
+//
+// Example usage:
+//
+//	type Foo interface { Bar() Value[Bar] }
+//	type Bar interface { Number() Value[int] }
+//	var foo Foo
+//	values.Chain(foo, Foo.Bar, bar.Number, func(number int) string { ... })
+func Chain2[T any, U any, V any, W any](
+	dt Value[T],
+	callback1 func(value T) Value[U],
+	callback2 func(value U) Value[V],
+	callback3 func(value V) W,
+) Value[W] {
+	dw := Deferred[W]()
+	dw.ResolveFunc(func() W {
+		du := callback1(dt.Value())
+		dv := callback2(du.Value())
+		return callback3(dv.Value())
+	})
+	return dw
+}