mirror of
https://github.com/pomerium/pomerium.git
synced 2025-06-22 20:48:10 +02:00
Refactor trace config to match supported otel options
This commit is contained in:
Joe Kralicky
parent
c307ca806a
commit
c510edf364
21 changed files with 991 additions and 701 deletions
|
|
@ -6,7 +6,6 @@ import (
|
|||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"os"
|
||||
"slices"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
|
@ -30,25 +29,6 @@ import (
|
|||
. "github.com/pomerium/pomerium/internal/testutil/tracetest" //nolint:revive
|
||||
)
|
||||
|
||||
func otlpTraceReceiverOrFromEnv(t *testing.T) (modifier testenv.Modifier, newRemoteClient func() otlptrace.Client, getResults func() *TraceResults) {
|
||||
t.Setenv("OTEL_TRACES_EXPORTER", "otlp")
|
||||
tracesEndpoint := os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")
|
||||
if tracesEndpoint == "" {
|
||||
tracesEndpoint = os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
|
||||
if tracesEndpoint == "" {
|
||||
srv := scenarios.NewOTLPTraceReceiver()
|
||||
return srv,
|
||||
func() otlptrace.Client {
|
||||
return srv.NewGRPCClient()
|
||||
},
|
||||
func() *TraceResults {
|
||||
return NewTraceResults(srv.FlushResourceSpans())
|
||||
}
|
||||
}
|
||||
}
|
||||
return testenv.NoopModifier(), trace.NewRemoteClientFromEnv, nil
|
||||
}
|
||||
|
||||
var allServices = []string{
|
||||
"Test Environment",
|
||||
"Authorize",
|
||||
|
|
@ -63,9 +43,9 @@ var allServices = []string{
|
|||
}
|
||||
|
||||
func TestOTLPTracing(t *testing.T) {
|
||||
modifier, newRemoteClient, getResults := otlpTraceReceiverOrFromEnv(t)
|
||||
env := testenv.New(t, testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags), testenv.WithTraceClient(newRemoteClient()))
|
||||
env.Add(modifier)
|
||||
srv := scenarios.NewOTLPTraceReceiver()
|
||||
env := testenv.New(t, testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags), testenv.WithTraceClient(srv.NewGRPCClient()))
|
||||
env.Add(srv)
|
||||
|
||||
up := upstreams.HTTP(nil, upstreams.WithDisplayName("Upstream"))
|
||||
up.Handle("/foo", func(w http.ResponseWriter, _ *http.Request) {
|
||||
|
|
@ -99,40 +79,38 @@ func TestOTLPTracing(t *testing.T) {
|
|||
|
||||
env.Stop()
|
||||
|
||||
if getResults != nil {
|
||||
results := getResults()
|
||||
var (
|
||||
testEnvironmentLocalTest = fmt.Sprintf("Test Environment: %s", t.Name())
|
||||
testEnvironmentAuthenticate = "Test Environment: Authenticate"
|
||||
authenticateOAuth2Client = "Authenticate: OAuth2 Client: GET /.well-known/jwks.json"
|
||||
idpServerGetUserinfo = "IDP: Server: GET /oidc/userinfo"
|
||||
idpServerPostToken = "IDP: Server: POST /oidc/token"
|
||||
controlPlaneEnvoyAccessLogs = "Control Plane: envoy.service.accesslog.v3.AccessLogService/StreamAccessLogs"
|
||||
controlPlaneEnvoyDiscovery = "Control Plane: envoy.service.discovery.v3.AggregatedDiscoveryService/DeltaAggregatedResources"
|
||||
controlPlaneExport = "Control Plane: opentelemetry.proto.collector.trace.v1.TraceService/Export"
|
||||
)
|
||||
results := NewTraceResults(srv.FlushResourceSpans())
|
||||
var (
|
||||
testEnvironmentLocalTest = fmt.Sprintf("Test Environment: %s", t.Name())
|
||||
testEnvironmentAuthenticate = "Test Environment: Authenticate"
|
||||
authenticateOAuth2Client = "Authenticate: OAuth2 Client: GET /.well-known/jwks.json"
|
||||
idpServerGetUserinfo = "IDP: Server: GET /oidc/userinfo"
|
||||
idpServerPostToken = "IDP: Server: POST /oidc/token"
|
||||
controlPlaneEnvoyAccessLogs = "Control Plane: envoy.service.accesslog.v3.AccessLogService/StreamAccessLogs"
|
||||
controlPlaneEnvoyDiscovery = "Control Plane: envoy.service.discovery.v3.AggregatedDiscoveryService/DeltaAggregatedResources"
|
||||
controlPlaneExport = "Control Plane: opentelemetry.proto.collector.trace.v1.TraceService/Export"
|
||||
)
|
||||
|
||||
results.MatchTraces(t,
|
||||
MatchOptions{
|
||||
Exact: true,
|
||||
CheckDetachedSpans: true,
|
||||
},
|
||||
Match{Name: testEnvironmentLocalTest, TraceCount: 1, Services: []string{"Authorize", "Test Environment", "Control Plane", "Data Broker"}},
|
||||
Match{Name: testEnvironmentAuthenticate, TraceCount: 1, Services: allServices},
|
||||
Match{Name: authenticateOAuth2Client, TraceCount: Greater(0)},
|
||||
Match{Name: idpServerGetUserinfo, TraceCount: EqualToMatch(authenticateOAuth2Client)},
|
||||
Match{Name: idpServerPostToken, TraceCount: EqualToMatch(authenticateOAuth2Client)},
|
||||
Match{Name: controlPlaneEnvoyDiscovery, TraceCount: 1},
|
||||
Match{Name: controlPlaneExport, TraceCount: Greater(0)},
|
||||
Match{Name: controlPlaneEnvoyAccessLogs, TraceCount: Any{}},
|
||||
)
|
||||
}
|
||||
results.MatchTraces(t,
|
||||
MatchOptions{
|
||||
Exact: true,
|
||||
CheckDetachedSpans: true,
|
||||
},
|
||||
Match{Name: testEnvironmentLocalTest, TraceCount: 1, Services: []string{"Authorize", "Test Environment", "Control Plane", "Data Broker"}},
|
||||
Match{Name: testEnvironmentAuthenticate, TraceCount: 1, Services: allServices},
|
||||
Match{Name: authenticateOAuth2Client, TraceCount: Greater(0)},
|
||||
Match{Name: idpServerGetUserinfo, TraceCount: EqualToMatch(authenticateOAuth2Client)},
|
||||
Match{Name: idpServerPostToken, TraceCount: EqualToMatch(authenticateOAuth2Client)},
|
||||
Match{Name: controlPlaneEnvoyDiscovery, TraceCount: 1},
|
||||
Match{Name: controlPlaneExport, TraceCount: Greater(0)},
|
||||
Match{Name: controlPlaneEnvoyAccessLogs, TraceCount: Any{}},
|
||||
)
|
||||
}
|
||||
|
||||
func TestOTLPTracing_TraceCorrelation(t *testing.T) {
|
||||
modifier, newRemoteClient, getResults := otlpTraceReceiverOrFromEnv(t)
|
||||
env := testenv.New(t, testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags), testenv.WithTraceClient(newRemoteClient()))
|
||||
env.Add(modifier)
|
||||
srv := scenarios.NewOTLPTraceReceiver()
|
||||
env := testenv.New(t, testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags), testenv.WithTraceClient(srv.NewGRPCClient()))
|
||||
env.Add(srv)
|
||||
|
||||
up := upstreams.HTTP(nil, upstreams.WithDisplayName("Upstream"), upstreams.WithNoClientTracing())
|
||||
up.Handle("/foo", func(w http.ResponseWriter, _ *http.Request) {
|
||||
|
|
@ -163,40 +141,37 @@ func TestOTLPTracing_TraceCorrelation(t *testing.T) {
|
|||
assert.Equal(t, "OK", string(body))
|
||||
|
||||
env.Stop()
|
||||
if getResults != nil {
|
||||
results := getResults()
|
||||
traces := results.GetTraces()
|
||||
// one unauthenticated (ends in /.pomerium/callback redirect), one authenticated
|
||||
assert.Len(t, traces.ByName[fmt.Sprintf("Envoy: ingress: GET foo.localhost.pomerium.io:%d/foo", env.Ports().ProxyHTTP.Value())].WithoutErrors(), 2)
|
||||
}
|
||||
results := NewTraceResults(srv.FlushResourceSpans())
|
||||
traces := results.GetTraces()
|
||||
// one unauthenticated (ends in /.pomerium/callback redirect), one authenticated
|
||||
assert.Len(t, traces.ByName[fmt.Sprintf("Envoy: ingress: GET foo.localhost.pomerium.io:%d/foo", env.Ports().ProxyHTTP.Value())].WithoutErrors(), 2)
|
||||
}
|
||||
|
||||
type SamplingTestSuite struct {
|
||||
suite.Suite
|
||||
env testenv.Environment
|
||||
getResults func() *TraceResults
|
||||
route testenv.Route
|
||||
upstream upstreams.HTTPUpstream
|
||||
env testenv.Environment
|
||||
receiver *scenarios.OTLPTraceReceiver
|
||||
route testenv.Route
|
||||
upstream upstreams.HTTPUpstream
|
||||
|
||||
sampled atomic.Int32
|
||||
notSampled atomic.Int32
|
||||
}
|
||||
|
||||
func (s *SamplingTestSuite) SetupTest() {
|
||||
modifier, newRemoteClient, getResults := otlpTraceReceiverOrFromEnv(s.T())
|
||||
s.getResults = getResults
|
||||
s.receiver = scenarios.NewOTLPTraceReceiver()
|
||||
s.env = testenv.New(s.T(),
|
||||
testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags|trace.EnvoyFlushEverySpan),
|
||||
testenv.WithTraceClient(newRemoteClient()),
|
||||
testenv.WithTraceClient(s.receiver.NewGRPCClient()),
|
||||
)
|
||||
s.env.Add(modifier)
|
||||
s.env.Add(s.receiver)
|
||||
|
||||
s.sampled.Store(0)
|
||||
s.notSampled.Store(0)
|
||||
|
||||
s.env.Add(testenv.ModifierFunc(func(_ context.Context, cfg *config.Config) {
|
||||
half := 0.5
|
||||
cfg.Options.TracingSampleRate = &half
|
||||
cfg.Options.Tracing.OtelTracesSamplerArg = &half
|
||||
}))
|
||||
s.env.Add(scenarios.NewIDP([]*scenarios.User{
|
||||
{
|
||||
|
|
@ -258,11 +233,9 @@ func (s *SamplingTestSuite) TestNoExternalTraceparent() {
|
|||
// Ideally we get ~50% sample rate, but CI will always be unlucky.
|
||||
s.Assert().Greater(s.notSampled.Load(), int32(0))
|
||||
|
||||
if s.getResults != nil {
|
||||
results := s.getResults()
|
||||
traces := results.GetTraces()
|
||||
s.Assert().Len(traces.ByParticipant["Upstream"], 20)
|
||||
}
|
||||
results := NewTraceResults(s.receiver.FlushResourceSpans())
|
||||
traces := results.GetTraces()
|
||||
s.Assert().Len(traces.ByParticipant["Upstream"], 20)
|
||||
}
|
||||
|
||||
func (s *SamplingTestSuite) TestExternalTraceparentAlwaysSample() {
|
||||
|
|
@ -282,11 +255,9 @@ func (s *SamplingTestSuite) TestExternalTraceparentAlwaysSample() {
|
|||
s.Assert().Equal(int32(100), s.sampled.Load())
|
||||
s.Assert().Equal(int32(0), s.notSampled.Load())
|
||||
|
||||
if s.getResults != nil {
|
||||
results := s.getResults()
|
||||
traces := results.GetTraces()
|
||||
s.Assert().Len(traces.ByParticipant["Envoy"], 100)
|
||||
}
|
||||
results := NewTraceResults(s.receiver.FlushResourceSpans())
|
||||
traces := results.GetTraces()
|
||||
s.Assert().Len(traces.ByParticipant["Envoy"], 100)
|
||||
}
|
||||
|
||||
func (s *SamplingTestSuite) TestExternalTraceparentNeverSample() {
|
||||
|
|
@ -303,22 +274,20 @@ func (s *SamplingTestSuite) TestExternalTraceparentNeverSample() {
|
|||
s.Assert().Equal(int32(0), s.sampled.Load())
|
||||
s.Assert().Equal(int32(100), s.notSampled.Load())
|
||||
|
||||
if s.getResults != nil {
|
||||
results := s.getResults()
|
||||
traces := results.GetTraces()
|
||||
if (len(traces.ByParticipant)) != 0 {
|
||||
// whether or not these show up is timing dependent, but not important
|
||||
possibleTraces := map[string]struct{}{
|
||||
"Test Environment: Start": {},
|
||||
"IDP: Server: POST /oidc/token": {},
|
||||
"IDP: Server: GET /oidc/userinfo": {},
|
||||
"Authenticate: OAuth2 Client: GET /.well-known/jwks.json": {},
|
||||
}
|
||||
actual := slices.Collect(maps.Keys(traces.ByName))
|
||||
for _, name := range actual {
|
||||
if _, ok := possibleTraces[name]; !ok {
|
||||
s.Fail("unexpected trace: " + name)
|
||||
}
|
||||
results := NewTraceResults(s.receiver.FlushResourceSpans())
|
||||
traces := results.GetTraces()
|
||||
if (len(traces.ByParticipant)) != 0 {
|
||||
// whether or not these show up is timing dependent, but not important
|
||||
possibleTraces := map[string]struct{}{
|
||||
"Test Environment: Start": {},
|
||||
"IDP: Server: POST /oidc/token": {},
|
||||
"IDP: Server: GET /oidc/userinfo": {},
|
||||
"Authenticate: OAuth2 Client: GET /.well-known/jwks.json": {},
|
||||
}
|
||||
actual := slices.Collect(maps.Keys(traces.ByName))
|
||||
for _, name := range actual {
|
||||
if _, ok := possibleTraces[name]; !ok {
|
||||
s.Fail("unexpected trace: " + name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -329,10 +298,10 @@ func TestSampling(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestExternalSpans(t *testing.T) {
|
||||
modifier, newRemoteClient, getResults := otlpTraceReceiverOrFromEnv(t)
|
||||
srv := scenarios.NewOTLPTraceReceiver()
|
||||
|
||||
// set up external tracer
|
||||
external := otlptrace.NewUnstarted(newRemoteClient())
|
||||
external := otlptrace.NewUnstarted(srv.NewGRPCClient())
|
||||
r, err := resource.Merge(
|
||||
resource.Empty(),
|
||||
resource.NewWithAttributes(
|
||||
|
|
@ -344,8 +313,8 @@ func TestExternalSpans(t *testing.T) {
|
|||
|
||||
externalTracerProvider := sdktrace.NewTracerProvider(sdktrace.WithBatcher(external), sdktrace.WithResource(r))
|
||||
|
||||
env := testenv.New(t, testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags|trace.EnvoyFlushEverySpan), testenv.WithTraceClient(newRemoteClient()))
|
||||
env.Add(modifier)
|
||||
env := testenv.New(t, testenv.WithTraceDebugFlags(testenv.StandardTraceDebugFlags|trace.EnvoyFlushEverySpan), testenv.WithTraceClient(srv.NewGRPCClient()))
|
||||
env.Add(srv)
|
||||
|
||||
up := upstreams.HTTP(nil, upstreams.WithNoClientTracing())
|
||||
up.Handle("/foo", func(w http.ResponseWriter, _ *http.Request) {
|
||||
|
|
@ -384,20 +353,18 @@ func TestExternalSpans(t *testing.T) {
|
|||
assert.NoError(t, external.Shutdown(ctx))
|
||||
env.Stop()
|
||||
|
||||
if getResults != nil {
|
||||
results := getResults()
|
||||
results.MatchTraces(t, MatchOptions{CheckDetachedSpans: true},
|
||||
Match{Name: "External: External Root", TraceCount: 1, Services: []string{
|
||||
"Authorize",
|
||||
"Authenticate",
|
||||
"Control Plane",
|
||||
"Data Broker",
|
||||
"Proxy",
|
||||
"IDP",
|
||||
"Envoy",
|
||||
"External",
|
||||
"HTTP Upstream",
|
||||
}},
|
||||
)
|
||||
}
|
||||
results := NewTraceResults(srv.FlushResourceSpans())
|
||||
results.MatchTraces(t, MatchOptions{CheckDetachedSpans: true},
|
||||
Match{Name: "External: External Root", TraceCount: 1, Services: []string{
|
||||
"Authorize",
|
||||
"Authenticate",
|
||||
"Control Plane",
|
||||
"Data Broker",
|
||||
"Proxy",
|
||||
"IDP",
|
||||
"Envoy",
|
||||
"External",
|
||||
"HTTP Upstream",
|
||||
}},
|
||||
)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue