diff --git a/authenticate/handlers.go b/authenticate/handlers.go index 5f3418f6c..b079861ab 100644 --- a/authenticate/handlers.go +++ b/authenticate/handlers.go @@ -3,6 +3,7 @@ package authenticate import ( "context" "encoding/base64" + "encoding/hex" "errors" "fmt" "net/http" @@ -21,6 +22,7 @@ import ( "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/middleware" "github.com/pomerium/pomerium/internal/sessions" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/urlutil" "github.com/pomerium/pomerium/pkg/cryptutil" "github.com/pomerium/pomerium/pkg/identity" @@ -282,9 +284,20 @@ func (a *Authenticate) reauthenticateOrFail(w http.ResponseWriter, r *http.Reque state.sessionStore.ClearSession(w, r) redirectURL := state.redirectURL.ResolveReference(r.URL) + redirectURLValues := redirectURL.Query() + var traceID string + if tp := trace.PomeriumURLQueryCarrier(redirectURLValues).Get("traceparent"); len(tp) == 55 { + if traceIDBytes, err := hex.DecodeString(tp[3:35]); err == nil { + traceFlags, _ := hex.DecodeString(tp[53:55]) + if len(traceFlags) != 1 { + traceFlags = []byte{0} + } + traceID = base64.RawURLEncoding.EncodeToString(append(traceIDBytes, traceFlags[0])) + } + } nonce := csrf.Token(r) now := time.Now().Unix() - b := []byte(fmt.Sprintf("%s|%d|", nonce, now)) + b := []byte(fmt.Sprintf("%s|%d|%s|", nonce, now, traceID)) enc := cryptutil.Encrypt(state.cookieCipher, []byte(redirectURL.String()), b) b = append(b, enc...) encodedState := base64.URLEncoding.EncodeToString(b) @@ -306,10 +319,6 @@ func (a *Authenticate) OAuthCallback(w http.ResponseWriter, r *http.Request) err if err != nil { return fmt.Errorf("authenticate.OAuthCallback: %w", err) } - q := redirect.Query() - if traceparent := q.Get(urlutil.QueryTraceparent); traceparent != "" { - w.Header().Set("X-Pomerium-Traceparent", traceparent) - } httputil.Redirect(w, r, redirect.String(), http.StatusFound) return nil } @@ -350,21 +359,20 @@ func (a *Authenticate) getOAuthCallback(w http.ResponseWriter, r *http.Request) } // split state into concat'd components - // (nonce|timestamp|redirect_url|encrypted_data(redirect_url)+mac(nonce,ts)) - statePayload := strings.SplitN(string(bytes), "|", 3) - if len(statePayload) != 3 { + // (nonce|timestamp|trace_id+flags|encrypted_data(redirect_url)+mac(nonce,ts)) + statePayload := strings.SplitN(string(bytes), "|", 4) + if len(statePayload) != 4 { return nil, httputil.NewError(http.StatusBadRequest, fmt.Errorf("state malformed, size: %d", len(statePayload))) } // Use our AEAD construct to enforce secrecy and authenticity: // mac: to validate the nonce again, and above timestamp // decrypt: to prevent leaking 'redirect_uri' to IdP or logs - b := []byte(fmt.Sprint(statePayload[0], "|", statePayload[1], "|")) - redirectString, err := cryptutil.Decrypt(state.cookieCipher, []byte(statePayload[2]), b) + b := []byte(fmt.Sprint(statePayload[0], "|", statePayload[1], "|", statePayload[2], "|")) + redirectString, err := cryptutil.Decrypt(state.cookieCipher, []byte(statePayload[3]), b) if err != nil { return nil, httputil.NewError(http.StatusBadRequest, err) } - redirectURL, err := urlutil.ParseAndValidateURL(string(redirectString)) if err != nil { return nil, httputil.NewError(http.StatusBadRequest, err) diff --git a/authenticate/handlers_test.go b/authenticate/handlers_test.go index 4dacd372a..b9ae56066 100644 --- a/authenticate/handlers_test.go +++ b/authenticate/handlers_test.go @@ -368,9 +368,8 @@ func TestAuthenticate_OAuthCallback(t *testing.T) { params.Add("error", tt.paramErr) params.Add("code", tt.code) nonce := cryptutil.NewBase64Key() // mock csrf - // (nonce|timestamp|redirect_url|encrypt(redirect_url),mac(nonce,ts)) - b := []byte(fmt.Sprintf("%s|%d|%s", nonce, tt.ts, tt.extraMac)) - + // (nonce|timestamp|trace_id+flags|encrypt(redirect_url),mac(nonce,ts)) + b := []byte(fmt.Sprintf("%s|%d||%s", nonce, tt.ts, tt.extraMac)) enc := cryptutil.Encrypt(a.state.Load().cookieCipher, []byte(tt.redirectURI), b) b = append(b, enc...) encodedState := base64.URLEncoding.EncodeToString(b) diff --git a/authorize/check_response.go b/authorize/check_response.go index 816a6bb3b..027402af4 100644 --- a/authorize/check_response.go +++ b/authorize/check_response.go @@ -232,11 +232,8 @@ func (a *Authorize) requireLoginResponse( headers := http.Header{} if id := in.GetAttributes().GetRequest().GetHttp().GetHeaders()["traceparent"]; id != "" { - headers["X-Pomerium-Traceparent"] = []string{id} - headers["X-Pomerium-Tracestate"] = []string{"pomerium.traceparent=" + id} // TODO: this might not be necessary anymore signInURLQuery = url.Values{} signInURLQuery.Add("pomerium_traceparent", id) - signInURLQuery.Add("pomerium_tracestate", "pomerium.traceparent="+id) } redirectTo, err := state.authenticateFlow.AuthenticateSignInURL( ctx, signInURLQuery, &checkRequestURL, idp.GetId()) diff --git a/config/envoyconfig/extensions/pomerium_otel.pb.go b/config/envoyconfig/extensions/pomerium_otel.pb.go new file mode 100644 index 000000000..db0801bed --- /dev/null +++ b/config/envoyconfig/extensions/pomerium_otel.pb.go @@ -0,0 +1,211 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.35.2 +// protoc (unknown) +// source: github.com/pomerium/pomerium/config/envoyconfig/extensions/pomerium_otel.proto + +package extensions + +import ( + _ "github.com/cncf/xds/go/udpa/annotations" + v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type OpenTelemetryConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + GrpcService *v3.GrpcService `protobuf:"bytes,1,opt,name=grpc_service,json=grpcService,proto3" json:"grpc_service,omitempty"` + HttpService *v3.HttpService `protobuf:"bytes,3,opt,name=http_service,json=httpService,proto3" json:"http_service,omitempty"` + ServiceName string `protobuf:"bytes,2,opt,name=service_name,json=serviceName,proto3" json:"service_name,omitempty"` + ResourceDetectors []*v3.TypedExtensionConfig `protobuf:"bytes,4,rep,name=resource_detectors,json=resourceDetectors,proto3" json:"resource_detectors,omitempty"` + Sampler *v3.TypedExtensionConfig `protobuf:"bytes,5,opt,name=sampler,proto3" json:"sampler,omitempty"` +} + +func (x *OpenTelemetryConfig) Reset() { + *x = OpenTelemetryConfig{} + mi := &file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *OpenTelemetryConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*OpenTelemetryConfig) ProtoMessage() {} + +func (x *OpenTelemetryConfig) ProtoReflect() protoreflect.Message { + mi := &file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use OpenTelemetryConfig.ProtoReflect.Descriptor instead. +func (*OpenTelemetryConfig) Descriptor() ([]byte, []int) { + return file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_rawDescGZIP(), []int{0} +} + +func (x *OpenTelemetryConfig) GetGrpcService() *v3.GrpcService { + if x != nil { + return x.GrpcService + } + return nil +} + +func (x *OpenTelemetryConfig) GetHttpService() *v3.HttpService { + if x != nil { + return x.HttpService + } + return nil +} + +func (x *OpenTelemetryConfig) GetServiceName() string { + if x != nil { + return x.ServiceName + } + return "" +} + +func (x *OpenTelemetryConfig) GetResourceDetectors() []*v3.TypedExtensionConfig { + if x != nil { + return x.ResourceDetectors + } + return nil +} + +func (x *OpenTelemetryConfig) GetSampler() *v3.TypedExtensionConfig { + if x != nil { + return x.Sampler + } + return nil +} + +var File_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto protoreflect.FileDescriptor + +var file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_rawDesc = []byte{ + 0x0a, 0x4e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x6f, 0x6d, + 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2f, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2f, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x2f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x2f, 0x70, 0x6f, 0x6d, + 0x65, 0x72, 0x69, 0x75, 0x6d, 0x5f, 0x6f, 0x74, 0x65, 0x6c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x13, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x6e, + 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x24, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2f, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x76, 0x33, 0x2f, 0x65, 0x78, 0x74, 0x65, + 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x27, 0x65, 0x6e, 0x76, + 0x6f, 0x79, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x76, + 0x33, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x27, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2f, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x76, 0x33, 0x2f, 0x68, 0x74, 0x74, 0x70, 0x5f, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x75, + 0x64, 0x70, 0x61, 0x2f, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2f, + 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1d, 0x75, + 0x64, 0x70, 0x61, 0x2f, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2f, + 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x93, 0x03, 0x0a, + 0x13, 0x4f, 0x70, 0x65, 0x6e, 0x54, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x12, 0x5b, 0x0a, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x65, 0x6e, 0x76, + 0x6f, 0x79, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, + 0x33, 0x2e, 0x47, 0x72, 0x70, 0x63, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x42, 0x15, 0xf2, + 0x98, 0xfe, 0x8f, 0x05, 0x0f, 0x12, 0x0d, 0x6f, 0x74, 0x6c, 0x70, 0x5f, 0x65, 0x78, 0x70, 0x6f, + 0x72, 0x74, 0x65, 0x72, 0x52, 0x0b, 0x67, 0x72, 0x70, 0x63, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x12, 0x5b, 0x0a, 0x0c, 0x68, 0x74, 0x74, 0x70, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2e, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x33, 0x2e, 0x48, + 0x74, 0x74, 0x70, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x42, 0x15, 0xf2, 0x98, 0xfe, 0x8f, + 0x05, 0x0f, 0x12, 0x0d, 0x6f, 0x74, 0x6c, 0x70, 0x5f, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, + 0x72, 0x52, 0x0b, 0x68, 0x74, 0x74, 0x70, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x21, + 0x0a, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, 0x6d, + 0x65, 0x12, 0x59, 0x0a, 0x12, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x64, 0x65, + 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, + 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x63, 0x6f, 0x72, + 0x65, 0x2e, 0x76, 0x33, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x64, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, + 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x11, 0x72, 0x65, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x44, 0x0a, 0x07, + 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, + 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x63, 0x6f, 0x72, + 0x65, 0x2e, 0x76, 0x33, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x64, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, + 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x07, 0x73, 0x61, 0x6d, 0x70, 0x6c, + 0x65, 0x72, 0x42, 0x44, 0xba, 0x80, 0xc8, 0xd1, 0x06, 0x02, 0x10, 0x02, 0x5a, 0x3a, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75, + 0x6d, 0x2f, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x2f, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x65, 0x78, + 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_rawDescOnce sync.Once + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_rawDescData = file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_rawDesc +) + +func file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_rawDescGZIP() []byte { + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_rawDescOnce.Do(func() { + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_rawDescData = protoimpl.X.CompressGZIP(file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_rawDescData) + }) + return file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_rawDescData +} + +var file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_goTypes = []any{ + (*OpenTelemetryConfig)(nil), // 0: pomerium.extensions.OpenTelemetryConfig + (*v3.GrpcService)(nil), // 1: envoy.config.core.v3.GrpcService + (*v3.HttpService)(nil), // 2: envoy.config.core.v3.HttpService + (*v3.TypedExtensionConfig)(nil), // 3: envoy.config.core.v3.TypedExtensionConfig +} +var file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_depIdxs = []int32{ + 1, // 0: pomerium.extensions.OpenTelemetryConfig.grpc_service:type_name -> envoy.config.core.v3.GrpcService + 2, // 1: pomerium.extensions.OpenTelemetryConfig.http_service:type_name -> envoy.config.core.v3.HttpService + 3, // 2: pomerium.extensions.OpenTelemetryConfig.resource_detectors:type_name -> envoy.config.core.v3.TypedExtensionConfig + 3, // 3: pomerium.extensions.OpenTelemetryConfig.sampler:type_name -> envoy.config.core.v3.TypedExtensionConfig + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name +} + +func init() { + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_init() +} +func file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_init() { + if File_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_goTypes, + DependencyIndexes: file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_depIdxs, + MessageInfos: file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_msgTypes, + }.Build() + File_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto = out.File + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_rawDesc = nil + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_goTypes = nil + file_github_com_pomerium_pomerium_config_envoyconfig_extensions_pomerium_otel_proto_depIdxs = nil +} diff --git a/config/envoyconfig/extensions/pomerium_otel.proto b/config/envoyconfig/extensions/pomerium_otel.proto new file mode 100644 index 000000000..022a7189f --- /dev/null +++ b/config/envoyconfig/extensions/pomerium_otel.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package pomerium.extensions; + +import "envoy/config/core/v3/extension.proto"; +import "envoy/config/core/v3/grpc_service.proto"; +import "envoy/config/core/v3/http_service.proto"; +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/status.proto"; + +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +message OpenTelemetryConfig { + envoy.config.core.v3.GrpcService grpc_service = 1 [(udpa.annotations.field_migrate).oneof_promotion = "otlp_exporter"]; + envoy.config.core.v3.HttpService http_service = 3 [(udpa.annotations.field_migrate).oneof_promotion = "otlp_exporter"]; + string service_name = 2; + repeated envoy.config.core.v3.TypedExtensionConfig resource_detectors = 4; + envoy.config.core.v3.TypedExtensionConfig sampler = 5; +} diff --git a/config/envoyconfig/filters.go b/config/envoyconfig/filters.go index fda1984cb..2292f73e4 100644 --- a/config/envoyconfig/filters.go +++ b/config/envoyconfig/filters.go @@ -34,16 +34,6 @@ func ExtAuthzFilter(grpcClientTimeout *durationpb.Duration) *envoy_extensions_fi ClusterName: "pomerium-authorize", }, }, - InitialMetadata: []*envoy_config_core_v3.HeaderValue{ - { - Key: "x-pomerium-traceparent", - Value: `%DYNAMIC_METADATA(pomerium.internal:traceparent)%`, - }, - { - Key: "x-pomerium-tracestate", - Value: `%DYNAMIC_METADATA(pomerium.internal:tracestate)%`, - }, - }, }, }, MetadataContextNamespaces: []string{"com.pomerium.client-certificate-info"}, diff --git a/config/envoyconfig/http_connection_manager.go b/config/envoyconfig/http_connection_manager.go index 2b7791a9b..5d956b373 100644 --- a/config/envoyconfig/http_connection_manager.go +++ b/config/envoyconfig/http_connection_manager.go @@ -39,22 +39,6 @@ func (b *Builder) buildVirtualHost( return nil, err } vh.Routes = append(vh.Routes, rs...) - vh.RequestHeadersToAdd = []*envoy_config_core_v3.HeaderValueOption{ - { - Header: &envoy_config_core_v3.HeaderValue{ - Key: "x-pomerium-traceparent", - Value: `%DYNAMIC_METADATA(pomerium.internal:traceparent)%`, - }, - AppendAction: envoy_config_core_v3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, - }, - { - Header: &envoy_config_core_v3.HeaderValue{ - Key: "x-pomerium-tracestate", - Value: `%DYNAMIC_METADATA(pomerium.internal:tracestate)%`, - }, - AppendAction: envoy_config_core_v3.HeaderValueOption_APPEND_IF_EXISTS_OR_ADD, - }, - } return vh, nil } diff --git a/config/envoyconfig/listeners_test.go b/config/envoyconfig/listeners_test.go index 95a1eba54..d38c83f4e 100644 --- a/config/envoyconfig/listeners_test.go +++ b/config/envoyconfig/listeners_test.go @@ -103,7 +103,7 @@ func TestBuildListeners(t *testing.T) { }] } } - }`, httpConfig.Get("httpFilters.7").String(), + }`, httpConfig.Get("httpFilters.6").String(), "should add alt-svc header") case "quic-ingress": hasQUIC = true @@ -149,6 +149,7 @@ func Test_buildMainHTTPConnectionManagerFilter(t *testing.T) { options.SkipXffAppend = true options.XffNumTrustedHops = 1 options.AuthenticateURLString = "https://authenticate.example.com" + options.TracingProvider = "otlp" filter, err := b.buildMainHTTPConnectionManagerFilter(context.Background(), &config.Config{Options: options}, false, false) require.NoError(t, err) diff --git a/config/envoyconfig/route_configurations_test.go b/config/envoyconfig/route_configurations_test.go index 4cc103068..06593b567 100644 --- a/config/envoyconfig/route_configurations_test.go +++ b/config/envoyconfig/route_configurations_test.go @@ -42,21 +42,6 @@ func TestBuilder_buildMainRouteConfiguration(t *testing.T) { { "name": "catch-all", "domains": ["*"], - "requestHeadersToAdd": [ - { - "appendAction": "OVERWRITE_IF_EXISTS_OR_ADD", - "header": { - "key": "x-pomerium-traceparent", - "value": "%DYNAMIC_METADATA(pomerium.internal:traceparent)%" - } - }, - { - "header": { - "key": "x-pomerium-tracestate", - "value": "%DYNAMIC_METADATA(pomerium.internal:tracestate)%" - } - } - ], "routes": [ `+protojson.Format(b.buildControlPlanePathRoute(cfg.Options, "/ping"))+`, `+protojson.Format(b.buildControlPlanePathRoute(cfg.Options, "/healthz"))+`, diff --git a/config/envoyconfig/testdata/main_http_connection_manager_filter.json b/config/envoyconfig/testdata/main_http_connection_manager_filter.json index f286d777a..af30417dc 100644 --- a/config/envoyconfig/testdata/main_http_connection_manager_filter.json +++ b/config/envoyconfig/testdata/main_http_connection_manager_filter.json @@ -32,60 +32,6 @@ } ], "httpFilters": [ - { - "name": "envoy.filters.http.header_to_metadata", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.header_to_metadata.v3.Config", - "requestRules": [ - { - "header": "x-pomerium-traceparent", - "onHeaderPresent": { - "metadataNamespace": "pomerium.internal", - "key": "traceparent" - } - }, - { - "header": "x-pomerium-tracestate", - "onHeaderPresent": { - "metadataNamespace": "pomerium.internal", - "key": "tracestate" - } - }, - { - "header": "x-pomerium-external-parent-span", - "onHeaderPresent": { - "key": "external-parent-span", - "metadataNamespace": "pomerium.internal" - }, - "remove": true - }, - { - "header": "x-pomerium-sampling-decision", - "onHeaderPresent": { - "metadataNamespace": "pomerium.internal", - "key": "sampling-decision" - }, - "remove": true - } - ], - "responseRules": [ - { - "header": "x-pomerium-traceparent", - "onHeaderPresent": { - "metadataNamespace": "pomerium.internal", - "key": "traceparent" - } - }, - { - "header": "x-pomerium-tracestate", - "onHeaderPresent": { - "metadataNamespace": "pomerium.internal", - "key": "tracestate" - } - } - ] - } - }, { "name": "envoy.filters.http.lua", "typedConfig": { @@ -112,17 +58,7 @@ "envoyGrpc": { "clusterName": "pomerium-authorize" }, - "timeout": "10s", - "initialMetadata": [ - { - "key": "x-pomerium-traceparent", - "value": "%DYNAMIC_METADATA(pomerium.internal:traceparent)%" - }, - { - "key": "x-pomerium-tracestate", - "value": "%DYNAMIC_METADATA(pomerium.internal:tracestate)%" - } - ] + "timeout": "10s" }, "transportApiVersion": "V3", "statusOnError": { @@ -265,60 +201,10 @@ }, "verbose": true, "maxPathTagLength": 1024, - "customTags": [ - { - "tag": "pomerium.traceparent", - "metadata": { - "kind": { - "request": {} - }, - "metadataKey": { - "key": "pomerium.internal", - "path": [ - { - "key": "traceparent" - } - ] - } - } - }, - { - "tag": "pomerium.tracestate", - "metadata": { - "kind": { - "request": {} - }, - "metadataKey": { - "key": "pomerium.internal", - "path": [ - { - "key": "tracestate" - } - ] - } - } - }, - { - "metadata": { - "kind": { - "request": {} - }, - "metadataKey": { - "key": "pomerium.internal", - "path": [ - { - "key": "external-parent-span" - } - ] - } - }, - "tag": "pomerium.external-parent-span" - } - ], "provider": { - "name": "envoy.tracers.opentelemetry", + "name": "envoy.tracers.pomerium_otel", "typedConfig": { - "@type": "type.googleapis.com/envoy.config.trace.v3.OpenTelemetryConfig", + "@type": "type.googleapis.com/pomerium.extensions.OpenTelemetryConfig", "grpcService": { "envoyGrpc": { "clusterName": "pomerium-control-plane-grpc" diff --git a/config/envoyconfig/tracing.go b/config/envoyconfig/tracing.go index fe9b73a98..ecf41971a 100644 --- a/config/envoyconfig/tracing.go +++ b/config/envoyconfig/tracing.go @@ -1,17 +1,13 @@ package envoyconfig import ( - "fmt" "os" "strconv" envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" tracev3 "github.com/envoyproxy/go-control-plane/envoy/config/trace/v3" - envoy_extensions_filters_http_header_to_metadata "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/header_to_metadata/v3" envoy_extensions_filters_network_http_connection_manager "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" envoy_extensions_tracers_otel "github.com/envoyproxy/go-control-plane/envoy/extensions/tracers/opentelemetry/resource_detectors/v3" - metadatav3 "github.com/envoyproxy/go-control-plane/envoy/type/metadata/v3" - envoy_tracing_v3 "github.com/envoyproxy/go-control-plane/envoy/type/tracing/v3" envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3" "github.com/pomerium/pomerium/config" "github.com/pomerium/pomerium/config/envoyconfig/extensions" @@ -40,9 +36,6 @@ func applyTracingConfig( if !isTracingEnabled(opts) { return } - mgr.HttpFilters = append([]*envoy_extensions_filters_network_http_connection_manager.HttpFilter{ - tracingMetadataFilter(), - }, mgr.HttpFilters...) mgr.EarlyHeaderMutationExtensions = []*envoy_config_core_v3.TypedExtensionConfig{ { @@ -63,38 +56,14 @@ func applyTracingConfig( maxPathTagLength = max(64, uint32(num)) } } - requestTag := func(key string) *envoy_tracing_v3.CustomTag { - return &envoy_tracing_v3.CustomTag{ - Tag: fmt.Sprintf("pomerium.%s", key), - Type: &envoy_tracing_v3.CustomTag_Metadata_{ - Metadata: &envoy_tracing_v3.CustomTag_Metadata{ - Kind: &metadatav3.MetadataKind{ - Kind: &metadatav3.MetadataKind_Request_{ - Request: &metadatav3.MetadataKind_Request{}, - }, - }, - MetadataKey: &metadatav3.MetadataKey{ - Key: "pomerium.internal", - Path: []*metadatav3.MetadataKey_PathSegment{ - { - Segment: &metadatav3.MetadataKey_PathSegment_Key{ - Key: key, - }, - }, - }, - }, - }, - }, - } - } mgr.Tracing = &envoy_extensions_filters_network_http_connection_manager.HttpConnectionManager_Tracing{ RandomSampling: &envoy_type_v3.Percent{Value: opts.TracingSampleRate * 100}, Verbose: true, SpawnUpstreamSpan: wrapperspb.Bool(true), Provider: &tracev3.Tracing_Http{ - Name: "envoy.tracers.opentelemetry", + Name: "envoy.tracers.pomerium_otel", ConfigType: &tracev3.Tracing_Http_TypedConfig{ - TypedConfig: marshalAny(&tracev3.OpenTelemetryConfig{ + TypedConfig: marshalAny(&extensions.OpenTelemetryConfig{ GrpcService: &envoy_config_core_v3.GrpcService{ TargetSpecifier: &envoy_config_core_v3.GrpcService_EnvoyGrpc_{ EnvoyGrpc: &envoy_config_core_v3.GrpcService_EnvoyGrpc{ @@ -118,62 +87,5 @@ func applyTracingConfig( }, // this allows full URLs to be displayed in traces, they are otherwise truncated MaxPathTagLength: wrapperspb.UInt32(maxPathTagLength), - CustomTags: []*envoy_tracing_v3.CustomTag{ - requestTag("traceparent"), - requestTag("tracestate"), - requestTag("external-parent-span"), - }, - } -} - -func tracingMetadataFilter() *envoy_extensions_filters_network_http_connection_manager.HttpFilter { - traceparentRule := &envoy_extensions_filters_http_header_to_metadata.Config_Rule{ - Header: "x-pomerium-traceparent", - OnHeaderPresent: &envoy_extensions_filters_http_header_to_metadata.Config_KeyValuePair{ - MetadataNamespace: "pomerium.internal", - Key: "traceparent", - }, - Remove: false, - } - tracestateRule := &envoy_extensions_filters_http_header_to_metadata.Config_Rule{ - Header: "x-pomerium-tracestate", - OnHeaderPresent: &envoy_extensions_filters_http_header_to_metadata.Config_KeyValuePair{ - MetadataNamespace: "pomerium.internal", - Key: "tracestate", - }, - Remove: false, - } - externalParentSpanRule := &envoy_extensions_filters_http_header_to_metadata.Config_Rule{ - Header: "x-pomerium-external-parent-span", - OnHeaderPresent: &envoy_extensions_filters_http_header_to_metadata.Config_KeyValuePair{ - MetadataNamespace: "pomerium.internal", - Key: "external-parent-span", - }, - Remove: true, - } - samplingDecisionRule := &envoy_extensions_filters_http_header_to_metadata.Config_Rule{ - Header: "x-pomerium-sampling-decision", - OnHeaderPresent: &envoy_extensions_filters_http_header_to_metadata.Config_KeyValuePair{ - MetadataNamespace: "pomerium.internal", - Key: "sampling-decision", - }, - Remove: true, - } - return &envoy_extensions_filters_network_http_connection_manager.HttpFilter{ - Name: "envoy.filters.http.header_to_metadata", - ConfigType: &envoy_extensions_filters_network_http_connection_manager.HttpFilter_TypedConfig{ - TypedConfig: marshalAny(&envoy_extensions_filters_http_header_to_metadata.Config{ - RequestRules: []*envoy_extensions_filters_http_header_to_metadata.Config_Rule{ - traceparentRule, - tracestateRule, - externalParentSpanRule, - samplingDecisionRule, - }, - ResponseRules: []*envoy_extensions_filters_http_header_to_metadata.Config_Rule{ - traceparentRule, - tracestateRule, - }, - }), - }, } } diff --git a/databroker/cache.go b/databroker/cache.go index 78f8db5e4..f3110ad87 100644 --- a/databroker/cache.go +++ b/databroker/cache.go @@ -96,7 +96,7 @@ func New(ctx context.Context, cfg *config.Config, eventsMgr *events.Manager, opt // No metrics handler because we have one in the control plane. Add one // if we no longer register with that grpc Server localGRPCServer := grpc.NewServer( - grpc.StatsHandler(trace.NewServerStatsHandler(otelgrpc.NewServerHandler(otelgrpc.WithTracerProvider(tracerProvider)))), + grpc.StatsHandler(otelgrpc.NewServerHandler(otelgrpc.WithTracerProvider(tracerProvider))), grpc.ChainStreamInterceptor(log.StreamServerInterceptor(log.Ctx(ctx)), si), grpc.ChainUnaryInterceptor(log.UnaryServerInterceptor(log.Ctx(ctx)), ui), ) diff --git a/go.mod b/go.mod index 67ee5fdfe..34719621b 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 github.com/cespare/xxhash/v2 v2.3.0 github.com/cloudflare/circl v1.5.0 + github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 github.com/coreos/go-oidc/v3 v3.11.0 github.com/docker/docker v27.4.1+incompatible github.com/envoyproxy/go-control-plane/envoy v1.32.2 @@ -133,7 +134,6 @@ require ( github.com/aws/smithy-go v1.22.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/caddyserver/zerossl v0.1.3 // indirect - github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go index ccb37e514..96bf1f3d4 100644 --- a/internal/controlplane/server.go +++ b/internal/controlplane/server.go @@ -114,7 +114,7 @@ func NewServer( ), ) srv.GRPCServer = grpc.NewServer( - grpc.StatsHandler(trace.NewServerStatsHandler(otelgrpc.NewServerHandler(otelgrpc.WithTracerProvider(tracerProvider)))), + grpc.StatsHandler(otelgrpc.NewServerHandler(otelgrpc.WithTracerProvider(tracerProvider))), grpc.ChainUnaryInterceptor( log.UnaryServerInterceptor(log.Ctx(ctx)), requestid.UnaryServerInterceptor(), diff --git a/internal/telemetry/trace/client_test.go b/internal/telemetry/trace/client_test.go index 17a3f15ca..3b4b55c3e 100644 --- a/internal/telemetry/trace/client_test.go +++ b/internal/telemetry/trace/client_test.go @@ -3,7 +3,6 @@ package trace_test import ( "context" "fmt" - "regexp" "runtime" "strings" "sync/atomic" @@ -12,11 +11,11 @@ import ( "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/telemetry/trace" - "github.com/pomerium/pomerium/internal/telemetry/trace/mock_otlptrace" "github.com/pomerium/pomerium/internal/testenv" "github.com/pomerium/pomerium/internal/testenv/scenarios" "github.com/pomerium/pomerium/internal/testenv/snippets" - "github.com/pomerium/pomerium/internal/testutil" + . "github.com/pomerium/pomerium/internal/testutil/tracetest" //nolint:revive + "github.com/pomerium/pomerium/internal/testutil/tracetest/mock_otlptrace" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel" @@ -281,7 +280,7 @@ func TestNewRemoteClientFromEnv(t *testing.T) { "OTEL_TRACES_EXPORTER": "otlp", "OTEL_EXPORTER_OTLP_ENDPOINT": grpcEndpoint.Value(), }, - uploadErr: "net/http: HTTP/1.x transport connection broken", + uploadErr: "net/http: HTTP/1.x transport connection broken: malformed HTTP response", }, { name: "HTTP endpoint, auto protocol", @@ -314,7 +313,7 @@ func TestNewRemoteClientFromEnv(t *testing.T) { }, }, { - name: "no exporter", + name: "exporter unset", env: map[string]string{ "OTEL_TRACES_EXPORTER": "", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": httpEndpoint.Value(), @@ -323,7 +322,7 @@ func TestNewRemoteClientFromEnv(t *testing.T) { expectNoSpans: true, }, { - name: "no exporter", + name: "exporter noop", env: map[string]string{ "OTEL_TRACES_EXPORTER": "noop", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": httpEndpoint.Value(), @@ -332,7 +331,7 @@ func TestNewRemoteClientFromEnv(t *testing.T) { expectNoSpans: true, }, { - name: "no exporter", + name: "exporter none", env: map[string]string{ "OTEL_TRACES_EXPORTER": "none", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": httpEndpoint.Value(), @@ -403,19 +402,6 @@ func TestNewRemoteClientFromEnv(t *testing.T) { otel.SetErrorHandler(handler) t.Cleanup(func() { otel.SetErrorHandler(oldErrHandler) }) - if tc.uploadErr != "" { - recorder := env.NewLogRecorder(testenv.WithSkipCloseDelay()) - defer func() { - recorder.Match([]map[string]any{ - { - "level": "error", - "error": regexp.MustCompile(`^Post "[^"]+": net/http: HTTP/1.x transport connection broken: malformed HTTP response.*$`), - "message": "error uploading traces", - }, - }) - }() - } - remoteClient := trace.NewRemoteClientFromEnv() ctx := trace.Options{ RemoteClient: remoteClient, @@ -431,20 +417,20 @@ func TestNewRemoteClientFromEnv(t *testing.T) { _, span := tp.Tracer(trace.PomeriumCoreTracer).Start(ctx, "test span") span.End() - assert.NoError(t, trace.ForceFlush(ctx)) - assert.NoError(t, trace.ShutdownContext(ctx)) - if tc.uploadErr != "" { + assert.ErrorContains(t, trace.ForceFlush(ctx), tc.uploadErr) + assert.NoError(t, trace.ShutdownContext(ctx)) return } + assert.NoError(t, trace.ShutdownContext(ctx)) - results := testutil.NewTraceResults(receiver.FlushResourceSpans()) + results := NewTraceResults(receiver.FlushResourceSpans()) if tc.expectNoSpans { - results.MatchTraces(t, testutil.MatchOptions{Exact: true}) + results.MatchTraces(t, MatchOptions{Exact: true}) } else { - results.MatchTraces(t, testutil.MatchOptions{ + results.MatchTraces(t, MatchOptions{ Exact: true, - }, testutil.Match{Name: t.Name() + ": test span", TraceCount: 1, Services: []string{t.Name()}}) + }, Match{Name: t.Name() + ": test span", TraceCount: 1, Services: []string{t.Name()}}) } }) } diff --git a/internal/telemetry/trace/debug.go b/internal/telemetry/trace/debug.go index 1809ebe40..e2dcbaff7 100644 --- a/internal/telemetry/trace/debug.go +++ b/internal/telemetry/trace/debug.go @@ -2,14 +2,23 @@ package trace import ( "context" + "encoding/binary" + "encoding/json" + "errors" "fmt" "io" "os" "runtime" + "slices" "strings" + "sync" + "time" "go.opentelemetry.io/otel/attribute" sdktrace "go.opentelemetry.io/otel/sdk/trace" + oteltrace "go.opentelemetry.io/otel/trace" + coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" + "google.golang.org/protobuf/encoding/protojson" ) type DebugFlags uint32 @@ -30,8 +39,10 @@ const ( // over time. TrackAllSpans = (1 << iota) | TrackSpanCallers - // If set, will log all trace ID mappings on close. - LogTraceIDMappings = (1 << iota) + // If set, will log all trace IDs and their span counts on close. + // + // Enables [TrackAllSpans] + LogTraceIDs = (1 << iota) | TrackAllSpans // If set, will log all spans observed by the exporter on close. These spans // may belong to incomplete traces. @@ -39,12 +50,6 @@ const ( // Enables [TrackAllSpans] LogAllSpans = (1 << iota) | TrackAllSpans - // If set, will log the raw json payloads and timestamps of export requests - // on close. - // Use with caution, this will cause significantly increasing memory usage - // over time. - LogAllEvents = (1 << iota) - // If set, will log all exported spans when a warning is issued on close // (requires warning flags to also be set) // @@ -53,7 +58,7 @@ const ( // If set, will log all trace ID mappings when a warning is issued on close. // (requires warning flags to also be set) - LogTraceIDMappingsOnWarn = (1 << iota) + LogTraceIDsOnWarn = (1 << iota) // If set, will print a warning to stderr on close if there are any incomplete // traces (traces with no observed root spans) @@ -78,6 +83,49 @@ func (df DebugFlags) Check(flags DebugFlags) bool { return (df & flags) == flags } +var ( + ErrIncompleteSpans = errors.New("exporter shut down with incomplete spans") + ErrMissingParentSpans = errors.New("exporter shut down with missing parent spans") +) + +// WaitForSpans will block up to the given max duration and wait for all +// in-flight spans from tracers created with the given context to end. This +// function can be called more than once, and is safe to call from multiple +// goroutines in parallel. +// +// This requires the [TrackSpanReferences] debug flag to have been set with +// [Options.NewContext]. Otherwise, this function is a no-op and will return +// immediately. +// +// If this function blocks for more than 10 seconds, it will print a warning +// to stderr containing a list of span IDs it is waiting for, and the IDs of +// their parents (if known). Additionally, if the [TrackAllSpans] debug flag +// is set, details about parent spans will be displayed, including call site +// and trace ID. +func WaitForSpans(ctx context.Context, maxDuration time.Duration) error { + if sys := systemContextFromContext(ctx); sys != nil && sys.observer != nil { + done := make(chan struct{}) + go func() { + defer close(done) + sys.observer.wait(10 * time.Second) + }() + select { + case <-done: + return nil + case <-time.After(maxDuration): + return ErrMissingParentSpans + } + } + return nil +} + +func DebugFlagsFromContext(ctx context.Context) DebugFlags { + if sys := systemContextFromContext(ctx); sys != nil { + return sys.options.DebugFlags + } + return 0 +} + type stackTraceProcessor struct{} // ForceFlush implements trace.SpanProcessor. @@ -117,3 +165,329 @@ func endMsg(msg *strings.Builder) { } fmt.Fprint(w, msg.String()) } + +type DebugEvent struct { + Timestamp time.Time `json:"timestamp"` + Request *coltracepb.ExportTraceServiceRequest `json:"request"` +} + +func (e DebugEvent) MarshalJSON() ([]byte, error) { + type debugEvent struct { + Timestamp time.Time `json:"timestamp"` + Request json.RawMessage `json:"request"` + } + reqData, _ := protojson.Marshal(e.Request) + return json.Marshal(debugEvent{ + Timestamp: e.Timestamp, + Request: reqData, + }) +} + +func (e *DebugEvent) UnmarshalJSON(b []byte) error { + type debugEvent struct { + Timestamp time.Time `json:"timestamp"` + Request json.RawMessage `json:"request"` + } + var ev debugEvent + if err := json.Unmarshal(b, &ev); err != nil { + return err + } + e.Timestamp = ev.Timestamp + var msg coltracepb.ExportTraceServiceRequest + if err := protojson.Unmarshal(ev.Request, &msg); err != nil { + return err + } + e.Request = &msg + return nil +} + +const shardCount = 64 + +type ( + shardedSet [shardCount]map[oteltrace.SpanID]struct{} + shardedLocks [shardCount]sync.Mutex +) + +func (s *shardedSet) Range(f func(key oteltrace.SpanID)) { + for i := range shardCount { + for k := range s[i] { + f(k) + } + } +} + +func (s *shardedLocks) LockAll() { + for i := range shardCount { + s[i].Lock() + } +} + +func (s *shardedLocks) UnlockAll() { + for i := range shardCount { + s[i].Unlock() + } +} + +type spanTracker struct { + inflightSpansMu shardedLocks + inflightSpans shardedSet + allSpans sync.Map + debugFlags DebugFlags + observer *spanObserver + shutdownOnce sync.Once +} + +func newSpanTracker(observer *spanObserver, debugFlags DebugFlags) *spanTracker { + st := &spanTracker{ + observer: observer, + debugFlags: debugFlags, + } + for i := range len(st.inflightSpans) { + st.inflightSpans[i] = make(map[oteltrace.SpanID]struct{}) + } + return st +} + +type spanInfo struct { + Name string + SpanContext oteltrace.SpanContext + Parent oteltrace.SpanContext + caller string + startTime time.Time +} + +// ForceFlush implements trace.SpanProcessor. +func (t *spanTracker) ForceFlush(context.Context) error { + return nil +} + +// OnEnd implements trace.SpanProcessor. +func (t *spanTracker) OnEnd(s sdktrace.ReadOnlySpan) { + id := s.SpanContext().SpanID() + bucket := binary.BigEndian.Uint64(id[:]) % shardCount + t.inflightSpansMu[bucket].Lock() + defer t.inflightSpansMu[bucket].Unlock() + delete(t.inflightSpans[bucket], id) +} + +// OnStart implements trace.SpanProcessor. +func (t *spanTracker) OnStart(_ context.Context, s sdktrace.ReadWriteSpan) { + id := s.SpanContext().SpanID() + bucket := binary.BigEndian.Uint64(id[:]) % shardCount + t.inflightSpansMu[bucket].Lock() + defer t.inflightSpansMu[bucket].Unlock() + t.inflightSpans[bucket][id] = struct{}{} + + if t.debugFlags.Check(TrackSpanReferences) { + if s.Parent().IsValid() { + t.observer.ObserveReference(s.Parent().SpanID(), id) + } + t.observer.Observe(id) + } + + if t.debugFlags.Check(TrackAllSpans) { + var caller string + for _, attr := range s.Attributes() { + if attr.Key == "caller" { + caller = attr.Value.AsString() + break + } + } + t.allSpans.Store(id, &spanInfo{ + Name: s.Name(), + SpanContext: s.SpanContext(), + Parent: s.Parent(), + caller: caller, + startTime: s.StartTime(), + }) + } +} + +// Shutdown implements trace.SpanProcessor. +func (t *spanTracker) Shutdown(_ context.Context) error { + if t.debugFlags == 0 { + return nil + } + didWarn := false + t.shutdownOnce.Do(func() { + if t.debugFlags.Check(WarnOnUnresolvedReferences) { + var unknownParentIDs []string + for id, via := range t.observer.referencedIDs { + if via.IsValid() { + if t.debugFlags.Check(TrackAllSpans) { + if viaSpan, ok := t.allSpans.Load(via); ok { + unknownParentIDs = append(unknownParentIDs, fmt.Sprintf("%s via %s (%s)", id, via, viaSpan.(*spanInfo).Name)) + } else { + unknownParentIDs = append(unknownParentIDs, fmt.Sprintf("%s via %s", id, via)) + } + } + } + } + if len(unknownParentIDs) > 0 { + didWarn = true + msg := startMsg("WARNING: parent spans referenced but never seen:\n") + for _, str := range unknownParentIDs { + msg.WriteString(str) + msg.WriteString("\n") + } + endMsg(msg) + } + } + if t.debugFlags.Check(WarnOnIncompleteSpans) { + if t.debugFlags.Check(TrackAllSpans) { + incompleteSpans := []*spanInfo{} + t.inflightSpansMu.LockAll() + t.inflightSpans.Range(func(key oteltrace.SpanID) { + if info, ok := t.allSpans.Load(key); ok { + incompleteSpans = append(incompleteSpans, info.(*spanInfo)) + } + }) + t.inflightSpansMu.UnlockAll() + if len(incompleteSpans) > 0 { + didWarn = true + msg := startMsg("WARNING: spans not ended:\n") + longestName := 0 + for _, span := range incompleteSpans { + longestName = max(longestName, len(span.Name)+2) + } + for _, span := range incompleteSpans { + var startedAt string + if span.caller != "" { + startedAt = " | started at: " + span.caller + } + fmt.Fprintf(msg, "%-*s (trace: %s | span: %s | parent: %s%s)\n", longestName, "'"+span.Name+"'", + span.SpanContext.TraceID(), span.SpanContext.SpanID(), span.Parent.SpanID(), startedAt) + } + endMsg(msg) + } + } else { + incompleteSpans := []oteltrace.SpanID{} + t.inflightSpansMu.LockAll() + t.inflightSpans.Range(func(key oteltrace.SpanID) { + incompleteSpans = append(incompleteSpans, key) + }) + t.inflightSpansMu.UnlockAll() + if len(incompleteSpans) > 0 { + didWarn = true + msg := startMsg("WARNING: spans not ended:\n") + for _, span := range incompleteSpans { + fmt.Fprintf(msg, "%s\n", span) + } + msg.WriteString("Note: set TrackAllSpans flag for more info\n") + endMsg(msg) + } + } + } + + if t.debugFlags.Check(LogAllSpans) || (t.debugFlags.Check(LogAllSpansOnWarn) && didWarn) { + allSpans := []*spanInfo{} + t.allSpans.Range(func(_, value any) bool { + allSpans = append(allSpans, value.(*spanInfo)) + return true + }) + slices.SortFunc(allSpans, func(a, b *spanInfo) int { + return a.startTime.Compare(b.startTime) + }) + msg := startMsg("All observed spans:\n") + longestName := 0 + for _, span := range allSpans { + longestName = max(longestName, len(span.Name)+2) + } + for _, span := range allSpans { + var startedAt string + if span.caller != "" { + startedAt = " | started at: " + span.caller + } + fmt.Fprintf(msg, "%-*s (trace: %s | span: %s | parent: %s%s)\n", longestName, "'"+span.Name+"'", + span.SpanContext.TraceID(), span.SpanContext.SpanID(), span.Parent.SpanID(), startedAt) + } + endMsg(msg) + } + + if t.debugFlags.Check(LogTraceIDs) || (didWarn && t.debugFlags.Check(LogTraceIDsOnWarn)) { + msg := startMsg("Known trace ids:\n") + traceIDs := map[oteltrace.TraceID]int{} + t.allSpans.Range(func(_, value any) bool { + v := value.(*spanInfo) + traceIDs[v.SpanContext.TraceID()]++ + return true + }) + for id, n := range traceIDs { + fmt.Fprintf(msg, "%s (%d spans)\n", id.String(), n) + } + endMsg(msg) + } + }) + if didWarn { + return ErrIncompleteSpans + } + return nil +} + +func newSpanObserver() *spanObserver { + return &spanObserver{ + referencedIDs: map[oteltrace.SpanID]oteltrace.SpanID{}, + cond: sync.NewCond(&sync.Mutex{}), + } +} + +type spanObserver struct { + cond *sync.Cond + referencedIDs map[oteltrace.SpanID]oteltrace.SpanID + unobservedIDs int +} + +func (obs *spanObserver) ObserveReference(id oteltrace.SpanID, via oteltrace.SpanID) { + obs.cond.L.Lock() + defer obs.cond.L.Unlock() + if _, referenced := obs.referencedIDs[id]; !referenced { + obs.referencedIDs[id] = via // referenced, but not observed + // It is possible for new unobserved references to come in while waiting, + // but incrementing the counter wouldn't satisfy the condition so we don't + // need to signal the waiters + obs.unobservedIDs++ + } +} + +func (obs *spanObserver) Observe(id oteltrace.SpanID) { + obs.cond.L.Lock() + defer obs.cond.L.Unlock() + if observed, referenced := obs.referencedIDs[id]; !referenced || observed.IsValid() { // NB: subtle condition + obs.referencedIDs[id] = zeroSpanID + if referenced { + obs.unobservedIDs-- + obs.cond.Broadcast() + } + } +} + +func (obs *spanObserver) wait(warnAfter time.Duration) { + done := make(chan struct{}) + defer close(done) + go func() { + select { + case <-done: + return + case <-time.After(warnAfter): + obs.debugWarnWaiting() + } + }() + + obs.cond.L.Lock() + for obs.unobservedIDs > 0 { + obs.cond.Wait() + } + obs.cond.L.Unlock() +} + +func (obs *spanObserver) debugWarnWaiting() { + obs.cond.L.Lock() + msg := startMsg(fmt.Sprintf("Waiting on %d unobserved spans:\n", obs.unobservedIDs)) + for id, via := range obs.referencedIDs { + if via.IsValid() { + fmt.Fprintf(msg, "%s via %s\n", id, via) + } + } + endMsg(msg) + obs.cond.L.Unlock() +} diff --git a/internal/telemetry/trace/debug_test.go b/internal/telemetry/trace/debug_test.go new file mode 100644 index 000000000..95a1c0ff0 --- /dev/null +++ b/internal/telemetry/trace/debug_test.go @@ -0,0 +1,288 @@ +package trace_test + +import ( + "bytes" + "context" + "fmt" + "runtime" + "sync/atomic" + "testing" + "time" + + "github.com/pomerium/pomerium/internal/telemetry/trace" + . "github.com/pomerium/pomerium/internal/testutil/tracetest" //nolint:revive + "github.com/stretchr/testify/assert" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + oteltrace "go.opentelemetry.io/otel/trace" +) + +func TestSpanObserver(t *testing.T) { + t.Run("observe single reference", func(t *testing.T) { + obs := trace.NewSpanObserver() + assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) + + obs.ObserveReference(Span(1).ID(), Span(2).ID()) + assert.Equal(t, []oteltrace.SpanID{Span(1).ID()}, obs.XUnobservedIDs()) + obs.Observe(Span(1).ID()) + assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) + }) + t.Run("observe multiple references", func(t *testing.T) { + obs := trace.NewSpanObserver() + + obs.ObserveReference(Span(1).ID(), Span(2).ID()) + obs.ObserveReference(Span(1).ID(), Span(3).ID()) + obs.ObserveReference(Span(1).ID(), Span(4).ID()) + assert.Equal(t, []oteltrace.SpanID{Span(1).ID()}, obs.XUnobservedIDs()) + obs.Observe(Span(1).ID()) + assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) + }) + t.Run("observe before reference", func(t *testing.T) { + obs := trace.NewSpanObserver() + + obs.Observe(Span(1).ID()) + assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) + obs.ObserveReference(Span(1).ID(), Span(2).ID()) + assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) + }) + + t.Run("wait", func(t *testing.T) { + obs := trace.NewSpanObserver() + obs.ObserveReference(Span(1).ID(), Span(2).ID()) + obs.Observe(Span(2).ID()) + obs.ObserveReference(Span(3).ID(), Span(4).ID()) + obs.Observe(Span(4).ID()) + obs.ObserveReference(Span(5).ID(), Span(6).ID()) + obs.Observe(Span(6).ID()) + waitOkToExit := atomic.Bool{} + waitExited := atomic.Bool{} + go func() { + defer waitExited.Store(true) + obs.XWait() + assert.True(t, waitOkToExit.Load(), "wait exited early") + }() + + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + obs.Observe(Span(1).ID()) + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + obs.Observe(Span(3).ID()) + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + waitOkToExit.Store(true) + obs.Observe(Span(5).ID()) + assert.Eventually(t, waitExited.Load, 10*time.Millisecond, 1*time.Millisecond) + }) + + t.Run("new references observed during wait", func(t *testing.T) { + obs := trace.NewSpanObserver() + obs.ObserveReference(Span(1).ID(), Span(2).ID()) + obs.Observe(Span(2).ID()) + obs.ObserveReference(Span(3).ID(), Span(4).ID()) + obs.Observe(Span(4).ID()) + obs.ObserveReference(Span(5).ID(), Span(6).ID()) + obs.Observe(Span(6).ID()) + waitOkToExit := atomic.Bool{} + waitExited := atomic.Bool{} + go func() { + defer waitExited.Store(true) + obs.XWait() + assert.True(t, waitOkToExit.Load(), "wait exited early") + }() + + assert.Equal(t, []oteltrace.SpanID{Span(1).ID(), Span(3).ID(), Span(5).ID()}, obs.XUnobservedIDs()) + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + obs.Observe(Span(1).ID()) + assert.Equal(t, []oteltrace.SpanID{Span(3).ID(), Span(5).ID()}, obs.XUnobservedIDs()) + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + obs.Observe(Span(3).ID()) + assert.Equal(t, []oteltrace.SpanID{Span(5).ID()}, obs.XUnobservedIDs()) + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + // observe a new reference + obs.ObserveReference(Span(7).ID(), Span(8).ID()) + obs.Observe(Span(8).ID()) + assert.Equal(t, []oteltrace.SpanID{Span(5).ID(), Span(7).ID()}, obs.XUnobservedIDs()) + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + obs.Observe(Span(5).ID()) + assert.Equal(t, []oteltrace.SpanID{Span(7).ID()}, obs.XUnobservedIDs()) + time.Sleep(10 * time.Millisecond) + assert.False(t, waitExited.Load()) + + waitOkToExit.Store(true) + obs.Observe(Span(7).ID()) + assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) + assert.Eventually(t, waitExited.Load, 10*time.Millisecond, 1*time.Millisecond) + }) + + t.Run("multiple waiters", func(t *testing.T) { + t.Parallel() + obs := trace.NewSpanObserver() + obs.ObserveReference(Span(1).ID(), Span(2).ID()) + obs.Observe(Span(2).ID()) + + waitersExited := atomic.Int32{} + for range 10 { + go func() { + defer waitersExited.Add(1) + obs.XWait() + }() + } + + assert.Equal(t, []oteltrace.SpanID{Span(1).ID()}, obs.XUnobservedIDs()) + time.Sleep(10 * time.Millisecond) + assert.Equal(t, int32(0), waitersExited.Load()) + + obs.Observe(Span(1).ID()) + + startTime := time.Now() + for waitersExited.Load() != 10 { + if time.Since(startTime) > 1*time.Millisecond { + t.Fatal("timed out") + } + runtime.Gosched() + } + }) +} + +func TestSpanTracker(t *testing.T) { + t.Run("no debug flags", func(t *testing.T) { + t.Parallel() + obs := trace.NewSpanObserver() + tracker := trace.NewSpanTracker(obs, 0) + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracker)) + tracer := tp.Tracer("test") + assert.Equal(t, []oteltrace.SpanID{}, tracker.XInflightSpans()) + _, span1 := tracer.Start(context.Background(), "span 1") + assert.Equal(t, []oteltrace.SpanID{span1.SpanContext().SpanID()}, tracker.XInflightSpans()) + assert.Equal(t, []oteltrace.SpanID{}, obs.XObservedIDs()) + span1.End() + assert.Equal(t, []oteltrace.SpanID{}, tracker.XInflightSpans()) + assert.Equal(t, []oteltrace.SpanID{}, obs.XObservedIDs()) + }) + t.Run("with TrackSpanReferences debug flag", func(t *testing.T) { + t.Parallel() + obs := trace.NewSpanObserver() + tracker := trace.NewSpanTracker(obs, trace.TrackSpanReferences) + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracker)) + tracer := tp.Tracer("test") + assert.Equal(t, []oteltrace.SpanID{}, tracker.XInflightSpans()) + _, span1 := tracer.Start(context.Background(), "span 1") + assert.Equal(t, []oteltrace.SpanID{span1.SpanContext().SpanID()}, tracker.XInflightSpans()) + assert.Equal(t, []oteltrace.SpanID{span1.SpanContext().SpanID()}, obs.XObservedIDs()) + span1.End() + assert.Equal(t, []oteltrace.SpanID{}, tracker.XInflightSpans()) + assert.Equal(t, []oteltrace.SpanID{span1.SpanContext().SpanID()}, obs.XObservedIDs()) + }) +} + +func TestSpanTrackerWarnings(t *testing.T) { + t.Run("WarnOnIncompleteSpans", func(t *testing.T) { + var buf bytes.Buffer + trace.SetDebugMessageWriterForTest(t, &buf) + + obs := trace.NewSpanObserver() + tracker := trace.NewSpanTracker(obs, trace.WarnOnIncompleteSpans) + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracker)) + tracer := tp.Tracer("test") + _, span1 := tracer.Start(context.Background(), "span 1") + + assert.ErrorIs(t, tp.Shutdown(context.Background()), trace.ErrIncompleteSpans) + + assert.Equal(t, fmt.Sprintf(` +================================================== +WARNING: spans not ended: +%s +Note: set TrackAllSpans flag for more info +================================================== +`, span1.SpanContext().SpanID()), buf.String()) + }) + + t.Run("WarnOnIncompleteSpans with TrackAllSpans", func(t *testing.T) { + var buf bytes.Buffer + trace.SetDebugMessageWriterForTest(t, &buf) + + obs := trace.NewSpanObserver() + tracker := trace.NewSpanTracker(obs, trace.WarnOnIncompleteSpans|trace.TrackAllSpans) + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracker)) + tracer := tp.Tracer("test") + _, span1 := tracer.Start(context.Background(), "span 1") + + assert.ErrorIs(t, tp.Shutdown(context.Background()), trace.ErrIncompleteSpans) + + assert.Equal(t, fmt.Sprintf(` +================================================== +WARNING: spans not ended: +'span 1' (trace: %s | span: %s | parent: 0000000000000000) +================================================== +`, span1.SpanContext().TraceID(), span1.SpanContext().SpanID()), buf.String()) + }) + + t.Run("WarnOnIncompleteSpans with TrackAllSpans and stackTraceProcessor", func(t *testing.T) { + var buf bytes.Buffer + trace.SetDebugMessageWriterForTest(t, &buf) + + obs := trace.NewSpanObserver() + tracker := trace.NewSpanTracker(obs, trace.WarnOnIncompleteSpans|trace.TrackAllSpans) + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(&trace.XStackTraceProcessor{}), sdktrace.WithSpanProcessor(tracker)) + tracer := tp.Tracer("test") + _, span1 := tracer.Start(context.Background(), "span 1") + _, file, line, _ := runtime.Caller(0) + line-- + + assert.ErrorIs(t, tp.Shutdown(context.Background()), trace.ErrIncompleteSpans) + + assert.Equal(t, fmt.Sprintf(` +================================================== +WARNING: spans not ended: +'span 1' (trace: %s | span: %s | parent: 0000000000000000 | started at: %s:%d) +================================================== +`, span1.SpanContext().TraceID(), span1.SpanContext().SpanID(), file, line), buf.String()) + }) + + t.Run("LogAllSpansOnWarn", func(t *testing.T) { + var buf bytes.Buffer + trace.SetDebugMessageWriterForTest(t, &buf) + + obs := trace.NewSpanObserver() + tracker := trace.NewSpanTracker(obs, trace.WarnOnIncompleteSpans|trace.TrackAllSpans|trace.LogAllSpansOnWarn) + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(&trace.XStackTraceProcessor{}), sdktrace.WithSpanProcessor(tracker)) + tracer := tp.Tracer("test") + _, span1 := tracer.Start(context.Background(), "span 1") + time.Sleep(10 * time.Millisecond) + span1.End() + time.Sleep(10 * time.Millisecond) + _, span2 := tracer.Start(context.Background(), "span 2") + _, file, line, _ := runtime.Caller(0) + line-- + + tp.Shutdown(context.Background()) + + assert.Equal(t, + fmt.Sprintf(` +================================================== +WARNING: spans not ended: +'span 2' (trace: %[1]s | span: %[2]s | parent: 0000000000000000 | started at: %[3]s:%[4]d) +================================================== + +================================================== +All observed spans: +'span 1' (trace: %[5]s | span: %[6]s | parent: 0000000000000000 | started at: %[3]s:%[7]d) +'span 2' (trace: %[1]s | span: %[2]s | parent: 0000000000000000 | started at: %[3]s:%[4]d) +================================================== +`, + span2.SpanContext().TraceID(), span2.SpanContext().SpanID(), file, line, + span1.SpanContext().TraceID(), span1.SpanContext().SpanID(), line-4, + ), buf.String()) + }) +} diff --git a/internal/telemetry/trace/middleware.go b/internal/telemetry/trace/middleware.go index f80d04d1b..771e44265 100644 --- a/internal/telemetry/trace/middleware.go +++ b/internal/telemetry/trace/middleware.go @@ -8,91 +8,25 @@ import ( "github.com/gorilla/mux" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/propagation" - "google.golang.org/grpc/metadata" "google.golang.org/grpc/stats" ) -func NewHTTPMiddleware(opts ...otelhttp.Option) func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - routeStr := "" - route := mux.CurrentRoute(r) - if route != nil { - var err error - routeStr, err = route.GetPathTemplate() +func NewHTTPMiddleware(opts ...otelhttp.Option) mux.MiddlewareFunc { + return otelhttp.NewMiddleware("Server: %s %s", append(opts, otelhttp.WithSpanNameFormatter(func(operation string, r *http.Request) string { + routeStr := "" + route := mux.CurrentRoute(r) + if route != nil { + var err error + routeStr, err = route.GetPathTemplate() + if err != nil { + routeStr, err = route.GetPathRegexp() if err != nil { - routeStr, err = route.GetPathRegexp() - if err != nil { - routeStr = "" - } + routeStr = "" } } - traceparent := r.Header.Get("Traceparent") - if traceparent != "" { - xPomeriumTraceparent := r.Header.Get("X-Pomerium-Traceparent") - if xPomeriumTraceparent != "" { - sc, err := ParseTraceparent(xPomeriumTraceparent) - if err == nil { - r.Header.Set("Traceparent", WithTraceFromSpanContext(traceparent, sc)) - ctx := otel.GetTextMapPropagator().Extract(r.Context(), propagation.HeaderCarrier(r.Header)) - r = r.WithContext(ctx) - } - } - } - otelhttp.NewHandler(next, fmt.Sprintf("Server: %s %s", r.Method, routeStr), opts...).ServeHTTP(w, r) - }) - } -} - -func NewServerStatsHandler(base stats.Handler) stats.Handler { - return &serverStatsHandlerWrapper{ - base: base, - } -} - -type serverStatsHandlerWrapper struct { - base stats.Handler -} - -func (w *serverStatsHandlerWrapper) wrapContext(ctx context.Context) context.Context { - md, ok := metadata.FromIncomingContext(ctx) - if !ok { - return ctx - } - traceparent := md.Get("traceparent") - xPomeriumTraceparent := md.Get("x-pomerium-traceparent") - if len(traceparent) > 0 && traceparent[0] != "" && len(xPomeriumTraceparent) > 0 && xPomeriumTraceparent[0] != "" { - newTracectx, err := ParseTraceparent(xPomeriumTraceparent[0]) - if err != nil { - return ctx } - - md.Set("traceparent", WithTraceFromSpanContext(traceparent[0], newTracectx)) - return metadata.NewIncomingContext(ctx, md) - } - return ctx -} - -// HandleConn implements stats.Handler. -func (w *serverStatsHandlerWrapper) HandleConn(ctx context.Context, stats stats.ConnStats) { - w.base.HandleConn(w.wrapContext(ctx), stats) -} - -// HandleRPC implements stats.Handler. -func (w *serverStatsHandlerWrapper) HandleRPC(ctx context.Context, stats stats.RPCStats) { - w.base.HandleRPC(w.wrapContext(ctx), stats) -} - -// TagConn implements stats.Handler. -func (w *serverStatsHandlerWrapper) TagConn(ctx context.Context, info *stats.ConnTagInfo) context.Context { - return w.base.TagConn(w.wrapContext(ctx), info) -} - -// TagRPC implements stats.Handler. -func (w *serverStatsHandlerWrapper) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context { - return w.base.TagRPC(w.wrapContext(ctx), info) + return fmt.Sprintf(operation, r.Method, routeStr) + }))...) } type clientStatsHandlerWrapper struct { diff --git a/internal/telemetry/trace/middleware_test.go b/internal/telemetry/trace/middleware_test.go index 47fc6b369..ff01cc4d2 100644 --- a/internal/telemetry/trace/middleware_test.go +++ b/internal/telemetry/trace/middleware_test.go @@ -3,161 +3,35 @@ package trace_test import ( "context" "errors" - "net" "net/http" "net/http/httptest" "testing" "time" + "github.com/gorilla/mux" "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + sdktrace "go.opentelemetry.io/otel/sdk/trace" oteltrace "go.opentelemetry.io/otel/trace" - "go.opentelemetry.io/otel/trace/noop" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/interop/grpc_testing" "google.golang.org/grpc/metadata" "google.golang.org/grpc/stats" - "google.golang.org/grpc/test/bufconn" ) -var cases = []struct { - name string - setTraceparent string - setPomeriumTraceparent string - check func(t testing.TB, ctx context.Context) -}{ - { - name: "x-pomerium-traceparent not present", - setTraceparent: Traceparent(Trace(1), Span(1), true), - check: func(t testing.TB, ctx context.Context) { - span := oteltrace.SpanFromContext(ctx) - assert.Equal(t, Trace(1).ID().Value(), span.SpanContext().TraceID()) - assert.Equal(t, Span(1).ID(), span.SpanContext().SpanID()) - assert.True(t, span.SpanContext().IsSampled()) - }, - }, - { - name: "x-pomerium-traceparent present", - setTraceparent: Traceparent(Trace(2), Span(2), true), - setPomeriumTraceparent: Traceparent(Trace(1), Span(1), true), - check: func(t testing.TB, ctx context.Context) { - span := oteltrace.SpanFromContext(ctx) - assert.Equal(t, Trace(1).ID().Value(), span.SpanContext().TraceID()) - assert.Equal(t, Span(2).ID(), span.SpanContext().SpanID()) - assert.True(t, span.SpanContext().IsSampled()) - }, - }, - { - name: "x-pomerium-traceparent present, force sampling off", - setTraceparent: Traceparent(Trace(2), Span(2), true), - setPomeriumTraceparent: Traceparent(Trace(1), Span(1), false), - check: func(t testing.TB, ctx context.Context) { - span := oteltrace.SpanFromContext(ctx) - assert.Equal(t, Trace(1).ID().Value(), span.SpanContext().TraceID()) - assert.Equal(t, Span(2).ID(), span.SpanContext().SpanID()) - assert.Equal(t, false, span.SpanContext().IsSampled()) - }, - }, - { - name: "x-pomerium-traceparent present, force sampling on", - setTraceparent: Traceparent(Trace(2), Span(2), false), - setPomeriumTraceparent: Traceparent(Trace(1), Span(1), true), - check: func(t testing.TB, ctx context.Context) { - span := oteltrace.SpanFromContext(ctx) - assert.Equal(t, Trace(1).ID().Value(), span.SpanContext().TraceID()) - assert.Equal(t, Span(2).ID(), span.SpanContext().SpanID()) - assert.Equal(t, true, span.SpanContext().IsSampled()) - }, - }, - { - name: "malformed x-pomerium-traceparent", - setTraceparent: Traceparent(Trace(2), Span(2), false), - setPomeriumTraceparent: "00-xxxxxx-yyyyyy-03", - check: func(t testing.TB, ctx context.Context) { - span := oteltrace.SpanFromContext(ctx) - assert.Equal(t, Trace(2).ID().Value(), span.SpanContext().TraceID()) - assert.Equal(t, Span(2).ID(), span.SpanContext().SpanID()) - assert.Equal(t, false, span.SpanContext().IsSampled()) - }, - }, -} - func TestHTTPMiddleware(t *testing.T) { - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - r := httptest.NewRequest(http.MethodGet, "/foo", nil) - if tc.setTraceparent != "" { - r.Header.Add("Traceparent", tc.setTraceparent) - } - if tc.setPomeriumTraceparent != "" { - r.Header.Add("X-Pomerium-Traceparent", tc.setPomeriumTraceparent) - } - w := httptest.NewRecorder() - trace.NewHTTPMiddleware( - otelhttp.WithTracerProvider(noop.NewTracerProvider()), - )(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { - tc.check(t, r.Context()) - })).ServeHTTP(w, r) - }) - } -} - -func TestGRPCMiddleware(t *testing.T) { - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - srv := grpc.NewServer( - grpc.StatsHandler(trace.NewServerStatsHandler(otelgrpc.NewServerHandler( - otelgrpc.WithTracerProvider(noop.NewTracerProvider())))), - grpc.Creds(insecure.NewCredentials()), - ) - lis := bufconn.Listen(4096) - grpc_testing.RegisterTestServiceServer(srv, &testServer{ - fn: func(ctx context.Context) { - tc.check(t, ctx) - }, - }) - go srv.Serve(lis) - t.Cleanup(srv.Stop) - - client, err := grpc.NewClient("passthrough://ignore", - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithStatsHandler(otelgrpc.NewClientHandler( - otelgrpc.WithTracerProvider(noop.NewTracerProvider()))), - grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { - return lis.DialContext(ctx) - }), - ) - require.NoError(t, err) - - ctx := context.Background() - if tc.setTraceparent != "" { - ctx = metadata.AppendToOutgoingContext(ctx, - "traceparent", tc.setTraceparent, - ) - } - if tc.setPomeriumTraceparent != "" { - ctx = metadata.AppendToOutgoingContext(ctx, - "x-pomerium-traceparent", tc.setPomeriumTraceparent, - ) - } - _, err = grpc_testing.NewTestServiceClient(client).EmptyCall(ctx, &grpc_testing.Empty{}) - assert.NoError(t, err) - }) - } -} - -type testServer struct { - grpc_testing.UnimplementedTestServiceServer - fn func(ctx context.Context) -} - -func (ts *testServer) EmptyCall(ctx context.Context, _ *grpc_testing.Empty) (*grpc_testing.Empty, error) { - ts.fn(ctx) - return &grpc_testing.Empty{}, nil + router := mux.NewRouter() + tp := sdktrace.NewTracerProvider() + router.Use(trace.NewHTTPMiddleware( + otelhttp.WithTracerProvider(tp), + )) + router.Path("/foo").HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + span := oteltrace.SpanFromContext(r.Context()) + assert.Equal(t, "Server: GET /foo", span.(interface{ Name() string }).Name()) + }).Methods(http.MethodGet) + w := httptest.NewRecorder() + ctx, span := tp.Tracer("test").Start(context.Background(), "test") + router.ServeHTTP(w, httptest.NewRequestWithContext(ctx, http.MethodGet, "/foo", nil)) + span.End() } type mockHandler struct { diff --git a/internal/telemetry/trace/queue.go b/internal/telemetry/trace/queue.go deleted file mode 100644 index e7ed06d31..000000000 --- a/internal/telemetry/trace/queue.go +++ /dev/null @@ -1,900 +0,0 @@ -package trace - -import ( - "context" - "encoding/binary" - "encoding/json" - "errors" - "fmt" - "os" - "slices" - "strconv" - "sync" - "sync/atomic" - "time" - "unique" - - lru "github.com/hashicorp/golang-lru/v2" - "github.com/pomerium/pomerium/internal/log" - "github.com/rs/zerolog" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - oteltrace "go.opentelemetry.io/otel/trace" - coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" - tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" - "google.golang.org/protobuf/encoding/protojson" - "google.golang.org/protobuf/proto" -) - -var ( - maxPendingTraces atomic.Int32 - maxCachedTraceIDs atomic.Int32 -) - -func init() { - envOrDefault := func(envName string, def int32) int32 { - if val, ok := os.LookupEnv(envName); ok { - if num, err := strconv.ParseInt(val, 10, 32); err == nil { - return int32(num) - } - } - return def - } - maxPendingTraces.Store(envOrDefault("POMERIUM_OTEL_MAX_PENDING_TRACES", 8192)) - maxCachedTraceIDs.Store(envOrDefault("POMERIUM_OTEL_MAX_CACHED_TRACE_IDS", 16384)) -} - -func SetMaxPendingTraces(num int32) { - maxPendingTraces.Store(max(num, 0)) -} - -func SetMaxCachedTraceIDs(num int32) { - maxCachedTraceIDs.Store(max(num, 0)) -} - -type eviction struct { - traceID unique.Handle[oteltrace.TraceID] - buf *Buffer -} - -type SpanExportQueue struct { - closing chan struct{} - uploadC chan []*tracev1.ResourceSpans - requestC chan *coltracepb.ExportTraceServiceRequest - evictC chan eviction - client otlptrace.Client - pendingResourcesByTraceID *lru.Cache[unique.Handle[oteltrace.TraceID], *Buffer] - knownTraceIDMappings *lru.Cache[unique.Handle[oteltrace.TraceID], unique.Handle[oteltrace.TraceID]] - tracker *spanTracker - observer *spanObserver - debugEvents []DebugEvent - logger *zerolog.Logger - debugFlags DebugFlags - debugAllEnqueuedSpans map[oteltrace.SpanID]*tracev1.Span - wg sync.WaitGroup -} - -func NewSpanExportQueue(ctx context.Context, client otlptrace.Client) *SpanExportQueue { - debug := DebugFlagsFromContext(ctx) - var observer *spanObserver - if debug.Check(TrackSpanReferences) { - observer = newSpanObserver() - } - q := &SpanExportQueue{ - logger: log.Ctx(ctx), - client: client, - closing: make(chan struct{}), - uploadC: make(chan []*tracev1.ResourceSpans, 64), - requestC: make(chan *coltracepb.ExportTraceServiceRequest, 256), - evictC: make(chan eviction, 64), - debugFlags: debug, - debugAllEnqueuedSpans: make(map[oteltrace.SpanID]*tracev1.Span), - tracker: newSpanTracker(observer, debug), - observer: observer, - } - var err error - q.pendingResourcesByTraceID, err = lru.NewWithEvict(int(maxPendingTraces.Load()), q.onEvict) - if err != nil { - panic(err) - } - q.knownTraceIDMappings, err = lru.New[unique.Handle[oteltrace.TraceID], unique.Handle[oteltrace.TraceID]](int(maxCachedTraceIDs.Load())) - if err != nil { - panic(err) - } - q.wg.Add(2) - go q.runUploader() - go q.runProcessor() - return q -} - -func (q *SpanExportQueue) runUploader() { - defer q.wg.Done() - for resourceSpans := range q.uploadC { - ctx, ca := context.WithTimeout(context.Background(), 10*time.Second) - if err := q.client.UploadTraces(ctx, resourceSpans); err != nil { - q.logger.Err(err).Msg("error uploading traces") - } - ca() - } -} - -func (q *SpanExportQueue) runProcessor() { - defer q.wg.Done() - for { - select { - case req := <-q.requestC: - q.processRequestLocked(req) - case ev := <-q.evictC: - q.processEvictionLocked(ev) - case <-q.closing: - for { - select { - case req := <-q.requestC: - q.processRequestLocked(req) - case ev := <-q.evictC: - q.processEvictionLocked(ev) - default: // all channels empty - close(q.uploadC) - return - } - } - } - } -} - -func (q *SpanExportQueue) onEvict(traceID unique.Handle[oteltrace.TraceID], buf *Buffer) { - q.evictC <- eviction{ - traceID: traceID, - buf: buf, - } -} - -func (q *SpanExportQueue) insertPendingSpanLocked( - resource *ResourceInfo, - scope *ScopeInfo, - traceID unique.Handle[oteltrace.TraceID], - span *tracev1.Span, -) { - var pendingTraceResources *Buffer - - if ptr, ok := q.pendingResourcesByTraceID.Get(traceID); ok { - pendingTraceResources = ptr - } else { - pendingTraceResources = NewBuffer() - q.pendingResourcesByTraceID.Add(traceID, pendingTraceResources) - } - pendingTraceResources.Insert(resource, scope, span) -} - -func (q *SpanExportQueue) resolveTraceIDMappingLocked(out *Buffer, original, target unique.Handle[oteltrace.TraceID]) { - q.knownTraceIDMappings.Add(original, target) - - if target == zeroTraceID && original != zeroTraceID { - // mapping a trace id to zero indicates we should drop the trace - q.pendingResourcesByTraceID.Remove(original) - return - } - - if originalPending, ok := q.pendingResourcesByTraceID.Peek(original); ok { - if original == target { - out.Merge(originalPending) - } else { - // check if the target id is also pending - if targetPending, ok := q.pendingResourcesByTraceID.Peek(target); ok { - targetPending.MergeAs(originalPending, target) - } else { - out.MergeAs(originalPending, target) - } - } - q.pendingResourcesByTraceID.Remove(original) - } -} - -func (q *SpanExportQueue) getTraceIDMappingLocked(id unique.Handle[oteltrace.TraceID]) (unique.Handle[oteltrace.TraceID], bool) { - v, ok := q.knownTraceIDMappings.Get(id) - return v, ok -} - -func (q *SpanExportQueue) isKnownTracePendingLocked(id unique.Handle[oteltrace.TraceID]) bool { - _, ok := q.pendingResourcesByTraceID.Get(id) // will update the key's recent-ness in the lru - return ok -} - -var ErrShuttingDown = errors.New("exporter is shutting down") - -func (q *SpanExportQueue) Enqueue(_ context.Context, req *coltracepb.ExportTraceServiceRequest) error { - select { - case <-q.closing: - return ErrShuttingDown - default: - q.requestC <- req - return nil - } -} - -func (q *SpanExportQueue) processRequestLocked(req *coltracepb.ExportTraceServiceRequest) { - if q.debugFlags.Check(LogAllEvents) { - q.debugEvents = append(q.debugEvents, DebugEvent{ - Timestamp: time.Now(), - Request: proto.Clone(req).(*coltracepb.ExportTraceServiceRequest), - }) - } - - // Spans are processed in two passes: - // 1. Look through each span to check if we have not yet seen its trace ID. - // If we haven't, and the span is a root span (no parent, or marked as such - // by us), mark the trace as observed, and (if indicated) keep track of the - // trace ID we need to rewrite it as, so that other spans we see later in - // this trace can also be rewritten the same way. - // If we find a new trace ID for which there are pending non-root spans, - // collect them and rewrite their trace IDs (if necessary), and prepare - // them to be uploaded. - // - // At this point, all trace IDs for the spans in the request are known. - // - // 2. Look through each span again, this time to filter out any spans in - // the request which belong to "pending" traces (known trace IDs for which - // we have not yet seen a root span), adding them to the list of pending - // spans for their corresponding trace IDs. They will be uploaded in the - // future once we have observed a root span for those traces, or if they - // are evicted by the queue. - - // Pass 1 - toUpload := NewBuffer() - for _, resource := range req.ResourceSpans { - for _, scope := range resource.ScopeSpans { - SPANS: - for _, span := range scope.Spans { - FormatSpanName(span) - spanID, ok := ToSpanID(span.SpanId) - if !ok { - continue - } - if q.debugFlags.Check(TrackAllSpans) { - q.debugAllEnqueuedSpans[spanID] = span - } - trackSpanReferences := q.debugFlags.Check(TrackSpanReferences) - parentSpanID, ok := ToSpanID(span.ParentSpanId) - if !ok { - continue - } - traceID, ok := ToTraceID(span.TraceId) - if !ok { - continue - } - if trackSpanReferences { - q.observer.Observe(spanID) - } - if mapping, ok := q.getTraceIDMappingLocked(traceID); ok { - if trackSpanReferences && mapping != zeroTraceID && parentSpanID.IsValid() { - q.observer.ObserveReference(parentSpanID, spanID) - } - } else { - // Observed a new trace ID. Check if the span is a root span - isRootSpan := !parentSpanID.IsValid() // no parent == root span - - // Assume the trace is sampled, because it was exported. span.Flags - // is an unreliable way to detect whether the span was sampled, - // because neither envoy nor opentelemetry-go encode the sampling - // decision there, assuming unsampled spans would not be exported - // (this was not taking into account tail-based sampling strategies) - // https://github.com/open-telemetry/opentelemetry-proto/issues/166 - isSampled := true - - mappedTraceID := traceID - for _, attr := range span.Attributes { - switch attr.Key { - case "pomerium.traceparent": - tp, err := ParseTraceparent(attr.GetValue().GetStringValue()) - if err != nil { - data, _ := protojson.Marshal(span) - q.logger. - Err(err). - Str("span", string(data)). - Msg("error processing span") - continue SPANS - } - mappedTraceID = unique.Make(tp.TraceID()) - // use the sampling decision from pomerium.traceparent instead - isSampled = tp.IsSampled() - case "pomerium.external-parent-span": - // This is a non-root span whose parent we do not expect to see - // here. For example, if a request originated externally from a - // system that is uploading its own spans out-of-band from us, - // we will never observe a root span for this trace and it would - // otherwise get stuck in the queue. - if !isRootSpan && q.debugFlags.Check(TrackSpanReferences) { - value, err := oteltrace.SpanIDFromHex(attr.GetValue().GetStringValue()) - if err != nil { - data, _ := protojson.Marshal(span) - q.logger. - Err(err). - Str("span", string(data)). - Msg("error processing span: invalid value for pomerium.external-parent-span") - } else { - q.observer.Observe(value) // mark this id as observed - } - } - isRootSpan = true - } - } - - if q.debugFlags.Check(TrackSpanReferences) { - if isSampled && parentSpanID.IsValid() { - q.observer.ObserveReference(parentSpanID, spanID) - } - } - - if !isSampled { - // We have observed a new trace that is not sampled (regardless of - // whether or not it is a root span). Resolve it using the zero - // trace ID to indicate that all spans for this trace should be - // dropped. - q.resolveTraceIDMappingLocked(toUpload, traceID, zeroTraceID) - } else if isRootSpan { - // We have observed a new trace that is sampled and is a root span. - // Resolve it using the mapped trace ID (if present), or its own - // trace ID (indicating it does not need to be rewritten). - // If the mapped trace is pending, this does not flush pending - // spans to the output buffer (toUpload), but instead merges them - // into the mapped trace's pending buffer. - q.resolveTraceIDMappingLocked(toUpload, traceID, mappedTraceID) - } - } - } - } - } - - // Pass 2 - for _, resource := range req.ResourceSpans { - resourceInfo := NewResourceInfo(resource.Resource, resource.SchemaUrl) - for _, scope := range resource.ScopeSpans { - scopeInfo := NewScopeInfo(scope.Scope, scope.SchemaUrl) - for _, span := range scope.Spans { - traceID, ok := ToTraceID(span.TraceId) - if !ok { - continue - } - - if mapping, hasMapping := q.getTraceIDMappingLocked(traceID); hasMapping { - if mapping == zeroTraceID { - continue // the trace has been dropped - } - id := mapping.Value() - copy(span.TraceId, id[:]) - // traceID = mapping - if q.isKnownTracePendingLocked(mapping) { - q.insertPendingSpanLocked(resourceInfo, scopeInfo, mapping, span) - } else { - toUpload.Insert(resourceInfo, scopeInfo, span) - } - } else { - q.insertPendingSpanLocked(resourceInfo, scopeInfo, traceID, span) - } - } - } - } - if resourceSpans := toUpload.Flush(); len(resourceSpans) > 0 { - q.uploadC <- resourceSpans - } -} - -func (q *SpanExportQueue) processEvictionLocked(ev eviction) { - if ev.buf.IsEmpty() { - // if the buffer is not empty, it was evicted automatically - return - } else if mapping, ok := q.knownTraceIDMappings.Get(ev.traceID); ok && mapping == zeroTraceID { - q.logger.Debug(). - Str("traceID", ev.traceID.Value().String()). - Msg("dropping unsampled trace") - return - } - - select { - case q.uploadC <- ev.buf.Flush(): - q.logger.Warn(). - Str("traceID", ev.traceID.Value().String()). - Msg("trace export buffer is full, uploading oldest incomplete trace") - default: - q.logger.Warn(). - Str("traceID", ev.traceID.Value().String()). - Msg("trace export buffer and upload queues are full, dropping trace") - } -} - -var ( - ErrIncompleteTraces = errors.New("exporter shut down with incomplete traces") - ErrIncompleteSpans = errors.New("exporter shut down with incomplete spans") - ErrIncompleteUploads = errors.New("exporter shut down with pending trace uploads") - ErrMissingParentSpans = errors.New("exporter shut down with missing parent spans") -) - -func (q *SpanExportQueue) WaitForSpans(maxDuration time.Duration) error { - if !q.debugFlags.Check(TrackSpanReferences) { - return nil - } - done := make(chan struct{}) - go func() { - defer close(done) - q.observer.wait(q.debugAllEnqueuedSpans, 10*time.Second) - }() - select { - case <-done: - return nil - case <-time.After(maxDuration): - return ErrMissingParentSpans - } -} - -func (q *SpanExportQueue) Close(ctx context.Context) error { - closed := make(chan struct{}) - go func() { - q.wg.Wait() - close(closed) - }() - close(q.closing) - select { - case <-ctx.Done(): - log.Ctx(ctx).Error().Msg("exporter stopped before all traces could be exported") - return context.Cause(ctx) - case <-closed: - err := q.runOnCloseChecksLocked() - log.Ctx(ctx).Debug().Err(err).Msg("exporter stopped") - return err - } -} - -func (q *SpanExportQueue) runOnCloseChecksLocked() error { - didWarn := false - if q.debugFlags.Check(TrackSpanReferences) { - var unknownParentIDs []string - for id, via := range q.observer.referencedIDs { - if via.IsValid() { - if q.debugFlags.Check(TrackAllSpans) { - if viaSpan, ok := q.debugAllEnqueuedSpans[via]; ok { - unknownParentIDs = append(unknownParentIDs, fmt.Sprintf("%s via %s (%s)", id, via, viaSpan.Name)) - } else { - unknownParentIDs = append(unknownParentIDs, fmt.Sprintf("%s via %s", id, via)) - } - } - } - } - if len(unknownParentIDs) > 0 { - didWarn = true - msg := startMsg("WARNING: parent spans referenced but never seen:\n") - for _, str := range unknownParentIDs { - msg.WriteString(str) - msg.WriteString("\n") - } - endMsg(msg) - } - } - incomplete := q.pendingResourcesByTraceID.Len() > 0 - if incomplete && q.debugFlags.Check(WarnOnIncompleteTraces) { - didWarn = true - msg := startMsg("WARNING: exporter shut down with incomplete traces\n") - keys := q.pendingResourcesByTraceID.Keys() - values := q.pendingResourcesByTraceID.Values() - for i, k := range keys { - v := values[i] - fmt.Fprintf(msg, "- Trace: %s\n", k.Value()) - for _, pendingScope := range v.scopesByResourceID { - msg.WriteString(" - Resource:\n") - for _, v := range pendingScope.resource.Resource.Attributes { - fmt.Fprintf(msg, " %s=%s\n", v.Key, v.Value.String()) - } - for _, spanBuffer := range pendingScope.spansByScope { - if spanBuffer.scope != nil { - fmt.Fprintf(msg, " Scope: %s\n", spanBuffer.scope.ID()) - } else { - msg.WriteString(" Scope: (unknown)\n") - } - msg.WriteString(" Spans:\n") - longestName := 0 - for _, span := range spanBuffer.spans { - longestName = max(longestName, len(span.Name)+2) - } - for _, span := range spanBuffer.spans { - spanID, ok := ToSpanID(span.SpanId) - if !ok { - continue - } - traceID, ok := ToTraceID(span.TraceId) - if !ok { - continue - } - parentSpanID, ok := ToSpanID(span.ParentSpanId) - if !ok { - continue - } - _, seenParent := q.debugAllEnqueuedSpans[parentSpanID] - var missing string - if !seenParent { - missing = " [missing]" - } - fmt.Fprintf(msg, " - %-*s (trace: %s | span: %s | parent:%s %s)\n", longestName, - "'"+span.Name+"'", traceID.Value(), spanID, missing, parentSpanID) - for _, attr := range span.Attributes { - if attr.Key == "caller" { - fmt.Fprintf(msg, " => caller: '%s'\n", attr.Value.GetStringValue()) - break - } - } - } - } - } - } - endMsg(msg) - } - - if q.debugFlags.Check(LogTraceIDMappings) || (didWarn && q.debugFlags.Check(LogTraceIDMappingsOnWarn)) { - msg := startMsg("Known trace ids:\n") - keys := q.knownTraceIDMappings.Keys() - values := q.knownTraceIDMappings.Values() - for i, k := range keys { - v := values[i] - if k != v { - if v == zeroTraceID { - fmt.Fprintf(msg, "%s (dropped)\n", k.Value()) - } else { - fmt.Fprintf(msg, "%s => %s\n", k.Value(), v.Value()) - } - } else { - fmt.Fprintf(msg, "%s (no change)\n", k.Value()) - } - } - endMsg(msg) - } - if q.debugFlags.Check(LogAllSpans) || (didWarn && q.debugFlags.Check(LogAllSpansOnWarn)) { - msg := startMsg("All exported spans:\n") - longestName := 0 - for _, span := range q.debugAllEnqueuedSpans { - longestName = max(longestName, len(span.Name)+2) - } - for _, span := range q.debugAllEnqueuedSpans { - spanID, ok := ToSpanID(span.SpanId) - if !ok { - continue - } - traceID, ok := ToTraceID(span.TraceId) - if !ok { - continue - } - parentSpanID, ok := ToSpanID(span.ParentSpanId) - if !ok { - continue - } - fmt.Fprintf(msg, "%-*s (trace: %s | span: %s | parent: %s)", longestName, - "'"+span.Name+"'", traceID.Value(), spanID, parentSpanID) - var foundCaller bool - for _, attr := range span.Attributes { - if attr.Key == "caller" { - fmt.Fprintf(msg, " => %s\n", attr.Value.GetStringValue()) - foundCaller = true - break - } - } - if !foundCaller { - msg.WriteString("\n") - } - } - endMsg(msg) - } - if q.debugFlags.Check(LogAllEvents) { - msg := startMsg("All Events:\n") - msg.WriteByte('[') - for i, event := range q.debugEvents { - msg.WriteString("\n ") - eventData, _ := json.Marshal(event) - msg.Write(eventData) - if i < len(q.debugEvents)-1 { - msg.WriteByte(',') - } else { - msg.WriteString("\n]") - } - } - msg.WriteByte('\n') - endMsg(msg) - } - if incomplete { - return ErrIncompleteTraces - } - return nil -} - -type DebugEvent struct { - Timestamp time.Time `json:"timestamp"` - Request *coltracepb.ExportTraceServiceRequest `json:"request"` -} - -func (e DebugEvent) MarshalJSON() ([]byte, error) { - type debugEvent struct { - Timestamp time.Time `json:"timestamp"` - Request json.RawMessage `json:"request"` - } - reqData, _ := protojson.Marshal(e.Request) - return json.Marshal(debugEvent{ - Timestamp: e.Timestamp, - Request: reqData, - }) -} - -func (e *DebugEvent) UnmarshalJSON(b []byte) error { - type debugEvent struct { - Timestamp time.Time `json:"timestamp"` - Request json.RawMessage `json:"request"` - } - var ev debugEvent - if err := json.Unmarshal(b, &ev); err != nil { - return err - } - e.Timestamp = ev.Timestamp - var msg coltracepb.ExportTraceServiceRequest - if err := protojson.Unmarshal(ev.Request, &msg); err != nil { - return err - } - e.Request = &msg - return nil -} - -const shardCount = 64 - -type ( - shardedSet [shardCount]map[oteltrace.SpanID]struct{} - shardedLocks [shardCount]sync.Mutex -) - -func (s *shardedSet) Range(f func(key oteltrace.SpanID)) { - for i := range shardCount { - for k := range s[i] { - f(k) - } - } -} - -func (s *shardedLocks) LockAll() { - for i := range shardCount { - s[i].Lock() - } -} - -func (s *shardedLocks) UnlockAll() { - for i := range shardCount { - s[i].Unlock() - } -} - -type spanTracker struct { - inflightSpansMu shardedLocks - inflightSpans shardedSet - allSpans sync.Map - debugFlags DebugFlags - observer *spanObserver - shutdownOnce sync.Once -} - -func newSpanTracker(observer *spanObserver, debugFlags DebugFlags) *spanTracker { - st := &spanTracker{ - observer: observer, - debugFlags: debugFlags, - } - for i := range len(st.inflightSpans) { - st.inflightSpans[i] = make(map[oteltrace.SpanID]struct{}) - } - return st -} - -type spanInfo struct { - Name string - SpanContext oteltrace.SpanContext - Parent oteltrace.SpanContext - caller string - startTime time.Time -} - -// ForceFlush implements trace.SpanProcessor. -func (t *spanTracker) ForceFlush(context.Context) error { - return nil -} - -// OnEnd implements trace.SpanProcessor. -func (t *spanTracker) OnEnd(s sdktrace.ReadOnlySpan) { - id := s.SpanContext().SpanID() - bucket := binary.BigEndian.Uint64(id[:]) % shardCount - t.inflightSpansMu[bucket].Lock() - defer t.inflightSpansMu[bucket].Unlock() - delete(t.inflightSpans[bucket], id) -} - -// OnStart implements trace.SpanProcessor. -func (t *spanTracker) OnStart(_ context.Context, s sdktrace.ReadWriteSpan) { - id := s.SpanContext().SpanID() - bucket := binary.BigEndian.Uint64(id[:]) % shardCount - t.inflightSpansMu[bucket].Lock() - defer t.inflightSpansMu[bucket].Unlock() - t.inflightSpans[bucket][id] = struct{}{} - - if t.debugFlags.Check(TrackSpanReferences) { - t.observer.Observe(id) - } - if t.debugFlags.Check(TrackAllSpans) { - var caller string - for _, attr := range s.Attributes() { - if attr.Key == "caller" { - caller = attr.Value.AsString() - break - } - } - t.allSpans.Store(id, &spanInfo{ - Name: s.Name(), - SpanContext: s.SpanContext(), - Parent: s.Parent(), - caller: caller, - startTime: s.StartTime(), - }) - } -} - -// Shutdown implements trace.SpanProcessor. -func (t *spanTracker) Shutdown(_ context.Context) error { - if t.debugFlags == 0 { - return nil - } - didWarn := false - t.shutdownOnce.Do(func() { - if t.debugFlags.Check(WarnOnIncompleteSpans) { - if t.debugFlags.Check(TrackAllSpans) { - incompleteSpans := []*spanInfo{} - t.inflightSpansMu.LockAll() - t.inflightSpans.Range(func(key oteltrace.SpanID) { - if info, ok := t.allSpans.Load(key); ok { - incompleteSpans = append(incompleteSpans, info.(*spanInfo)) - } - }) - t.inflightSpansMu.UnlockAll() - if len(incompleteSpans) > 0 { - didWarn = true - msg := startMsg("WARNING: spans not ended:\n") - longestName := 0 - for _, span := range incompleteSpans { - longestName = max(longestName, len(span.Name)+2) - } - for _, span := range incompleteSpans { - var startedAt string - if span.caller != "" { - startedAt = " | started at: " + span.caller - } - fmt.Fprintf(msg, "%-*s (trace: %s | span: %s | parent: %s%s)\n", longestName, "'"+span.Name+"'", - span.SpanContext.TraceID(), span.SpanContext.SpanID(), span.Parent.SpanID(), startedAt) - } - endMsg(msg) - } - } else { - incompleteSpans := []oteltrace.SpanID{} - t.inflightSpansMu.LockAll() - t.inflightSpans.Range(func(key oteltrace.SpanID) { - incompleteSpans = append(incompleteSpans, key) - }) - t.inflightSpansMu.UnlockAll() - if len(incompleteSpans) > 0 { - didWarn = true - msg := startMsg("WARNING: spans not ended:\n") - for _, span := range incompleteSpans { - fmt.Fprintf(msg, "%s\n", span) - } - msg.WriteString("Note: set TrackAllSpans flag for more info\n") - endMsg(msg) - } - } - } - - if t.debugFlags.Check(LogAllSpans) || (t.debugFlags.Check(LogAllSpansOnWarn) && didWarn) { - allSpans := []*spanInfo{} - t.allSpans.Range(func(_, value any) bool { - allSpans = append(allSpans, value.(*spanInfo)) - return true - }) - slices.SortFunc(allSpans, func(a, b *spanInfo) int { - return a.startTime.Compare(b.startTime) - }) - msg := startMsg("All observed spans:\n") - longestName := 0 - for _, span := range allSpans { - longestName = max(longestName, len(span.Name)+2) - } - for _, span := range allSpans { - var startedAt string - if span.caller != "" { - startedAt = " | started at: " + span.caller - } - fmt.Fprintf(msg, "%-*s (trace: %s | span: %s | parent: %s%s)\n", longestName, "'"+span.Name+"'", - span.SpanContext.TraceID(), span.SpanContext.SpanID(), span.Parent.SpanID(), startedAt) - } - endMsg(msg) - } - }) - if didWarn { - return ErrIncompleteSpans - } - return nil -} - -func newSpanObserver() *spanObserver { - return &spanObserver{ - referencedIDs: map[oteltrace.SpanID]oteltrace.SpanID{}, - cond: sync.NewCond(&sync.Mutex{}), - } -} - -type spanObserver struct { - cond *sync.Cond - referencedIDs map[oteltrace.SpanID]oteltrace.SpanID - unobservedIDs int -} - -func (obs *spanObserver) ObserveReference(id oteltrace.SpanID, via oteltrace.SpanID) { - obs.cond.L.Lock() - defer obs.cond.L.Unlock() - if _, referenced := obs.referencedIDs[id]; !referenced { - obs.referencedIDs[id] = via // referenced, but not observed - // It is possible for new unobserved references to come in while waiting, - // but incrementing the counter wouldn't satisfy the condition so we don't - // need to signal the waiters - obs.unobservedIDs++ - } -} - -func (obs *spanObserver) Observe(id oteltrace.SpanID) { - obs.cond.L.Lock() - defer obs.cond.L.Unlock() - if observed, referenced := obs.referencedIDs[id]; !referenced || observed.IsValid() { // NB: subtle condition - obs.referencedIDs[id] = zeroSpanID - if referenced { - obs.unobservedIDs-- - obs.cond.Broadcast() - } - } -} - -func (obs *spanObserver) wait(debugAllEnqueuedSpans map[oteltrace.SpanID]*tracev1.Span, warnAfter time.Duration) { - done := make(chan struct{}) - defer close(done) - go func() { - select { - case <-done: - return - case <-time.After(warnAfter): - obs.debugWarnWaiting(debugAllEnqueuedSpans) - } - }() - - obs.cond.L.Lock() - for obs.unobservedIDs > 0 { - obs.cond.Wait() - } - obs.cond.L.Unlock() -} - -func (obs *spanObserver) debugWarnWaiting(debugAllEnqueuedSpans map[oteltrace.SpanID]*tracev1.Span) { - obs.cond.L.Lock() - msg := startMsg(fmt.Sprintf("Waiting on %d unobserved spans:\n", obs.unobservedIDs)) - for id, via := range obs.referencedIDs { - if via.IsValid() { - fmt.Fprintf(msg, "%s via %s", id, via) - if span := debugAllEnqueuedSpans[id]; span != nil { - createdAt := "(unknown)" - for _, attr := range span.Attributes { - if attr.Key == "caller" { - createdAt = attr.Value.GetStringValue() - break - } - } - fmt.Fprintf(msg, "'%s' (trace: %s | created: %s)\n", span.GetName(), span.TraceId, createdAt) - } else { - msg.WriteString("\n") - } - } - } - endMsg(msg) - obs.cond.L.Unlock() -} diff --git a/internal/telemetry/trace/queue_test.go b/internal/telemetry/trace/queue_test.go deleted file mode 100644 index d5d98004b..000000000 --- a/internal/telemetry/trace/queue_test.go +++ /dev/null @@ -1,798 +0,0 @@ -package trace_test - -import ( - "bytes" - "context" - "embed" - "fmt" - "io/fs" - "regexp" - "runtime" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/pomerium/pomerium/internal/telemetry/trace" - "github.com/pomerium/pomerium/internal/telemetry/trace/mock_otlptrace" - "github.com/pomerium/pomerium/internal/testutil" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - oteltrace "go.opentelemetry.io/otel/trace" - coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" - commonv1 "go.opentelemetry.io/proto/otlp/common/v1" - tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" - "go.uber.org/mock/gomock" - "google.golang.org/protobuf/encoding/protojson" - "google.golang.org/protobuf/proto" -) - -//go:embed testdata -var testdata embed.FS - -func TestSpanExportQueue_Replay(t *testing.T) { - for _, tc := range []struct { - name string - file string - check func(t testing.TB, inputs, outputs *testutil.TraceResults) - }{ - { - name: "single trace", - file: "testdata/recording_01_single_trace.json", - check: func(t testing.TB, inputs, outputs *testutil.TraceResults) { - inputs.AssertEqual(t, outputs) - }, - }, - { - name: "rewriting multiple traces", - file: "testdata/recording_02_multi_trace.json", - check: func(t testing.TB, inputs, outputs *testutil.TraceResults) { - inputTraces := inputs.GetTraces().WithoutErrors() - outputTraces := outputs.GetTraces().WithoutErrors() - - // find upstream trace - var inputUpstreamTrace, outputUpstreamTrace *testutil.TraceDetails - isUpstreamTrace := func(v *testutil.TraceDetails) bool { - if strings.HasPrefix(v.Name, "Envoy: ingress:") { - for _, attr := range v.Spans[0].Raw.Attributes { - if attr.Key == "http.url" { - if regexp.MustCompile(`https://127\.0\.0\.1:\d+/foo`).MatchString(attr.Value.GetStringValue()) { - return true - } - } - } - } - return false - } - for _, v := range inputTraces.ByID { - if isUpstreamTrace(v) { - inputUpstreamTrace = v - break - } - } - for _, v := range outputTraces.ByID { - if isUpstreamTrace(v) { - outputUpstreamTrace = v - break - } - } - equal, diff := inputUpstreamTrace.Equal(outputUpstreamTrace) - if !equal { - assert.Failf(t, "upstream traces not equal:\n%s", diff) - return - } - - // find downstream traces - // should be composed of: - // - 'ingress: GET foo.localhost.pomerium.io:/foo' - // - 'internal: GET authenticate.localhost.pomerium.io:/.pomerium/sign_in' (unauthorized) - // - 'internal: GET authenticate.localhost.pomerium.io:/oauth2/callback' - // - 'internal: GET authenticate.localhost.pomerium.io:/.pomerium/sign_in' (authorized) - // - 'internal: GET foo.localhost.pomerium.io:/.pomerium/callback/' - envoyOutputTraces := outputTraces.ByParticipant["Envoy"] - // there should be two - require.Len(t, envoyOutputTraces, 2) - // find which one is not the upstream trace - var downstreamTrace *testutil.TraceDetails - if envoyOutputTraces[0].ID == outputUpstreamTrace.ID { - downstreamTrace = envoyOutputTraces[1] - } else { - downstreamTrace = envoyOutputTraces[0] - } - tree := downstreamTrace.SpanTree() - require.Empty(t, tree.DetachedParents) - parts := tree.Root.Children - require.Len(t, parts, 5) - assert.True(t, regexp.MustCompile(`ingress: GET foo\.localhost\.pomerium\.io:\d+/foo`).MatchString(parts[0].Span.Raw.Name)) - assert.True(t, regexp.MustCompile(`internal: GET authenticate\.localhost\.pomerium\.io:\d+/.pomerium/sign_in`).MatchString(parts[1].Span.Raw.Name)) - assert.True(t, regexp.MustCompile(`internal: GET authenticate\.localhost\.pomerium\.io:\d+/oauth2/callback`).MatchString(parts[2].Span.Raw.Name)) - assert.True(t, regexp.MustCompile(`internal: GET authenticate\.localhost\.pomerium\.io:\d+/.pomerium/sign_in`).MatchString(parts[3].Span.Raw.Name)) - assert.True(t, regexp.MustCompile(`internal: GET foo\.localhost\.pomerium\.io:\d+/.pomerium/callback`).MatchString(parts[4].Span.Raw.Name)) - }, - }, - } { - t.Run(tc.name, func(t *testing.T) { - ctrl := gomock.NewController(t) - mockClient := mock_otlptrace.NewMockClient(ctrl) - var resultsMu sync.Mutex - outputSpans := [][]*tracev1.ResourceSpans{} - mockClient.EXPECT(). - UploadTraces(gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, protoSpans []*tracev1.ResourceSpans) error { - resultsMu.Lock() - defer resultsMu.Unlock() - outputSpans = append(outputSpans, protoSpans) - return nil - }). - AnyTimes() - recording1, err := fs.ReadFile(testdata, tc.file) - require.NoError(t, err) - - rec, err := testutil.LoadEventRecording(recording1) - require.NoError(t, err) - - ctx := trace.Options{ - DebugFlags: trace.TrackAllSpans | trace.WarnOnIncompleteSpans | trace.WarnOnIncompleteTraces | trace.WarnOnUnresolvedReferences, - }.NewContext(context.Background()) - queue := trace.NewSpanExportQueue(ctx, mockClient) - - recCloned := rec.Clone() - - err = rec.Replay(func(ctx context.Context, req *coltracepb.ExportTraceServiceRequest) (*coltracepb.ExportTraceServiceResponse, error) { - return &coltracepb.ExportTraceServiceResponse{}, queue.Enqueue(ctx, req) - }) - assert.NoError(t, err) - - // wait for all calls to UploadTraces to complete - ctx, ca := context.WithTimeout(context.Background(), 1*time.Second) - defer ca() - assert.NoError(t, queue.Close(ctx)) - - recCloned.Normalize(rec.NormalizedTo()) - - inputRequests := []*coltracepb.ExportTraceServiceRequest{} - for _, ev := range recCloned.Events() { - inputRequests = append(inputRequests, ev.Request) - } - inputs := testutil.NewTraceResults(testutil.FlattenExportRequests(inputRequests)) - outputs := testutil.NewTraceResults(testutil.FlattenResourceSpans(outputSpans)) - tc.check(t, inputs, outputs) - }) - } -} - -func TestSpanExportQueue_Enqueue(t *testing.T) { - type ( - mapped struct { - s Span - t Trace - } - action struct { - exports []Span - uploads []any // int|mapped|*tracev1.Span - } - testCase struct { - name string - spans []*tracev1.Span // note: span ids are set automatically by index - actions []action - // if actionSets is present, repeats the same test case for each entry - actionSets [][]action - } - ) - - traceparent := func(trace Trace, span Span, sampled ...bool) *commonv1.KeyValue { - if len(sampled) == 0 { - sampled = append(sampled, true) - } - return &commonv1.KeyValue{ - Key: "pomerium.traceparent", - Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_StringValue{ - StringValue: Traceparent(trace, span, sampled[0]), - }}, - } - } - externalParent := func(span Span) *commonv1.KeyValue { - return &commonv1.KeyValue{ - Key: "pomerium.external-parent-span", - Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_StringValue{ - StringValue: span.ID().String(), - }}, - } - } - attrs := func(kvs ...*commonv1.KeyValue) []*commonv1.KeyValue { return kvs } - - cases := []testCase{ - { - name: "single trace", - spans: []*tracev1.Span{ - // |<========>| Span 1 - // | <======> | Span 2 - // | <====> | Span 3 - // T123456789A- - Span(1): { - TraceId: Trace(1).B(), - ParentSpanId: nil, - StartTimeUnixNano: 1, - EndTimeUnixNano: 0xA, - }, - Span(2): { - TraceId: Trace(1).B(), - ParentSpanId: Span(1).B(), - StartTimeUnixNano: 2, - EndTimeUnixNano: 9, - }, - Span(3): { - TraceId: Trace(1).B(), - ParentSpanId: Span(2).B(), - StartTimeUnixNano: 3, - EndTimeUnixNano: 8, - }, - }, - actionSets: [][]action{ - // root span first - { - {exports: []Span{1}, uploads: []any{1}}, - {exports: []Span{2, 3}, uploads: []any{2, 3}}, - }, - { - {exports: []Span{1, 2}, uploads: []any{1, 2}}, - {exports: []Span{3}, uploads: []any{3}}, - }, - { - {exports: []Span{1, 2, 3}, uploads: []any{1, 2, 3}}, - }, - { - {exports: []Span{1, 3, 2}, uploads: []any{1, 2, 3}}, - }, - { - {exports: []Span{1}, uploads: []any{1}}, - {exports: []Span{2}, uploads: []any{2}}, - {exports: []Span{3}, uploads: []any{3}}, - }, - { - {exports: []Span{1}, uploads: []any{1}}, - {exports: []Span{3}, uploads: []any{3}}, - {exports: []Span{2}, uploads: []any{2}}, - }, - // root span last - { - {exports: []Span{2}, uploads: []any{}}, - {exports: []Span{3}, uploads: []any{}}, - {exports: []Span{1}, uploads: []any{1, 2, 3}}, - }, - { - {exports: []Span{3}, uploads: []any{}}, - {exports: []Span{2}, uploads: []any{}}, - {exports: []Span{1}, uploads: []any{1, 2, 3}}, - }, - { - {exports: []Span{2, 3}, uploads: []any{}}, - {exports: []Span{1}, uploads: []any{1, 2, 3}}, - }, - { - {exports: []Span{3, 2}, uploads: []any{}}, - {exports: []Span{1}, uploads: []any{1, 2, 3}}, - }, - { - {exports: []Span{3}, uploads: []any{}}, - {exports: []Span{2, 1}, uploads: []any{1, 2, 3}}, - }, - { - {exports: []Span{2, 3, 1}, uploads: []any{1, 2, 3}}, - }, - // root span in the middle - { - {exports: []Span{2}, uploads: []any{}}, - {exports: []Span{1}, uploads: []any{1, 2}}, - {exports: []Span{3}, uploads: []any{3}}, - }, - { - {exports: []Span{3}, uploads: []any{}}, - {exports: []Span{1}, uploads: []any{1, 3}}, - {exports: []Span{2}, uploads: []any{2}}, - }, - { - {exports: []Span{3}, uploads: []any{}}, - {exports: []Span{1, 2}, uploads: []any{1, 2, 3}}, - }, - { - {exports: []Span{2}, uploads: []any{}}, - {exports: []Span{1, 3}, uploads: []any{1, 2, 3}}, - }, - }, - }, - { - name: "two correlated traces", - spans: []*tracev1.Span{ - // |<=====> | Span 1 (Trace 1) - // | <===> | Span 2 (Trace 1) - // | <=> | Span 3 (Trace 1) - // | <======>| Span 4 (Trace 2) - // | <====> | Span 5 (Trace 2) - // T123456789ABCDEF- - Span(1): { - TraceId: Trace(1).B(), - ParentSpanId: nil, - StartTimeUnixNano: 1, - EndTimeUnixNano: 7, - }, - Span(2): { - TraceId: Trace(1).B(), - ParentSpanId: Span(1).B(), - StartTimeUnixNano: 2, - EndTimeUnixNano: 6, - }, - Span(3): { - TraceId: Trace(1).B(), - ParentSpanId: Span(2).B(), - StartTimeUnixNano: 3, - EndTimeUnixNano: 5, - }, - Span(4): { - TraceId: Trace(2).B(), - ParentSpanId: nil, - Attributes: attrs(traceparent(Trace(1), Span(1))), - StartTimeUnixNano: 8, - EndTimeUnixNano: 0xF, - }, - Span(5): { - TraceId: Trace(2).B(), - ParentSpanId: Span(4).B(), - Attributes: attrs(traceparent(Trace(1), Span(1))), - StartTimeUnixNano: 9, - EndTimeUnixNano: 0xE, - }, - }, - actionSets: [][]action{ - 0: { - { - exports: []Span{1, 2, 3, 4, 5}, - uploads: []any{1, 2, 3, mapped{4, Trace(1)}, mapped{5, Trace(1)}}, - }, - }, - 1: { - {exports: []Span{2, 3, 5}, uploads: []any{}}, - { - exports: []Span{1, 4}, - uploads: []any{1, 2, 3, mapped{4, Trace(1)}, mapped{5, Trace(1)}}, - }, - }, - 2: { - {exports: []Span{2, 3, 5}, uploads: []any{}}, - { - exports: []Span{1}, - uploads: []any{1, 2, 3}, - }, - { - exports: []Span{4}, - uploads: []any{mapped{4, Trace(1)}, mapped{5, Trace(1)}}, - }, - }, - 3: { - {exports: []Span{2, 3, 5}, uploads: []any{}}, - { - exports: []Span{4, 1}, - uploads: []any{1, 2, 3, mapped{4, Trace(1)}, mapped{5, Trace(1)}}, - }, - }, - 4: { - {exports: []Span{2, 3, 5}, uploads: []any{}}, - {exports: []Span{4}, uploads: []any{}}, // root span, but mapped to a pending trace - { - exports: []Span{1}, - uploads: []any{1, 2, 3, mapped{4, Trace(1)}, mapped{5, Trace(1)}}, - }, - }, - }, - }, - { - name: "external parent", - spans: []*tracev1.Span{ - // |??????????| Span 1 (external) - // | <======> | Span 2 (internal) - // | <====> | Span 3 - // T123456789A- - Span(2): { - TraceId: Trace(1).B(), - ParentSpanId: Span(1).B(), - StartTimeUnixNano: 2, - EndTimeUnixNano: 9, - Attributes: attrs(externalParent(Span(1))), - }, - Span(3): { - TraceId: Trace(1).B(), - ParentSpanId: Span(2).B(), - StartTimeUnixNano: 3, - EndTimeUnixNano: 8, - }, - }, - actionSets: [][]action{ - { - {exports: []Span{3}, uploads: []any{}}, - {exports: []Span{2}, uploads: []any{2, 3}}, - }, - { - {exports: []Span{2, 3}, uploads: []any{2, 3}}, - }, - { - {exports: []Span{3, 2}, uploads: []any{3, 2}}, - }, - }, - }, - } - - generatedCases := []testCase{} - for _, tc := range cases { - for i, s := range tc.spans { - if s == nil { - continue - } - s.SpanId = Span(i).B() - } - if len(tc.actionSets) > 0 { - generated := []testCase{} - for i, actions := range tc.actionSets { - generated = append(generated, testCase{ - name: fmt.Sprintf("%s (action set %d of %d)", tc.name, i+1, len(tc.actionSets)), - spans: tc.spans, - actions: actions, - }) - } - generatedCases = append(generatedCases, generated...) - } else { - generatedCases = append(generatedCases, tc) - } - } - for _, tc := range generatedCases { - t.Run(tc.name, func(t *testing.T) { - ctrl := gomock.NewController(t) - mockClient := mock_otlptrace.NewMockClient(ctrl) - var resultsMu sync.Mutex - outputSpans := make(chan []*tracev1.ResourceSpans, 64) - mockClient.EXPECT(). - UploadTraces(gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, protoSpans []*tracev1.ResourceSpans) error { - resultsMu.Lock() - defer resultsMu.Unlock() - outputSpans <- protoSpans - return nil - }). - AnyTimes() - - ctx := trace.Options{ - DebugFlags: trace.TrackAllSpans | trace.WarnOnIncompleteSpans | trace.WarnOnIncompleteTraces | trace.WarnOnUnresolvedReferences, - }.NewContext(context.Background()) - queue := trace.NewSpanExportQueue(ctx, mockClient) - - for actionIdx, action := range tc.actions { - spans := []*tracev1.Span{} - for _, idx := range action.exports { - spans = append(spans, proto.Clone(tc.spans[idx]).(*tracev1.Span)) - } - assert.NoError(t, queue.Enqueue(ctx, &coltracepb.ExportTraceServiceRequest{ - ResourceSpans: []*tracev1.ResourceSpans{ - { - Resource: Resource(1).Make().Resource, - ScopeSpans: []*tracev1.ScopeSpans{{Scope: Scope(1).Make().Scope, Spans: spans}}, - }, - }, - })) - if len(action.uploads) == 0 { - for range 5 { - runtime.Gosched() - require.Empty(t, outputSpans) - } - continue - } - expectedSpans := &tracev1.ResourceSpans{ - Resource: Resource(1).Make().Resource, - ScopeSpans: []*tracev1.ScopeSpans{{Scope: Scope(1).Make().Scope}}, - } - for _, expectedUpload := range action.uploads { - switch up := expectedUpload.(type) { - case int: - expectedSpans.ScopeSpans[0].Spans = append(expectedSpans.ScopeSpans[0].Spans, tc.spans[up]) - case mapped: - clone := proto.Clone(tc.spans[up.s]).(*tracev1.Span) - clone.TraceId = up.t.B() - expectedSpans.ScopeSpans[0].Spans = append(expectedSpans.ScopeSpans[0].Spans, clone) - case *tracev1.Span: - expectedSpans.ScopeSpans[0].Spans = append(expectedSpans.ScopeSpans[0].Spans, up) - default: - panic(fmt.Sprintf("test bug: unexpected type: %T", up)) - } - } - select { - case resourceSpans := <-outputSpans: - expected := testutil.NewTraceResults([]*tracev1.ResourceSpans{expectedSpans}) - actual := testutil.NewTraceResults(resourceSpans) - actual.AssertEqual(t, expected, "action %d/%d", actionIdx+1, len(tc.actions)) - case <-time.After(1 * time.Second): - t.Fatalf("timed out waiting for upload (action %d/%d)", actionIdx+1, len(tc.actions)) - } - } - if !t.Failed() { - close(outputSpans) - // ensure the queue is read fully - if !assert.Empty(t, outputSpans) { - for _, out := range <-outputSpans { - t.Log(protojson.Format(out)) - } - } - } - }) - } -} - -func TestSpanObserver(t *testing.T) { - t.Run("observe single reference", func(t *testing.T) { - obs := trace.NewSpanObserver() - assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) - - obs.ObserveReference(Span(1).ID(), Span(2).ID()) - assert.Equal(t, []oteltrace.SpanID{Span(1).ID()}, obs.XUnobservedIDs()) - obs.Observe(Span(1).ID()) - assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) - }) - t.Run("observe multiple references", func(t *testing.T) { - obs := trace.NewSpanObserver() - - obs.ObserveReference(Span(1).ID(), Span(2).ID()) - obs.ObserveReference(Span(1).ID(), Span(3).ID()) - obs.ObserveReference(Span(1).ID(), Span(4).ID()) - assert.Equal(t, []oteltrace.SpanID{Span(1).ID()}, obs.XUnobservedIDs()) - obs.Observe(Span(1).ID()) - assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) - }) - t.Run("observe before reference", func(t *testing.T) { - obs := trace.NewSpanObserver() - - obs.Observe(Span(1).ID()) - assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) - obs.ObserveReference(Span(1).ID(), Span(2).ID()) - assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) - }) - - t.Run("wait", func(t *testing.T) { - obs := trace.NewSpanObserver() - obs.ObserveReference(Span(1).ID(), Span(2).ID()) - obs.Observe(Span(2).ID()) - obs.ObserveReference(Span(3).ID(), Span(4).ID()) - obs.Observe(Span(4).ID()) - obs.ObserveReference(Span(5).ID(), Span(6).ID()) - obs.Observe(Span(6).ID()) - waitOkToExit := atomic.Bool{} - waitExited := atomic.Bool{} - go func() { - defer waitExited.Store(true) - obs.XWait() - assert.True(t, waitOkToExit.Load(), "wait exited early") - }() - - time.Sleep(10 * time.Millisecond) - assert.False(t, waitExited.Load()) - - obs.Observe(Span(1).ID()) - time.Sleep(10 * time.Millisecond) - assert.False(t, waitExited.Load()) - - obs.Observe(Span(3).ID()) - time.Sleep(10 * time.Millisecond) - assert.False(t, waitExited.Load()) - - waitOkToExit.Store(true) - obs.Observe(Span(5).ID()) - assert.Eventually(t, waitExited.Load, 10*time.Millisecond, 1*time.Millisecond) - }) - - t.Run("new references observed during wait", func(t *testing.T) { - obs := trace.NewSpanObserver() - obs.ObserveReference(Span(1).ID(), Span(2).ID()) - obs.Observe(Span(2).ID()) - obs.ObserveReference(Span(3).ID(), Span(4).ID()) - obs.Observe(Span(4).ID()) - obs.ObserveReference(Span(5).ID(), Span(6).ID()) - obs.Observe(Span(6).ID()) - waitOkToExit := atomic.Bool{} - waitExited := atomic.Bool{} - go func() { - defer waitExited.Store(true) - obs.XWait() - assert.True(t, waitOkToExit.Load(), "wait exited early") - }() - - assert.Equal(t, []oteltrace.SpanID{Span(1).ID(), Span(3).ID(), Span(5).ID()}, obs.XUnobservedIDs()) - time.Sleep(10 * time.Millisecond) - assert.False(t, waitExited.Load()) - - obs.Observe(Span(1).ID()) - assert.Equal(t, []oteltrace.SpanID{Span(3).ID(), Span(5).ID()}, obs.XUnobservedIDs()) - time.Sleep(10 * time.Millisecond) - assert.False(t, waitExited.Load()) - - obs.Observe(Span(3).ID()) - assert.Equal(t, []oteltrace.SpanID{Span(5).ID()}, obs.XUnobservedIDs()) - time.Sleep(10 * time.Millisecond) - assert.False(t, waitExited.Load()) - - // observe a new reference - obs.ObserveReference(Span(7).ID(), Span(8).ID()) - obs.Observe(Span(8).ID()) - assert.Equal(t, []oteltrace.SpanID{Span(5).ID(), Span(7).ID()}, obs.XUnobservedIDs()) - time.Sleep(10 * time.Millisecond) - assert.False(t, waitExited.Load()) - - obs.Observe(Span(5).ID()) - assert.Equal(t, []oteltrace.SpanID{Span(7).ID()}, obs.XUnobservedIDs()) - time.Sleep(10 * time.Millisecond) - assert.False(t, waitExited.Load()) - - waitOkToExit.Store(true) - obs.Observe(Span(7).ID()) - assert.Equal(t, []oteltrace.SpanID{}, obs.XUnobservedIDs()) - assert.Eventually(t, waitExited.Load, 10*time.Millisecond, 1*time.Millisecond) - }) - - t.Run("multiple waiters", func(t *testing.T) { - t.Parallel() - obs := trace.NewSpanObserver() - obs.ObserveReference(Span(1).ID(), Span(2).ID()) - obs.Observe(Span(2).ID()) - - waitersExited := atomic.Int32{} - for range 10 { - go func() { - defer waitersExited.Add(1) - obs.XWait() - }() - } - - assert.Equal(t, []oteltrace.SpanID{Span(1).ID()}, obs.XUnobservedIDs()) - time.Sleep(10 * time.Millisecond) - assert.Equal(t, int32(0), waitersExited.Load()) - - obs.Observe(Span(1).ID()) - - startTime := time.Now() - for waitersExited.Load() != 10 { - if time.Since(startTime) > 1*time.Millisecond { - t.Fatal("timed out") - } - runtime.Gosched() - } - }) -} - -func TestSpanTracker(t *testing.T) { - t.Run("no debug flags", func(t *testing.T) { - t.Parallel() - obs := trace.NewSpanObserver() - tracker := trace.NewSpanTracker(obs, 0) - tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracker)) - tracer := tp.Tracer("test") - assert.Equal(t, []oteltrace.SpanID{}, tracker.XInflightSpans()) - _, span1 := tracer.Start(context.Background(), "span 1") - assert.Equal(t, []oteltrace.SpanID{span1.SpanContext().SpanID()}, tracker.XInflightSpans()) - assert.Equal(t, []oteltrace.SpanID{}, obs.XObservedIDs()) - span1.End() - assert.Equal(t, []oteltrace.SpanID{}, tracker.XInflightSpans()) - assert.Equal(t, []oteltrace.SpanID{}, obs.XObservedIDs()) - }) - t.Run("with TrackSpanReferences debug flag", func(t *testing.T) { - t.Parallel() - obs := trace.NewSpanObserver() - tracker := trace.NewSpanTracker(obs, trace.TrackSpanReferences) - tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracker)) - tracer := tp.Tracer("test") - assert.Equal(t, []oteltrace.SpanID{}, tracker.XInflightSpans()) - _, span1 := tracer.Start(context.Background(), "span 1") - assert.Equal(t, []oteltrace.SpanID{span1.SpanContext().SpanID()}, tracker.XInflightSpans()) - assert.Equal(t, []oteltrace.SpanID{span1.SpanContext().SpanID()}, obs.XObservedIDs()) - span1.End() - assert.Equal(t, []oteltrace.SpanID{}, tracker.XInflightSpans()) - assert.Equal(t, []oteltrace.SpanID{span1.SpanContext().SpanID()}, obs.XObservedIDs()) - }) -} - -func TestSpanTrackerWarnings(t *testing.T) { - t.Run("WarnOnIncompleteSpans", func(t *testing.T) { - var buf bytes.Buffer - trace.SetDebugMessageWriterForTest(t, &buf) - - obs := trace.NewSpanObserver() - tracker := trace.NewSpanTracker(obs, trace.WarnOnIncompleteSpans) - tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracker)) - tracer := tp.Tracer("test") - _, span1 := tracer.Start(context.Background(), "span 1") - - assert.ErrorIs(t, tp.Shutdown(context.Background()), trace.ErrIncompleteSpans) - - assert.Equal(t, fmt.Sprintf(` -================================================== -WARNING: spans not ended: -%s -Note: set TrackAllSpans flag for more info -================================================== -`, span1.SpanContext().SpanID()), buf.String()) - }) - - t.Run("WarnOnIncompleteSpans with TrackAllSpans", func(t *testing.T) { - var buf bytes.Buffer - trace.SetDebugMessageWriterForTest(t, &buf) - - obs := trace.NewSpanObserver() - tracker := trace.NewSpanTracker(obs, trace.WarnOnIncompleteSpans|trace.TrackAllSpans) - tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(tracker)) - tracer := tp.Tracer("test") - _, span1 := tracer.Start(context.Background(), "span 1") - - assert.ErrorIs(t, tp.Shutdown(context.Background()), trace.ErrIncompleteSpans) - - assert.Equal(t, fmt.Sprintf(` -================================================== -WARNING: spans not ended: -'span 1' (trace: %s | span: %s | parent: 0000000000000000) -================================================== -`, span1.SpanContext().TraceID(), span1.SpanContext().SpanID()), buf.String()) - }) - - t.Run("WarnOnIncompleteSpans with TrackAllSpans and stackTraceProcessor", func(t *testing.T) { - var buf bytes.Buffer - trace.SetDebugMessageWriterForTest(t, &buf) - - obs := trace.NewSpanObserver() - tracker := trace.NewSpanTracker(obs, trace.WarnOnIncompleteSpans|trace.TrackAllSpans) - tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(&trace.XStackTraceProcessor{}), sdktrace.WithSpanProcessor(tracker)) - tracer := tp.Tracer("test") - _, span1 := tracer.Start(context.Background(), "span 1") - _, file, line, _ := runtime.Caller(0) - line-- - - assert.ErrorIs(t, tp.Shutdown(context.Background()), trace.ErrIncompleteSpans) - - assert.Equal(t, fmt.Sprintf(` -================================================== -WARNING: spans not ended: -'span 1' (trace: %s | span: %s | parent: 0000000000000000 | started at: %s:%d) -================================================== -`, span1.SpanContext().TraceID(), span1.SpanContext().SpanID(), file, line), buf.String()) - }) - - t.Run("LogAllSpansOnWarn", func(t *testing.T) { - var buf bytes.Buffer - trace.SetDebugMessageWriterForTest(t, &buf) - - obs := trace.NewSpanObserver() - tracker := trace.NewSpanTracker(obs, trace.WarnOnIncompleteSpans|trace.TrackAllSpans|trace.LogAllSpansOnWarn) - tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(&trace.XStackTraceProcessor{}), sdktrace.WithSpanProcessor(tracker)) - tracer := tp.Tracer("test") - _, span1 := tracer.Start(context.Background(), "span 1") - time.Sleep(10 * time.Millisecond) - span1.End() - time.Sleep(10 * time.Millisecond) - _, span2 := tracer.Start(context.Background(), "span 2") - _, file, line, _ := runtime.Caller(0) - line-- - - tp.Shutdown(context.Background()) - - assert.Equal(t, - fmt.Sprintf(` -================================================== -WARNING: spans not ended: -'span 2' (trace: %[1]s | span: %[2]s | parent: 0000000000000000 | started at: %[3]s:%[4]d) -================================================== - -================================================== -All observed spans: -'span 1' (trace: %[5]s | span: %[6]s | parent: 0000000000000000 | started at: %[3]s:%[7]d) -'span 2' (trace: %[1]s | span: %[2]s | parent: 0000000000000000 | started at: %[3]s:%[4]d) -================================================== -`, - span2.SpanContext().TraceID(), span2.SpanContext().SpanID(), file, line, - span1.SpanContext().TraceID(), span1.SpanContext().SpanID(), line-4, - ), buf.String()) - }) -} diff --git a/internal/telemetry/trace/server.go b/internal/telemetry/trace/server.go index 204b8b41d..0384a0a53 100644 --- a/internal/telemetry/trace/server.go +++ b/internal/telemetry/trace/server.go @@ -6,19 +6,37 @@ import ( "net" "time" + "github.com/pomerium/pomerium/internal/log" coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" "google.golang.org/grpc/test/bufconn" "go.opentelemetry.io/otel/exporters/otlp/otlptrace" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" - sdktrace "go.opentelemetry.io/otel/sdk/trace" ) +const localExporterMetadataKey = "x-local-exporter" + // Export implements ptraceotlp.GRPCServer. func (srv *ExporterServer) Export(ctx context.Context, req *coltracepb.ExportTraceServiceRequest) (*coltracepb.ExportTraceServiceResponse, error) { - if err := srv.spanExportQueue.Enqueue(ctx, req); err != nil { + if srv.observer != nil { + isLocal := len(metadata.ValueFromIncomingContext(ctx, localExporterMetadataKey)) != 0 + if !isLocal { + for _, res := range req.ResourceSpans { + for _, scope := range res.ScopeSpans { + for _, span := range scope.Spans { + if id, ok := ToSpanID(span.SpanId); ok { + srv.observer.Observe(id) + } + } + } + } + } + } + if err := srv.remoteClient.UploadTraces(ctx, req.GetResourceSpans()); err != nil { + log.Ctx(ctx).Err(err).Msg("error uploading traces") return nil, err } return &coltracepb.ExportTraceServiceResponse{}, nil @@ -26,17 +44,18 @@ func (srv *ExporterServer) Export(ctx context.Context, req *coltracepb.ExportTra type ExporterServer struct { coltracepb.UnimplementedTraceServiceServer - spanExportQueue *SpanExportQueue - server *grpc.Server - remoteClient otlptrace.Client - cc *grpc.ClientConn + server *grpc.Server + observer *spanObserver + remoteClient otlptrace.Client + cc *grpc.ClientConn } -func NewServer(ctx context.Context, remoteClient otlptrace.Client) *ExporterServer { +func NewServer(ctx context.Context) *ExporterServer { + sys := systemContextFromContext(ctx) ex := &ExporterServer{ - spanExportQueue: NewSpanExportQueue(ctx, remoteClient), - remoteClient: remoteClient, - server: grpc.NewServer(grpc.Creds(insecure.NewCredentials())), + remoteClient: sys.options.RemoteClient, + observer: sys.observer, + server: grpc.NewServer(grpc.Creds(insecure.NewCredentials())), } coltracepb.RegisterTraceServiceServer(ex.server, ex) return ex @@ -64,13 +83,12 @@ func (srv *ExporterServer) NewClient() otlptrace.Client { return otlptracegrpc.NewClient( otlptracegrpc.WithGRPCConn(srv.cc), otlptracegrpc.WithTimeout(1*time.Minute), + otlptracegrpc.WithHeaders(map[string]string{ + localExporterMetadataKey: "1", + }), ) } -func (srv *ExporterServer) SpanProcessors() []sdktrace.SpanProcessor { - return []sdktrace.SpanProcessor{srv.spanExportQueue.tracker} -} - func (srv *ExporterServer) Shutdown(ctx context.Context) error { stopped := make(chan struct{}) go func() { @@ -83,10 +101,7 @@ func (srv *ExporterServer) Shutdown(ctx context.Context) error { return context.Cause(ctx) } var errs []error - if err := srv.spanExportQueue.WaitForSpans(30 * time.Second); err != nil { - errs = append(errs, err) - } - if err := srv.spanExportQueue.Close(ctx); err != nil { + if err := WaitForSpans(ctx, 30*time.Second); err != nil { errs = append(errs, err) } if err := srv.remoteClient.Stop(ctx); err != nil { diff --git a/internal/telemetry/trace/testdata/README.md b/internal/telemetry/trace/testdata/README.md deleted file mode 100644 index 46837ef8a..000000000 --- a/internal/telemetry/trace/testdata/README.md +++ /dev/null @@ -1,9 +0,0 @@ -These trace recordings are generated as follows: - -- recording_01_single_trace.json: - -`go test -v -run "^TestOTLPTracing$" -env.trace-debug-flags=+32 github.com/pomerium/pomerium/internal/testenv/selftests | grep -ozP "(?s)(?<=All Events:\n).*?(?=\n=====)"` - -- recording_02_multi_trace.json: - -`go test -v -run "^TestOTLPTracing_TraceCorrelation$" -env.trace-debug-flags=+32 github.com/pomerium/pomerium/internal/testenv/selftests | grep -ozP "(?s)(?<=All Events:\n).*?(?=\n=====)"` diff --git a/internal/telemetry/trace/testdata/recording_01_single_trace.json b/internal/telemetry/trace/testdata/recording_01_single_trace.json deleted file mode 100644 index baa0dbd9d..000000000 --- a/internal/telemetry/trace/testdata/recording_01_single_trace.json +++ /dev/null @@ -1,38 +0,0 @@ -[ - {"timestamp":"2024-12-27T21:30:25.813350216Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"RaoMfsRKjao=","parentSpanId":"5f1kYvXF3VA=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025808414000","endTimeUnixNano":"1735335025809828000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"dq09RB/npOA=","parentSpanId":"5f1kYvXF3VA=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025809683000","endTimeUnixNano":"1735335025812488000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"5f1kYvXF3VA=","parentSpanId":"kNePRE2AarU=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025808050000","endTimeUnixNano":"1735335025812520000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"b78a9ab1-d815-4a6d-b6a5-e473f7eeeb6b"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=Z1Fi-CygBcWOsGaemJ9Kjck8J5CFO2-Q0NaLJ_090KA%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-09f757396858b657-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-ea7902624e0bf002-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"891"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-ea7902624e0bf002-01"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-09f757396858b657-01"}}]}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.819585151Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"tjexoDV7/Hw=","parentSpanId":"CHEWQLz/Esg=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025815586000","endTimeUnixNano":"1735335025816391000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"dMDfBRWWNfg=","parentSpanId":"CHEWQLz/Esg=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025816362000","endTimeUnixNano":"1735335025819200000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"CHEWQLz/Esg=","parentSpanId":"M+elTBSUJY8=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025815369000","endTimeUnixNano":"1735335025819217000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"58a6db9c-74a6-9afc-b6a1-1ad0761df706"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=bVhDeVhqYXBxRzNLS0F5dm1LVXRzUTZrUHVtSDgrcjdhQSswdmZsT3o2eE81OGJsaEgvOEY4UUVZbFV6em95elFySUNmeFJYelJ4Y0RkRWErUVdtWnc9PXwxNzM1MzM1MDI1fFr-RTRGNqnLn2IMEbk7FiyJdpSLrfG2toEU04qhst7IcWtWqDDKpQf-rvb4AOVFEyb_DCJImrMZ_JKA0yiP0S_U54-vwXFi6BfICA4m3NAFaFycuWP-DbKAQAfk9Uba9ichigqepuu4l8w3EGZwlpOR_H98gkVp8i0wUsRSi0QvJExUdEYP2R6hvWCaEF24kJZGzEgc3K8tFLuKKXF51oZzS8uTXWxZKboeoN_QAE7OJ5s2oD_oUJeQW1mU7W5l1lslB214Zv24ft7fLAbVQYJBd14DDoHfg4WZpQE8v3qIuCCMNcgCod69B1Fa-h4Glz138FEfNbnOgFjiCkW63brqiutwbtIn_Voy034oAAK95qCQvwc0B5RCf2peV7-rb3IrZk6h_ABEeNeRZEkKZtsixVns0ew7rEzfUlcPFre-3D7RK0HfV66arIKuNr4qnMLBuIiDO9PuucWyY4Rft3VAp8Onv-ENIWwLVkrOGvnDFtqubfCnutmW1cqXzF1oVuckiEOwmG0k0vUCk7xR8LLEkNBkh3s3yF3F8LvZLlfesQIl-dNXAWWKIonfCyb_fLiciYxlROetfJXFkMr6LKvgCUHB4AAkx_X--KpHabCpBVnFrEIPv2E59D-eLo7J9Sf4Yhd6nDzdAgj5B4BgDPKSLWvAvjRe5-kD2AT2AGcG-pIBVuKZoI7avNO0GJM23k0-b6_djF13Cg8%3D"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"659"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"33e7a54c1494258f"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.823365016Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"8Z+UA49FUMc=","parentSpanId":"6nkCYk4L8AI=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025800352000","endTimeUnixNano":"1735335025804479000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"1ec195fd-430d-9427-9716-caddcb178407"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"1450"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-ea7902624e0bf002-01"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-ea7902624e0bf002-01"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"ea7902624e0bf002"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"P2zcRg+jsFU=","parentSpanId":"8Z+UA49FUMc=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025800818000","endTimeUnixNano":"1735335025804528000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_unauthorized"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"cnJtNlqDufs=","parentSpanId":"uV3raBlyRYI=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025820836000","endTimeUnixNano":"1735335025822775000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"adb206d8-f691-983a-a797-4e35bd587cea"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"1450"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-b95deb6819724582-01"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-b95deb6819724582-01"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"b95deb6819724582"}}]}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.826015448Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"d+7biQunPZg=","parentSpanId":"lJrLEsDUNLM=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025823671000","endTimeUnixNano":"1735335025824511000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"K5Za89EMcFE=","parentSpanId":"lJrLEsDUNLM=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025824482000","endTimeUnixNano":"1735335025825573000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"lJrLEsDUNLM=","parentSpanId":"I9tnFqm43jY=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025823485000","endTimeUnixNano":"1735335025825591000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"88f0b537-f3c1-4574-bfb1-4e9d3f07ec4f"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=PUoMiJkZMinjfOP-kixkfrqo8r54-oHR-9Z-N1_VU5I%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-985db05505a0159d-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-b95deb6819724582-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"894"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-985db05505a0159d-01"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-b95deb6819724582-01"}}]}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.830946117Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"12NNbneNbUM=","parentSpanId":"RmDNaV3g6Fw=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025826885000","endTimeUnixNano":"1735335025827735000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ETof3b34RNg=","parentSpanId":"RmDNaV3g6Fw=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025827707000","endTimeUnixNano":"1735335025830487000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"RmDNaV3g6Fw=","parentSpanId":"wsNTIF8DnsE=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025826714000","endTimeUnixNano":"1735335025830508000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"15101583-1ffb-9f6e-b511-efcd4ca14681"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=TUQrZ3dFSUpycGJ5TFp4c09pdGkvMDB3RWhsUkdLVEZ4Ni9OYmZOMWJuTG5xTlI3OE4vNjdQd0I4cGFSUU1QOUFTWXVqOEs4Z3lMenJhaks4ejRIdVE9PXwxNzM1MzM1MDI1fNlsbQS7GploSTVB0WfdtdH7gi0aLdGkm-Zw-kVVwqJF9rJZSPHGDEYpgyyhScfl32qa-0GvpPpCEPxXfroMBfIx0VSczgLJty1vJ_7-Wvuc_6EdlHR9x544l5p1a7zUqdftP975Xmdcj7tkJwsje3ttJ5uVESkFOy6sN0PAyQihwcYPd0aCvVnLPgVqroEbrbkPcVavCRI8sVE0Awx0X2YzkgBPWzKHmH43iAie-kM0uMZYFPFRtEFxFvD43PscwW-prc3qShMzp4C5cNJt2TRQCRJlXcUr_ccnecAjs3jdys3da_nFHMsbhpDgDfHaYsnL86zq_QVdjWJZFMpIEHOXQUVi2Q7PTm_n9iymHah7FPcWDaAo5BRsKr7fM4_JaHT_BsnrvLMRSV8c61gtzpBoDnIKrrvr7hzua4i1OFkR5XNRSOIy0gh1Re2ZpBFUo0H8miVsqD8Hqx7N1rxFopl3R-spO3a9Arin74L-5gZu1DsgkxVRa9oF5ntJ9ZigAKCnFFRgrmHynjxD4QMxGKR0tFBrlLBYiZFlWpxMYD-bmq6jccxwA8491DQAePeeyWbLIL4-9OIsBoZqIRmJac1gou6EqxJKsECNxxLCCreNKPYcNJKWdY6ZERWT2rVYlzaYowFnfV9KDvQAoaGV_D7rg3Dw7XGnQvo5NHAYGXv5sW7LB-vlPU_5CuVjV2bwmwcy6Ybez22aD1g%3D"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"660"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"c2c353205f039ec1"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.835894006Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"moqlVoGI3u8=","parentSpanId":"cnJtNlqDufs=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025821062000","endTimeUnixNano":"1735335025823076000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_unauthorized"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ANOpBD0iW7E=","parentSpanId":"wQTpo0tPYe8=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025833706000","endTimeUnixNano":"1735335025835562000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"d521e804-e422-9d58-9bd7-6b0ad4803942"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"1450"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-c104e9a34b4f61ef-01"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-c104e9a34b4f61ef-01"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"c104e9a34b4f61ef"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Cw7qX+Mq85E=","parentSpanId":"ANOpBD0iW7E=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025833842000","endTimeUnixNano":"1735335025835586000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_unauthorized"}}],"status":{}}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.838343385Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"aQWRcZQDEL0=","parentSpanId":"cc3o4WXmDUw=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025836159000","endTimeUnixNano":"1735335025836894000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"JMTzABiGm6s=","parentSpanId":"cc3o4WXmDUw=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025836862000","endTimeUnixNano":"1735335025837975000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"cc3o4WXmDUw=","parentSpanId":"dQPCN0PsSTM=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025835958000","endTimeUnixNano":"1735335025837991000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"2fad3a29-7864-427a-b449-6ee502094ba3"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=AwwHIua79vR9AvSBdLu7fXeMAAtUCvoAW35-81UtSuE%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-0f614b85a586f0da-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-c104e9a34b4f61ef-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"890"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-c104e9a34b4f61ef-01"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-0f614b85a586f0da-01"}}]}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.842701287Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"zZCpocublXw=","parentSpanId":"FWcsZeIymTo=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025839230000","endTimeUnixNano":"1735335025839938000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"rLKtQLC36X0=","parentSpanId":"FWcsZeIymTo=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025839887000","endTimeUnixNano":"1735335025842321000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"FWcsZeIymTo=","parentSpanId":"Z7eWk7+izdA=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025839066000","endTimeUnixNano":"1735335025842341000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"03113e71-88c3-94e9-8879-11a03f39105c"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=djBzMm5teGFzNDQxSlorL2ZYVUN2Nk1XVHAraVFnSjhtU01IRGRlU0czSm8zRUlsM296bjlEc0o4VVhXSHFPOTd3QnlDVEhtSlp1dElXS3ExOWx5dVE9PXwxNzM1MzM1MDI1fPYv0au-VRFGwhaPnlvMz_uEJ22N0pzR8VwFreItQnBEo4C5PRNtMUFyKPuWzxHL67dfMAFNXNaYKrpHwE-RshBnJbE4oxQd6W705zHKAwm2lJsMMMHJkHZmBUk_Tghkuj7AsIlCKQHAG4Dd3NuGBjqd3Kqs9kwin5JnZat8aa2-5KX5Lk4QUFo_sRMebvIARsqK2QubYU114RUM62nzm_HmkKMcvh1QsGQ1s7pr_QepoDs7hSj_oXUs_Tc7c9n97sBGwWXfGc4yKLdodqyubc31qdSmHxcU5I8-FEFbbhMWZ3Pk51N7sMOjoYoP2oupL6d5DSf9bCermdQNAm4UElB1aVY0l_rmg282JhxP_5qBCLAml8xhGHT6JxMHAKA8H6MpNBusSbcfPmfuO_px6ipSnKE9ZsmY4y8_nbMZUaIotKED5NJn9YUlKDgrI58R050HTSbv-zy-rBVpLRwFhk2aC5dhYSyxVc6QxEYIrKvnVyouumm4bvB9xankZB_A1ESpBrAPszKOoznYh1y72LR8P18o7tOvUWxonfsKbnNeZ92RZOOIeUcaJ6-ZYUVO4Os5DrL4QSTByrikq4hdsABQHu89omt_6iS2t_salQ6eVrGQLI9E9yHzoDTVzWsYf8DZ9NbpI9OHMBFB7nJ8DDVm7XXvpgKtRezh2JVu9C3BZHLswq_0dJrE2hH30U1_pXy4mAOjddNCVt4%3D"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"663"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"67b79693bfa2cdd0"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.851209249Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ix5QFxi6p84=","parentSpanId":"CXI9ud47kkA=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025849044000","endTimeUnixNano":"1735335025849730000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"UBrFUNOcDBc=","parentSpanId":"CXI9ud47kkA=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025849700000","endTimeUnixNano":"1735335025850755000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"CXI9ud47kkA=","parentSpanId":"yYgi/sxRPQ8=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025848863000","endTimeUnixNano":"1735335025850780000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"2c099712-3542-40aa-b3f2-f4f5cf85e4fc"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=KuYlGzyljFFgiYixx3bBF6bdlA-O1S065hn1lQuRHgM%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-6e31980f6e94316e-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5a6b77e41fb1dd4a-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"893"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-6e31980f6e94316e-01"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5a6b77e41fb1dd4a-01"}}]}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.856404271Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"RjOOCHrl7Pg=","parentSpanId":"+VzL5gg3U4o=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025851987000","endTimeUnixNano":"1735335025852708000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"B7oHC/7Ojus=","parentSpanId":"+VzL5gg3U4o=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025852679000","endTimeUnixNano":"1735335025855477000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"+VzL5gg3U4o=","parentSpanId":"xa5A0dmcNOo=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025851819000","endTimeUnixNano":"1735335025855507000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"e6ac2da1-292e-9a26-957c-d315a30ceae4"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=K05oR213S1ZvdUlZMlh0TDFaRlU3SUlZdmc4OGIxOUFoanBDaXkvNlpZc3ZUeklnc0VQMm1CYjFGYkYrK3ZYdXpnNkNtYS9MZUtleU9DY3NMN0VNUUE9PXwxNzM1MzM1MDI1fF-scOL_CXRp0kDbE4ZjdGwJnPC2MQY02ucYYTtu54IwwWu6QgQgYZ-L1nC0_8TPDlspFKyI_65zC2CAiEPwji6gGkC65hOD0mDcjWFyaMY8NgZJYvackeVgm0tLWY0KPOyJmJEk3oGmXGYlPoPJ_sz8ywot5peE8tV3_7FUgL2k61nEBRmqffg_ICtN21fxrk63NHmQZBw5E6_fj_DSwMT0XeLiu3I2sJSMP8gAXgZ-9KNAU34h9NmBwAqkyCJ5HP7N0cYwc0wCWoZVO8kgI_MrasJvIyvf6UKnp9oBDp8Ro4pTlrukgQgcj1-Wu2x6pCl9xpBT32Ftwl4Vuvr_zI7V6MoqZMKywDBNfsApHUkZTUedX0ZLK9A29RXEVkNqr9OevFVyVsMyOvTZ_tfS7_5hfwzHjyOnHq7q2COA1ocRzWCMaXtIeSk8WwL_K6hCf9vNrpSi0_agf2JWLbefcjy8AUAWjpp6jt4My9JhdRZ-qBmnL973LPUmyiIbYkC0Km1nIe71rItMLqx2SZCwuc8-9jq7DshSiSNzNqMD35PGdiiEawIhMFJwhWRWKCDMPBCYeAkYIODnfJWGBbOrD7kFReqvfO9Jm2zbuaw9tPw6HCHWY8XwIyAougw7FP15F9Vqtzc4imuTmsvkOYJNTvI8lz8Sk2ALwlPo63QbI9DFjlE_uDuGb-HqiUqe0eoIx4EqP9Ni_Qj_AkY%3D"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"660"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"c5ae40d1d99c34ea"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.870897034Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ojpEo1xw0Xg=","parentSpanId":"Wmt35B+x3Uo=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025846526000","endTimeUnixNano":"1735335025848559000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"c51ade07-1ab1-99e7-954d-d001ce8000e2"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"1450"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-5a6b77e41fb1dd4a-01"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"5a6b77e41fb1dd4a"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-5a6b77e41fb1dd4a-01"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"pMCRYOIpDEU=","parentSpanId":"ojpEo1xw0Xg=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025846682000","endTimeUnixNano":"1735335025848588000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_unauthorized"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"tKKFRvA/fOQ=","parentSpanId":"iwk4YAWR/kU=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025868257000","endTimeUnixNano":"1735335025870559000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"98273d47-4ad1-99b2-894b-1969b54c82a8"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"1450"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-8b0938600591fe45-01"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"8b0938600591fe45"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-8b0938600591fe45-01"}}]}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.873291353Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"zprk0rw2m8Q=","parentSpanId":"YtH+uCZXuRo=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025871096000","endTimeUnixNano":"1735335025871827000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"fOxEGb9Booo=","parentSpanId":"YtH+uCZXuRo=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025871799000","endTimeUnixNano":"1735335025872933000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"YtH+uCZXuRo=","parentSpanId":"ChvrQ2kjOv4=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025870948000","endTimeUnixNano":"1735335025872951000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"9f0a905b-af50-4ab6-bd35-31149b175f03"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=NfuC-nJofCctq_72Lvl602hVWbnwI3s-MmMAYICg8RM%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-9690e8cd7247b772-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-8b0938600591fe45-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"894"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-9690e8cd7247b772-01"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-8b0938600591fe45-01"}}]}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.877652095Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"zsr1/Xcbq2A=","parentSpanId":"99HdbStkmnE=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025874182000","endTimeUnixNano":"1735335025874878000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"rid7A7oKsBM=","parentSpanId":"99HdbStkmnE=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025874850000","endTimeUnixNano":"1735335025877277000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"99HdbStkmnE=","parentSpanId":"DtfBvkQHXZs=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025874035000","endTimeUnixNano":"1735335025877299000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"d609fa5c-f8fd-9547-bba0-89464298b9a1"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=NlpEbWhsdDRlbm1nNEVtSWR0U1BsY3RWU3kyRTNvR09DK3o1NnRLUEtGNCtCNUk5NmE0dUE2N01KM0xkdnk2WGgwTjN1eGQ2cG1rLzdweE4wc1JCbFE9PXwxNzM1MzM1MDI1fNPhIc3ToLHpvS_Gvc8aQRaU88cZYSsnQK8fP27HBXGJKzuRaASTXkCUMwIy2bfmT5GrsBxx8-MmupZH8PE_9xtv96G9GEbRgrrFdukuhrQei_80gU6OgUoTwTtCC9jC8T37myP2Oy72fpFGDoXxzEzRGhSyMAHQJy_HBv3PLX-UxHuResc5NH_Yj7aDx16xDkDAWmc-S1VeeUW0zqT8HgnzfXFYdZqf2_tnV12NUp-CLGXLRsU-3XJIsSwy5CRy1mfIGspxD-SxBHxPaRjKMY9FMxc4CsDmR4eiF28glstJrheqExh9wDhG0TFmgwLtEJ1Vq_hLTmV3RZO8jx8ZtdV1id1slzj8fwzRXH-IVKgTJ1uLcA28LLI1ZzJzD2PrTkfISjg6-5nUJv-OYQcQlQI21pxL1s4ICAa0l3KzE5i5_yeymQaGeAfSLIJ3-5RTTwBMrx5zgMlco96WRrJZ604XqQSOgiGoxSLA9aHu__QOK9OU7GFdga92Y67T3vkKwvWXeX3w2EAF__tzDF5dy--oGborqKGcv0fkKeMgRyfxkxJ-E1w0Z7ppcwL6LL1q78zKI8JcLzgV8MyVDcVnHRrkIToVZ5YYavQnbXK_7lfo_XbKyifs4ZqPmwNY0WyGZ0j_ZZTOLRo8lQR2y5NOlTZRWCz3sqkmOJ3SeAoGrHiQ0tZDfFgvO99ucbtQj1_PCyO6u2-JaIpH6fg%3D"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"660"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"0ed7c1be44075d9b"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.890024754Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"AxcQF3bUAPA=","parentSpanId":"tKKFRvA/fOQ=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025868412000","endTimeUnixNano":"1735335025870665000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_unauthorized"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"1e87sjvJiZc=","parentSpanId":"YEKY6t1fm8A=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025887639000","endTimeUnixNano":"1735335025889676000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"44d9cddd-502b-9709-803f-895824cdd2e5"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"1450"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-604298eadd5f9bc0-01"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"604298eadd5f9bc0"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-604298eadd5f9bc0-01"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"VlLCvKW20c8=","parentSpanId":"1e87sjvJiZc=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025887761000","endTimeUnixNano":"1735335025889701000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_unauthorized"}}],"status":{}}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.892699935Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"TnTIhQSeD8s=","parentSpanId":"L7c60gf+bKk=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025890215000","endTimeUnixNano":"1735335025890903000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"jh13uEWyq54=","parentSpanId":"L7c60gf+bKk=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025890877000","endTimeUnixNano":"1735335025892386000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"L7c60gf+bKk=","parentSpanId":"X170o5Ll3/I=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025890067000","endTimeUnixNano":"1735335025892402000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"7aa3c6c4-084b-4830-9982-8d15416ab378"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=dqScDf0iAzuxtV8kOVFuK6LkRdrngMgT7ZF5WZSWxrM%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-75518e62ed861ee4-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-604298eadd5f9bc0-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"890"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-604298eadd5f9bc0-01"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-75518e62ed861ee4-01"}}]}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.897046547Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"fpe5wrNxals=","parentSpanId":"wZndbBgTSW8=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025893566000","endTimeUnixNano":"1735335025894320000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"zaz8bXwr3x0=","parentSpanId":"wZndbBgTSW8=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025894292000","endTimeUnixNano":"1735335025896622000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"wZndbBgTSW8=","parentSpanId":"BDPsEOd6+nI=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025893422000","endTimeUnixNano":"1735335025896638000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"efd64acb-d818-977c-8592-7c99c36e1622"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=eE9BRU5YQUZpTzFaV0pPK28rTmF0NU1xempEV1NzU3hhM1lDL1I2Zm5KNFRkM0NPd3RQY2wxZDAvVVFJaVB1MTN6enlwa1h1NDFaZmRHZGFIdFQxVlE9PXwxNzM1MzM1MDI1fEw2htjJsKXWR0ubmlecrlxxyWaM5lyrfQKVov8UW14H8_5YI7twjd0qGPWd8uLPpHaUpE5zR5PVk37RnPpkNC2J5NteJv5IN51QbBerrsuGuDZ650Sbqr9-p9FXOKmBBcoKRPW_0N-B7PWMWkQIsY5rVbjYeXgfOjuadl0PNkCA0awKZQhOwfJwr8waX88ypxsRszpU26pXB75zl5tO-AOtrROKkjsv_Y0i-Vvvhj7VGlRPsJ2xvNqK9DuJLbdjmyKrSutz1IsGwhs6NWyd82yaid7x7qCob3VkStVsNeLhjcniP9xjD22Hbi50itMD1HyXCTu_hAVyEYw2SHC842nad-4tP1EEvQkB05PBwTT5G324_lbicp6lFCSzGT9rS2MPEuz12m8Uq9hdJEj18am1pZM7vBXgevH6K0mwlpb4eIgqceOwxbgSHeGw07op-0jdeGnbz1REdKGGui4pWW5PmxcCjDiAG6g6kP6DrXUhPvAVXPK3GhpK4lxuLfU84zuQr81Xo0EmYOxQGjvdcYepBDn9anS1tEghjCk7iptxM6zO11Q7vMFI9wcVHrNl3sSCAM2k21BFmPEyWcnWvPFsQMsegbSznjD2X52WjB7k19H406-bHN-d5Py0dcMo185Y1aAIZ3m2CyXNMgPlrBPT-5XDSvf8113VVpTk8Yazr1rx0DzKlmLhWaUo0Nxm9mPwxK17XRvpdSg%3D"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"660"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"0433ec10e77afa72"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.937586142Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"x4/fCcrN2l0=","parentSpanId":"nnEajOq1wt8=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025935495000","endTimeUnixNano":"1735335025936254000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"8j7gcdiyUN4=","parentSpanId":"nnEajOq1wt8=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025936225000","endTimeUnixNano":"1735335025937234000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"nnEajOq1wt8=","parentSpanId":"NEEa6gf13GY=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025935358000","endTimeUnixNano":"1735335025937250000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"bc0d83c8-6396-4767-ba4b-22aab151f5dd"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=1DclRi_1pUHTYKNXCgmLO7xIgxsFHQ6PnZo6vg2QNpc%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-e310a150afa55100-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-61e9711f41e1b82b-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"893"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-e310a150afa55100-01"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-61e9711f41e1b82b-01"}}]}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.945677139Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Ktn4PSE/Eio=","parentSpanId":"QacsP82TJOA=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025942057000","endTimeUnixNano":"1735335025942751000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"PbLOU7xbMgE=","parentSpanId":"QacsP82TJOA=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025942723000","endTimeUnixNano":"1735335025945322000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"QacsP82TJOA=","parentSpanId":"Sn6DR8I82JI=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025941876000","endTimeUnixNano":"1735335025945338000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"96383362-b0e1-9643-a55b-d3552e7f956d"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=Vld4RWw1Q2pKNkZ2NXNST3pzVTdFMi9qSTNMY0E3bm91bDNYUDNyNWVJbUMrekFzSW5WejIySEtxclJscnBvUkkvVWY1RStubmcrT1g3S1llcklSUWc9PXwxNzM1MzM1MDI1fCZPSA2y7mrm6xOOpuWELdtqzK8AmnUFMhpMijy4u4IE5SwEboZafPLVOuG6XLO-hnn5Xf2lTXraFGd6UpqWGelZT7VfHlpWPhDdf2uCYIAj1u6TYRgPS3_hMilmsO6AvSWmX37-XsPL4I78VhCqbL2FoEKz_CgyCAVhowdooxX6INcvZ7-G8BCJX16jnZKu7o3gX8xFl0mrHYKV1EAPY1V2YsxRTGRS2ptkcqEuxXw9YVOQIIsC6OsphlbN91V6qHGhzBRcLX8CZ-8E_2D9Osgiy4-C-koOCtzRLLFJy9S6oq-8Z1c3PK_wGhTu-syftdKFKjKjdRo97DWgjVz_lnKWeYC2usX32gaExGKkVkMERCaQ6JKoi-TXo4kY1g-AywEfn_gehBqAqzlXQ5VHEDdCth5C2BHjOhgUKcDv7W4Oqmp_93_U28elEz4Q72JUpSaDDe1fKA_eArNUrQBJTJ6N6bABEwL52xkh0ifRwWcERMCwnc1bOlAIDs2T1yWW27hdYrpSOZbAwdKuSji2x1ut0-DqgnGoDMMqzgS98MUPsC9I3PNltfu_DgC959VjH7H6_HmE2q1nPELeZUXRCGzAZTOFE23bD0ks6uyipS3jaVb1qxVClG3VMuSLf9SdV2ESH7MWJaXGyvhN8c1KBsiZp5L-7gqCnsZOMYQrJNQIsbdic-DkeRlmQYbs8iZykp-cADl2uTKek-0%3D"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"659"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"error","value":{"stringValue":"true"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"4a7e8347c23cd892"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.983824805Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"pVa2P4/shJY=","parentSpanId":"YelxH0HhuCs=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025933096000","endTimeUnixNano":"1735335025935055000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"92ff535f-4f0a-98fc-b12c-76a42b2de861"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"1450"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-61e9711f41e1b82b-01"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"61e9711f41e1b82b"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-61e9711f41e1b82b-01"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"HG6gDV9Mh1I=","parentSpanId":"pVa2P4/shJY=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025933254000","endTimeUnixNano":"1735335025935080000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_unauthorized"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"aOAQNAsmhVU=","parentSpanId":"VDHmxoPFG3g=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025979317000","endTimeUnixNano":"1735335025983431000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"846d72af-c15f-9579-83e0-abc9ad6219ab"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"1450"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-5431e6c683c51b78-01"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"5431e6c683c51b78"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-5431e6c683c51b78-01"}}]}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.986602409Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"U+zO7kgs2kw=","parentSpanId":"UEUDZ2+1HPY=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025984046000","endTimeUnixNano":"1735335025984989000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"C27ROXjc3Mg=","parentSpanId":"UEUDZ2+1HPY=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025984957000","endTimeUnixNano":"1735335025986182000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"UEUDZ2+1HPY=","parentSpanId":"V4KDwU2b7No=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025983829000","endTimeUnixNano":"1735335025986199000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"1b82a31f-db14-4a9a-8d57-7d7c791a52f5"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=ZnX70Wyz8dCAymop0VMwrObf9dt9EeGcdvdSJMWerio%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-a0e19ab5c251cf8a-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5431e6c683c51b78-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"894"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-a0e19ab5c251cf8a-01"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5431e6c683c51b78-01"}}]}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.993743113Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"pqDfjSA5zY8=","parentSpanId":"BHj177uY9Us=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025987402000","endTimeUnixNano":"1735335025988217000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"rn0P4iWyrGg=","parentSpanId":"BHj177uY9Us=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025988185000","endTimeUnixNano":"1735335025993144000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"BHj177uY9Us=","parentSpanId":"X0cwWL6jdNw=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025987256000","endTimeUnixNano":"1735335025993166000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"515da6a0-e13d-9b0c-b59f-618472b894db"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=TlVucmJGSVM1TTkxQjdVeFpFTndpblBPMUpqdjBoOTBCTkN1NVBPa1RWWGkzcC9YNE1Td3RYc3IyOHZQS05HSVA5am9EbngyT0pNdzBzdEQ4Kzhrbmc9PXwxNzM1MzM1MDI1fGFr6SjNR6oY3LbAckTb1zOQwi4DF7qp73tBPnxNCM9uAN-E7benf9dM4rTw1MXWLEUpA1XTw-NdsI5DXa5mG-guOz-4MjnubljhndMmKy6H-mdPjrJwqpirLHSjGNIOMh6prOpDlt1i-RnnOB_wtIY1wxUCPntd-aCiBGfViIgB0aCMQq6YcqcJYohYyPYYILz6IczkvpPa5cevuvnCJ20J7X69_6AJ0tYzwo9PVmYrGFyTJULaNfw-2G-nUAFuP4-FHv4PexN-NB_Qn0kM4KsX6482RPpNetBM8hHfHbdlr6J_018NOc7Jpv092cxDmMpwzeTQ7mKtEgntBj8WrClM6VTrkecRpdJ8yiZhLvgVhjK76JUFXlXNzYItTt-NvBKuXZ-869TmVrWADw6579M1IP2uCvQlE88v4e__LelTsipYfTzM_kN_Zl7hoDur57rTfzsbkipm5-szZ1W4wEksgtPybfnauIOJY3AtTQ1u6HSciCbGULA6N024tcnpDAJ2mOwYuYbMHlvVYpTv9AJ7P6WOuxaD9GQP5jDJb_AcmhzylwX6r7TQrHnPtbRfratJUOP85A9-5vgaKdLJ4QIPgVqPh4eyTR6xRWWkFnkJdUnJ8m8gshI0ZZI1KzM-ORzBPl8TrEFfqrHHVq1ujN7JQknxUgbrWQOU-jzncZnCYk6TBMbyFX7SdKMG8rgbLaDdnANGnoVCiuA%3D"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"352"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-a0e19ab5c251cf8a-01"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"5f473058bea374dc"}}]}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.996693919Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"IwEGClsNoWA=","parentSpanId":"YDVIt+KGpQQ=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025993892000","endTimeUnixNano":"1735335025994689000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"HLHEc+bHits=","parentSpanId":"YDVIt+KGpQQ=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025994653000","endTimeUnixNano":"1735335025996329000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"YDVIt+KGpQQ=","parentSpanId":"4mu5mTDooas=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025993722000","endTimeUnixNano":"1735335025996352000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"8fbba8a8-1d48-4c54-a298-e8193b239932"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=ZnX70Wyz8dCAymop0VMwrObf9dt9EeGcdvdSJMWerio%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-a0e19ab5c251cf8a-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5431e6c683c51b78-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"972"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5431e6c683c51b78-01"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-a0e19ab5c251cf8a-01"}}]}]}]}]}}, - {"timestamp":"2024-12-27T21:30:25.998791864Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"SRBjmOgMIao=","parentSpanId":"aOAQNAsmhVU=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025979443000","endTimeUnixNano":"1735335025983541000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_unauthorized"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"g8gyWKtu1KQ=","parentSpanId":"GeyplZHybww=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025997043000","endTimeUnixNano":"1735335025997853000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"c4K33/UOGkY=","parentSpanId":"GeyplZHybww=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025997706000","endTimeUnixNano":"1735335025998445000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:41823"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]}]}]}]}}, - {"timestamp":"2024-12-27T21:30:26.00426142Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"GeyplZHybww=","parentSpanId":"fUtVfh0g50A=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025996849000","endTimeUnixNano":"1735335025998532000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"cf4aa962-5833-431a-9c34-698325c32dde"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/.pomerium/callback/?pomerium_expiry=1735335325\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_session_encrypted=gnpoJSWHmBQQkVfvjiOomyk9_F63PJnoFtYhIeue3g62S-0NF5d6BOpeOkgOwptEfJSQ_kKjqSYQSDT6kkqtKF1QDaX1fRCjLC9LHcOeioxnfLQ2rglnkAbs6XFscaRpZFstZ3TlJXUXf4oL9C-DwpXx96tlbHS9RzPLPijhEMnnLi3_zjw0hlHFmk8a6laRLJFuf-n9vtjh8TVnhmeiE7hic2GNbQjGfu8dqWjOvWDhN1XwmMzKGdp3RosO8xLozBy4i9CP1C35UgXeuoPvS8JPa4Y2d3V3YazDKutWLoMdmtZVicgWb80MJRuR9NX4vlscb12wNk2QpO-qxXeLn6L8UennLI8v2RM-G-TPmK7GDQ2Ihfd8ifmIdRNCHIIKpTpNGHuc6uwWmc86lQt6VP7yyn2-6F0lUtMRLSFZMmfaqcEM2I1C1XULxqi-R8W5aQgO5Wb1sFA4quUUNQy2lHanVtFceM8lOVHeIBuOaieLxFvKmf0TILOdlaYDoeNKMLk9Z_gPBEUyngeDwfD-2RimJTkeUJxQUpjCuMtfc_0aWfKg8PpeFrwnQhSBBhdasBEbplMrRZ-YwcC_IBHeLRqANNaW8pTpVaimIsTCDoMq0WQu5UXhGWT_Xbc2BQ0vNZVCKylD47DijPzYP8fQd5ibiJtElOmxwuG_hl_rPEcPQKMnoByUd6MtVvFdkfik-iGqMEEGbYfixzO3D4q5ahd13lWCkaFsajMJS9Mapde3c2V-DwS53stVswOYFnOetvMYt4koEE5mqg27zdXEUTNImTExWvpo5swL9JgGbWlGwJIrTsSLpWcD7cfb\u0026pomerium_signatur"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"66"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-9827f4d83b31fdd3f0d0f5a17584e3dc-a0e19ab5c251cf8a-01"}},{"key":"pomerium.tracestate","value":{"stringValue":"pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5431e6c683c51b78-01"}}]},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Vh7x9OYP6sg=","parentSpanId":"hKGR0btH9xI=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025998973000","endTimeUnixNano":"1735335026003511000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"xLaYTRAzDqQ=","parentSpanId":"hKGR0btH9xI=","name":"router route-29d1ce2f2cda20fb egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335026003383000","endTimeUnixNano":"1735335026003898000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:43211"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:43211"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"http.status_code","value":{"stringValue":"200"}},{"key":"response_flags","value":{"stringValue":"-"}}]}]}]}]}}, - {"timestamp":"2024-12-27T21:30:29.645550038Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:377"}},{"key":"service.name","value":{"stringValue":"Test Environment"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"PcivIA1AuLg=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"Start","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024645655072","endTimeUnixNano":"1735335024646175548","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:543"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"Ea263bNDAzA=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.config_source.rebuild","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024738945717","endTimeUnixNano":"1735335024739703437","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"Wz1aMEqBJfk=","parentSpanId":"e8L/2jnMPhI=","flags":256,"name":"envoyconfig.Builder.BuildBootstrapStaticResources","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024957225167","endTimeUnixNano":"1735335024957253369","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"e8L/2jnMPhI=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"envoyconfig.Builder.BuildBootstrap","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024957107426","endTimeUnixNano":"1735335024957297089","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"r2S68piHLi0=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"authorize.newPolicyEvaluator","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024965662429","endTimeUnixNano":"1735335024973816647","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"R41i4hg7Nqc=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"authorize.newPolicyEvaluator","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024974090360","endTimeUnixNano":"1735335024974398313","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"yfcZPaW83Xk=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"snippets.WaitStartupComplete","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024646193238","endTimeUnixNano":"1735335025791941061","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"s/6I32KmWeU=","flags":256,"name":"Authenticate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025791948461","endTimeUnixNano":"1735335026004177499","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:88"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"ELScfHlrO3c=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"State Change Callback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026004322061","endTimeUnixNano":"1735335026004363371","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:932"}},{"key":"state","value":{"stringValue":"Stopping"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"lCPvM1rPzvc=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"State Change Callback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026004331111","endTimeUnixNano":"1735335026004390611","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:932"}},{"key":"state","value":{"stringValue":"Stopping"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"/Z4/vZfyMIg=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025790056388","endTimeUnixNano":"1735335025791635986","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"rBnm2/0CUj0=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"registry.Registry/Report","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025975496615","endTimeUnixNano":"1735335025975526226","attributes":[{"key":"rpc.service","value":{"stringValue":"registry.Registry"}},{"key":"rpc.method","value":{"stringValue":"Report"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"TGyFsg9jcEc=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"registry.Registry/Report","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335026621252279","endTimeUnixNano":"1735335026622954109","attributes":[{"key":"rpc.service","value":{"stringValue":"registry.Registry"}},{"key":"rpc.method","value":{"stringValue":"Report"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, - {"timestamp":"2024-12-27T21:30:29.647499751Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/upstreams/http.go:223"}},{"key":"service.name","value":{"stringValue":"Upstream"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp","version":"0.57.0"},"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"uSbobbStGD4=","parentSpanId":"xLaYTRAzDqQ=","flags":768,"name":"Server: GET /foo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335026003679492","endTimeUnixNano":"1735335026003700273","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"127.0.0.1"}},{"key":"net.host.port","value":{"intValue":"43211"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"35630"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/foo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"2"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, - {"timestamp":"2024-12-27T21:30:29.649291053Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/upstreams/http.go:223"}},{"key":"service.name","value":{"stringValue":"IDP"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp","version":"0.57.0"},"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Y/2ikMC4YHI=","parentSpanId":"68TkOnpSKwY=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025811671136","endTimeUnixNano":"1735335025811745867","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"wGyDo99t63A=","parentSpanId":"AgIT8HoFaKI=","flags":768,"name":"Server: GET /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025814635582","endTimeUnixNano":"1735335025814710443","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"512"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"mDIE54IHDzU=","parentSpanId":"+bOSkydEmoU=","flags":768,"name":"Server: POST /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025815045916","endTimeUnixNano":"1735335025815166098","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"/eKkXKPf76U=","parentSpanId":"7uQdjOjS7cI=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025816774837","endTimeUnixNano":"1735335025816830008","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"RM7RlGkPMJ799QjggkXaDg==","spanId":"aQejqjlsLbY=","flags":256,"name":"Server: POST /oidc/token","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025817074821","endTimeUnixNano":"1735335025817277083","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/token"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"193"}},{"key":"http.response_content_length","value":{"intValue":"673"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"PGcngHRgqCX4iaoO2fEC5g==","spanId":"wT/nSaqRA9g=","parentSpanId":"ZsmRwKi3Ki4=","flags":768,"name":"Server: GET /.well-known/jwks.json","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025817664798","endTimeUnixNano":"1735335025817768259","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/jwks.json"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"237"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"izW4Mbss9bQpEKa6oaesqg==","spanId":"IziQ2Bt/qwU=","flags":256,"name":"Server: GET /oidc/userinfo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025818151274","endTimeUnixNano":"1735335025818204185","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/userinfo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"150"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"hblCu3rYdXg=","parentSpanId":"uMSJ2tNZc4o=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025824939145","endTimeUnixNano":"1735335025824985545","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"R+9qz/UwFgM=","parentSpanId":"jqKetiY96Q0=","flags":768,"name":"Server: GET /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025826074959","endTimeUnixNano":"1735335025826105089","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"512"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"STcXbMrVBjs=","parentSpanId":"X83U/uv8a/A=","flags":768,"name":"Server: POST /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025826410313","endTimeUnixNano":"1735335025826494124","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"xZa7EacaVP4=","parentSpanId":"d/S+GCMxW/g=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025828197924","endTimeUnixNano":"1735335025828253395","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"3yAOnptzzqQGSMsp5b9s/Q==","spanId":"sVuwHgVcAxw=","flags":256,"name":"Server: POST /oidc/token","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025828508488","endTimeUnixNano":"1735335025828682680","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/token"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"193"}},{"key":"http.response_content_length","value":{"intValue":"673"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"Cm2eM1rGKwOPUvNrZ3JnIQ==","spanId":"wUiYdnapEuA=","parentSpanId":"2vnGWHtaRpY=","flags":768,"name":"Server: GET /.well-known/jwks.json","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025828998383","endTimeUnixNano":"1735335025829035064","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/jwks.json"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"237"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"gX+ojROBZc82P4SUdheIqQ==","spanId":"AjkxbAMUXEE=","flags":256,"name":"Server: GET /oidc/userinfo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025829408768","endTimeUnixNano":"1735335025829437668","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/userinfo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"150"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"63DJFKv/oEw=","parentSpanId":"STVwwUHTbwU=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025837400994","endTimeUnixNano":"1735335025837448364","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"EcKjsRkfXgw=","parentSpanId":"pOYMzXr9GH0=","flags":768,"name":"Server: GET /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025838435756","endTimeUnixNano":"1735335025838474057","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"512"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"b+dMjtK4KO4=","parentSpanId":"iIl8PJ7gVyQ=","flags":768,"name":"Server: POST /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025838776380","endTimeUnixNano":"1735335025838842161","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"hIU1ya0kjXw=","parentSpanId":"rhwZXUEWDec=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025840392170","endTimeUnixNano":"1735335025840438940","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"v2TRBVlwtUQGp9pqypOIsg==","spanId":"B/CQbn5aNW0=","flags":256,"name":"Server: POST /oidc/token","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025840684803","endTimeUnixNano":"1735335025840820895","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/token"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"193"}},{"key":"http.response_content_length","value":{"intValue":"673"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"Ng1V9c7eduVtG8yBTNSUdw==","spanId":"KGHvI14mYcw=","parentSpanId":"uAH2dcySq2g=","flags":768,"name":"Server: GET /.well-known/jwks.json","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025841139289","endTimeUnixNano":"1735335025841186439","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/jwks.json"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"237"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"9iXAmWolAOXVMR/gXbFaDg==","spanId":"SAuhBchhq/U=","flags":256,"name":"Server: GET /oidc/userinfo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025841526593","endTimeUnixNano":"1735335025841551763","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/userinfo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"150"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"AmcPtu7pXmc=","parentSpanId":"KVfrvuRAMxg=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025850262848","endTimeUnixNano":"1735335025850319459","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"NQ1MMF1Ptvk=","parentSpanId":"yabq+U/JonU=","flags":768,"name":"Server: GET /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025851255979","endTimeUnixNano":"1735335025851289730","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"512"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Whn0hDahaBI=","parentSpanId":"zFaBQK23WvM=","flags":768,"name":"Server: POST /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025851556073","endTimeUnixNano":"1735335025851635154","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"V6EIVmKug9I=","parentSpanId":"Ua0wEfIKbXg=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025853150272","endTimeUnixNano":"1735335025853212453","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"jW95QpYR4b2l5wJxuDeadQ==","spanId":"WnrnskhtQlM=","flags":256,"name":"Server: POST /oidc/token","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025853460966","endTimeUnixNano":"1735335025853593097","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/token"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"193"}},{"key":"http.response_content_length","value":{"intValue":"673"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"1F2INkD1DM1Gd2zZRmjgeQ==","spanId":"fL9XHs7b2BM=","parentSpanId":"TOYv2v4FgWY=","flags":768,"name":"Server: GET /.well-known/jwks.json","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025853865651","endTimeUnixNano":"1735335025853898042","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/jwks.json"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"237"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"dVmW0pZN9Eh8Lwurw9K1WA==","spanId":"MAiXzy9HNN0=","flags":256,"name":"Server: GET /oidc/userinfo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025854295756","endTimeUnixNano":"1735335025854335637","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/userinfo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"150"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"3FKwAcrzrXo=","parentSpanId":"FBXEc0lb4OY=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025872366062","endTimeUnixNano":"1735335025872423993","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"BrGdyC2/FrU=","parentSpanId":"mOp99KCsZuE=","flags":768,"name":"Server: GET /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025873414044","endTimeUnixNano":"1735335025873448105","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"512"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"L97J195gDzQ=","parentSpanId":"xm+TiiRVATY=","flags":768,"name":"Server: POST /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025873705238","endTimeUnixNano":"1735335025873790809","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"DFKYB+M6GSA=","parentSpanId":"u5Nt8T6Ejo0=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025875318107","endTimeUnixNano":"1735335025875368818","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"tEoEJcw2IpRwMPyBWPFRNA==","spanId":"DrI0XThkTj0=","flags":256,"name":"Server: POST /oidc/token","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025875602100","endTimeUnixNano":"1735335025875774792","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/token"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"193"}},{"key":"http.response_content_length","value":{"intValue":"673"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"3QpZE6RgWdOi5jhjakP3iw==","spanId":"rk1WHi4jjew=","parentSpanId":"ieD2I+EcHco=","flags":768,"name":"Server: GET /.well-known/jwks.json","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025876076737","endTimeUnixNano":"1735335025876116057","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/jwks.json"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"237"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"knYBLzILdB/6QmbrWijZ7Q==","spanId":"gh51Il616ow=","flags":256,"name":"Server: GET /oidc/userinfo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025876497642","endTimeUnixNano":"1735335025876534372","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/userinfo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"150"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"C24Dm6NaHuw=","parentSpanId":"Vb1M0/gTV54=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025891402500","endTimeUnixNano":"1735335025891463470","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ouFveyHvX5w=","parentSpanId":"CMjIXZ4ROSY=","flags":768,"name":"Server: GET /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025892835007","endTimeUnixNano":"1735335025892863517","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"512"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"YkQT7tUnsR0=","parentSpanId":"HWuw3R1htnc=","flags":768,"name":"Server: POST /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025893167181","endTimeUnixNano":"1735335025893234171","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"I/3OAGfQkYM=","parentSpanId":"E3uSnGr1nuo=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025894728240","endTimeUnixNano":"1735335025894779730","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"eRji0KpJP6RxIM1Uo1uVWA==","spanId":"e3qZQ2rnVW8=","flags":256,"name":"Server: POST /oidc/token","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025895040823","endTimeUnixNano":"1735335025895203635","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/token"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"193"}},{"key":"http.response_content_length","value":{"intValue":"673"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"PgqwQu60TK/M1HnlytJ0OA==","spanId":"SBNFjFlUR6o=","parentSpanId":"bO5cO4l7/ZY=","flags":768,"name":"Server: GET /.well-known/jwks.json","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025895488488","endTimeUnixNano":"1735335025895524139","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/jwks.json"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"237"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"NePcUVjMrbGMdEnmUOzekA==","spanId":"h1Xyietj7ck=","flags":256,"name":"Server: GET /oidc/userinfo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025895906514","endTimeUnixNano":"1735335025895972025","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/userinfo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"150"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"z4aamFIOXuE=","parentSpanId":"s5LlPWRVkt0=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025936688012","endTimeUnixNano":"1735335025936746062","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"fk5+rIJna+I=","parentSpanId":"rjRgfziAtxI=","flags":768,"name":"Server: GET /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025937647773","endTimeUnixNano":"1735335025937705593","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"512"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"0IzKTkJIBnQ=","parentSpanId":"AxuIpomNTe4=","flags":768,"name":"Server: POST /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025940776870","endTimeUnixNano":"1735335025941095344","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"S3OrW/9qYDk=","parentSpanId":"MN31qbGJoAI=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025943250280","endTimeUnixNano":"1735335025943297390","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"RuF2XmPgNlsgjQjJ8yCiOw==","spanId":"qll3f4pm2tw=","flags":256,"name":"Server: POST /oidc/token","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025943549793","endTimeUnixNano":"1735335025943753606","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/token"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"193"}},{"key":"http.response_content_length","value":{"intValue":"673"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"Taxuplp6DB+VUFbngmAAPw==","spanId":"pEHc5RZ44Kk=","parentSpanId":"75tyAErilUw=","flags":768,"name":"Server: GET /.well-known/jwks.json","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025944066200","endTimeUnixNano":"1735335025944116931","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/jwks.json"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"237"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"zdmjmNjkYA1wcwCsZgX1hA==","spanId":"kzagdp0+10Y=","flags":256,"name":"Server: GET /oidc/userinfo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025944444054","endTimeUnixNano":"1735335025944473565","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/userinfo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"150"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"v6U7DP7doY8=","parentSpanId":"11tLsq+ze/k=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025985549525","endTimeUnixNano":"1735335025985615986","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"dWO7PrLMczE=","parentSpanId":"zKjX115N+9k=","flags":768,"name":"Server: GET /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025986659259","endTimeUnixNano":"1735335025986699100","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"512"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"GYkxU/Q8BWU=","parentSpanId":"Jz4teHxKmjI=","flags":768,"name":"Server: POST /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025986949583","endTimeUnixNano":"1735335025987023313","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60868"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"XtNQUxuKbrM=","parentSpanId":"EqhoNIgJKpM=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025988592292","endTimeUnixNano":"1735335025988645103","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"bnPd9sGAr+N9kLYSZbNJdw==","spanId":"SvEOChXT7SI=","flags":256,"name":"Server: POST /oidc/token","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025988865176","endTimeUnixNano":"1735335025989028967","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/token"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"193"}},{"key":"http.response_content_length","value":{"intValue":"673"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"u+VskjE7NJ5A/WdVWZgX0g==","spanId":"t7GKDNGmpVI=","parentSpanId":"eNxd/O1kD1M=","flags":768,"name":"Server: GET /.well-known/jwks.json","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025989337901","endTimeUnixNano":"1735335025989370341","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/jwks.json"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"237"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"9noCFY0zqk9a5LpKHBqUpg==","spanId":"UgilqIFLTj8=","flags":256,"name":"Server: GET /oidc/userinfo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025989668615","endTimeUnixNano":"1735335025989705785","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"33771"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"60866"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/userinfo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"150"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, - {"timestamp":"2024-12-27T21:30:29.649769198Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/upstreams/http.go:228"}},{"key":"service.name","value":{"stringValue":"HTTP Client"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp","version":"0.57.0"},"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"6nkCYk4L8AI=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /foo","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025792124133","endTimeUnixNano":"1735335025804742723","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"net.peer.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}},{"key":"http.response_content_length","value":{"intValue":"1450"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"kNePRE2AarU=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /.pomerium/sign_in","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025804767524","endTimeUnixNano":"1735335025813157644","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=Z1Fi-CygBcWOsGaemJ9Kjck8J5CFO2-Q0NaLJ_090KA%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-09f757396858b657-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-ea7902624e0bf002-01"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"AgIT8HoFaKI=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025813175384","endTimeUnixNano":"1735335025814805524","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=bVhDeVhqYXBxRzNLS0F5dm1LVXRzUTZrUHVtSDgrcjdhQSswdmZsT3o2eE81OGJsaEgvOEY4UUVZbFV6em95elFySUNmeFJYelJ4Y0RkRWErUVdtWnc9PXwxNzM1MzM1MDI1fFr-RTRGNqnLn2IMEbk7FiyJdpSLrfG2toEU04qhst7IcWtWqDDKpQf-rvb4AOVFEyb_DCJImrMZ_JKA0yiP0S_U54-vwXFi6BfICA4m3NAFaFycuWP-DbKAQAfk9Uba9ichigqepuu4l8w3EGZwlpOR_H98gkVp8i0wUsRSi0QvJExUdEYP2R6hvWCaEF24kJZGzEgc3K8tFLuKKXF51oZzS8uTXWxZKboeoN_QAE7OJ5s2oD_oUJeQW1mU7W5l1lslB214Zv24ft7fLAbVQYJBd14DDoHfg4WZpQE8v3qIuCCMNcgCod69B1Fa-h4Glz138FEfNbnOgFjiCkW63brqiutwbtIn_Voy034oAAK95qCQvwc0B5RCf2peV7-rb3IrZk6h_ABEeNeRZEkKZtsixVns0ew7rEzfUlcPFre-3D7RK0HfV66arIKuNr4qnMLBuIiDO9PuucWyY4Rft3VAp8Onv-ENIWwLVkrOGvnDFtqubfCnutmW1cqXzF1oVuckiEOwmG0k0vUCk7xR8LLEkNBkh3s3yF3F8LvZLlfesQIl-dNXAWWKIonfCyb_fLiciYxlROetfJXFkMr6LKvgCUHB4AAkx_X--KpHabCpBVnFrEIPv2E59D-eLo7J9Sf4Yhd6nDzdAgj5B4BgDPKSLWvAvjRe5-kD2AT2AGcG-pIBVuKZoI7avNO0GJM23k0-b6_djF13Cg8%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"512"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"+bOSkydEmoU=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: POST /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025814897615","endTimeUnixNano":"1735335025815243729","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=bVhDeVhqYXBxRzNLS0F5dm1LVXRzUTZrUHVtSDgrcjdhQSswdmZsT3o2eE81OGJsaEgvOEY4UUVZbFV6em95elFySUNmeFJYelJ4Y0RkRWErUVdtWnc9PXwxNzM1MzM1MDI1fFr-RTRGNqnLn2IMEbk7FiyJdpSLrfG2toEU04qhst7IcWtWqDDKpQf-rvb4AOVFEyb_DCJImrMZ_JKA0yiP0S_U54-vwXFi6BfICA4m3NAFaFycuWP-DbKAQAfk9Uba9ichigqepuu4l8w3EGZwlpOR_H98gkVp8i0wUsRSi0QvJExUdEYP2R6hvWCaEF24kJZGzEgc3K8tFLuKKXF51oZzS8uTXWxZKboeoN_QAE7OJ5s2oD_oUJeQW1mU7W5l1lslB214Zv24ft7fLAbVQYJBd14DDoHfg4WZpQE8v3qIuCCMNcgCod69B1Fa-h4Glz138FEfNbnOgFjiCkW63brqiutwbtIn_Voy034oAAK95qCQvwc0B5RCf2peV7-rb3IrZk6h_ABEeNeRZEkKZtsixVns0ew7rEzfUlcPFre-3D7RK0HfV66arIKuNr4qnMLBuIiDO9PuucWyY4Rft3VAp8Onv-ENIWwLVkrOGvnDFtqubfCnutmW1cqXzF1oVuckiEOwmG0k0vUCk7xR8LLEkNBkh3s3yF3F8LvZLlfesQIl-dNXAWWKIonfCyb_fLiciYxlROetfJXFkMr6LKvgCUHB4AAkx_X--KpHabCpBVnFrEIPv2E59D-eLo7J9Sf4Yhd6nDzdAgj5B4BgDPKSLWvAvjRe5-kD2AT2AGcG-pIBVuKZoI7avNO0GJM23k0-b6_djF13Cg8%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"M+elTBSUJY8=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oauth2/callback","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025815258469","endTimeUnixNano":"1735335025819513500","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=bVhDeVhqYXBxRzNLS0F5dm1LVXRzUTZrUHVtSDgrcjdhQSswdmZsT3o2eE81OGJsaEgvOEY4UUVZbFV6em95elFySUNmeFJYelJ4Y0RkRWErUVdtWnc9PXwxNzM1MzM1MDI1fFr-RTRGNqnLn2IMEbk7FiyJdpSLrfG2toEU04qhst7IcWtWqDDKpQf-rvb4AOVFEyb_DCJImrMZ_JKA0yiP0S_U54-vwXFi6BfICA4m3NAFaFycuWP-DbKAQAfk9Uba9ichigqepuu4l8w3EGZwlpOR_H98gkVp8i0wUsRSi0QvJExUdEYP2R6hvWCaEF24kJZGzEgc3K8tFLuKKXF51oZzS8uTXWxZKboeoN_QAE7OJ5s2oD_oUJeQW1mU7W5l1lslB214Zv24ft7fLAbVQYJBd14DDoHfg4WZpQE8v3qIuCCMNcgCod69B1Fa-h4Glz138FEfNbnOgFjiCkW63brqiutwbtIn_Voy034oAAK95qCQvwc0B5RCf2peV7-rb3IrZk6h_ABEeNeRZEkKZtsixVns0ew7rEzfUlcPFre-3D7RK0HfV66arIKuNr4qnMLBuIiDO9PuucWyY4Rft3VAp8Onv-ENIWwLVkrOGvnDFtqubfCnutmW1cqXzF1oVuckiEOwmG0k0vUCk7xR8LLEkNBkh3s3yF3F8LvZLlfesQIl-dNXAWWKIonfCyb_fLiciYxlROetfJXFkMr6LKvgCUHB4AAkx_X--KpHabCpBVnFrEIPv2E59D-eLo7J9Sf4Yhd6nDzdAgj5B4BgDPKSLWvAvjRe5-kD2AT2AGcG-pIBVuKZoI7avNO0GJM23k0-b6_djF13Cg8%3D"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"uV3raBlyRYI=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /foo","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025820715914","endTimeUnixNano":"1735335025823340045","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"net.peer.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}},{"key":"http.response_content_length","value":{"intValue":"1450"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"I9tnFqm43jY=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /.pomerium/sign_in","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025823362446","endTimeUnixNano":"1735335025825936897","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=PUoMiJkZMinjfOP-kixkfrqo8r54-oHR-9Z-N1_VU5I%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-985db05505a0159d-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-b95deb6819724582-01"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"jqKetiY96Q0=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025825951567","endTimeUnixNano":"1735335025826191870","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=TUQrZ3dFSUpycGJ5TFp4c09pdGkvMDB3RWhsUkdLVEZ4Ni9OYmZOMWJuTG5xTlI3OE4vNjdQd0I4cGFSUU1QOUFTWXVqOEs4Z3lMenJhaks4ejRIdVE9PXwxNzM1MzM1MDI1fNlsbQS7GploSTVB0WfdtdH7gi0aLdGkm-Zw-kVVwqJF9rJZSPHGDEYpgyyhScfl32qa-0GvpPpCEPxXfroMBfIx0VSczgLJty1vJ_7-Wvuc_6EdlHR9x544l5p1a7zUqdftP975Xmdcj7tkJwsje3ttJ5uVESkFOy6sN0PAyQihwcYPd0aCvVnLPgVqroEbrbkPcVavCRI8sVE0Awx0X2YzkgBPWzKHmH43iAie-kM0uMZYFPFRtEFxFvD43PscwW-prc3qShMzp4C5cNJt2TRQCRJlXcUr_ccnecAjs3jdys3da_nFHMsbhpDgDfHaYsnL86zq_QVdjWJZFMpIEHOXQUVi2Q7PTm_n9iymHah7FPcWDaAo5BRsKr7fM4_JaHT_BsnrvLMRSV8c61gtzpBoDnIKrrvr7hzua4i1OFkR5XNRSOIy0gh1Re2ZpBFUo0H8miVsqD8Hqx7N1rxFopl3R-spO3a9Arin74L-5gZu1DsgkxVRa9oF5ntJ9ZigAKCnFFRgrmHynjxD4QMxGKR0tFBrlLBYiZFlWpxMYD-bmq6jccxwA8491DQAePeeyWbLIL4-9OIsBoZqIRmJac1gou6EqxJKsECNxxLCCreNKPYcNJKWdY6ZERWT2rVYlzaYowFnfV9KDvQAoaGV_D7rg3Dw7XGnQvo5NHAYGXv5sW7LB-vlPU_5CuVjV2bwmwcy6Ybez22aD1g%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"512"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"X83U/uv8a/A=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: POST /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025826262341","endTimeUnixNano":"1735335025826593365","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=TUQrZ3dFSUpycGJ5TFp4c09pdGkvMDB3RWhsUkdLVEZ4Ni9OYmZOMWJuTG5xTlI3OE4vNjdQd0I4cGFSUU1QOUFTWXVqOEs4Z3lMenJhaks4ejRIdVE9PXwxNzM1MzM1MDI1fNlsbQS7GploSTVB0WfdtdH7gi0aLdGkm-Zw-kVVwqJF9rJZSPHGDEYpgyyhScfl32qa-0GvpPpCEPxXfroMBfIx0VSczgLJty1vJ_7-Wvuc_6EdlHR9x544l5p1a7zUqdftP975Xmdcj7tkJwsje3ttJ5uVESkFOy6sN0PAyQihwcYPd0aCvVnLPgVqroEbrbkPcVavCRI8sVE0Awx0X2YzkgBPWzKHmH43iAie-kM0uMZYFPFRtEFxFvD43PscwW-prc3qShMzp4C5cNJt2TRQCRJlXcUr_ccnecAjs3jdys3da_nFHMsbhpDgDfHaYsnL86zq_QVdjWJZFMpIEHOXQUVi2Q7PTm_n9iymHah7FPcWDaAo5BRsKr7fM4_JaHT_BsnrvLMRSV8c61gtzpBoDnIKrrvr7hzua4i1OFkR5XNRSOIy0gh1Re2ZpBFUo0H8miVsqD8Hqx7N1rxFopl3R-spO3a9Arin74L-5gZu1DsgkxVRa9oF5ntJ9ZigAKCnFFRgrmHynjxD4QMxGKR0tFBrlLBYiZFlWpxMYD-bmq6jccxwA8491DQAePeeyWbLIL4-9OIsBoZqIRmJac1gou6EqxJKsECNxxLCCreNKPYcNJKWdY6ZERWT2rVYlzaYowFnfV9KDvQAoaGV_D7rg3Dw7XGnQvo5NHAYGXv5sW7LB-vlPU_5CuVjV2bwmwcy6Ybez22aD1g%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"wsNTIF8DnsE=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oauth2/callback","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025826614025","endTimeUnixNano":"1735335025830887906","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=TUQrZ3dFSUpycGJ5TFp4c09pdGkvMDB3RWhsUkdLVEZ4Ni9OYmZOMWJuTG5xTlI3OE4vNjdQd0I4cGFSUU1QOUFTWXVqOEs4Z3lMenJhaks4ejRIdVE9PXwxNzM1MzM1MDI1fNlsbQS7GploSTVB0WfdtdH7gi0aLdGkm-Zw-kVVwqJF9rJZSPHGDEYpgyyhScfl32qa-0GvpPpCEPxXfroMBfIx0VSczgLJty1vJ_7-Wvuc_6EdlHR9x544l5p1a7zUqdftP975Xmdcj7tkJwsje3ttJ5uVESkFOy6sN0PAyQihwcYPd0aCvVnLPgVqroEbrbkPcVavCRI8sVE0Awx0X2YzkgBPWzKHmH43iAie-kM0uMZYFPFRtEFxFvD43PscwW-prc3qShMzp4C5cNJt2TRQCRJlXcUr_ccnecAjs3jdys3da_nFHMsbhpDgDfHaYsnL86zq_QVdjWJZFMpIEHOXQUVi2Q7PTm_n9iymHah7FPcWDaAo5BRsKr7fM4_JaHT_BsnrvLMRSV8c61gtzpBoDnIKrrvr7hzua4i1OFkR5XNRSOIy0gh1Re2ZpBFUo0H8miVsqD8Hqx7N1rxFopl3R-spO3a9Arin74L-5gZu1DsgkxVRa9oF5ntJ9ZigAKCnFFRgrmHynjxD4QMxGKR0tFBrlLBYiZFlWpxMYD-bmq6jccxwA8491DQAePeeyWbLIL4-9OIsBoZqIRmJac1gou6EqxJKsECNxxLCCreNKPYcNJKWdY6ZERWT2rVYlzaYowFnfV9KDvQAoaGV_D7rg3Dw7XGnQvo5NHAYGXv5sW7LB-vlPU_5CuVjV2bwmwcy6Ybez22aD1g%3D"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"wQTpo0tPYe8=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /foo","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025833577948","endTimeUnixNano":"1735335025835834705","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"net.peer.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}},{"key":"http.response_content_length","value":{"intValue":"1450"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"dQPCN0PsSTM=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /.pomerium/sign_in","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025835850156","endTimeUnixNano":"1735335025838314955","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=AwwHIua79vR9AvSBdLu7fXeMAAtUCvoAW35-81UtSuE%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-0f614b85a586f0da-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-c104e9a34b4f61ef-01"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"pOYMzXr9GH0=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025838326145","endTimeUnixNano":"1735335025838546248","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=djBzMm5teGFzNDQxSlorL2ZYVUN2Nk1XVHAraVFnSjhtU01IRGRlU0czSm8zRUlsM296bjlEc0o4VVhXSHFPOTd3QnlDVEhtSlp1dElXS3ExOWx5dVE9PXwxNzM1MzM1MDI1fPYv0au-VRFGwhaPnlvMz_uEJ22N0pzR8VwFreItQnBEo4C5PRNtMUFyKPuWzxHL67dfMAFNXNaYKrpHwE-RshBnJbE4oxQd6W705zHKAwm2lJsMMMHJkHZmBUk_Tghkuj7AsIlCKQHAG4Dd3NuGBjqd3Kqs9kwin5JnZat8aa2-5KX5Lk4QUFo_sRMebvIARsqK2QubYU114RUM62nzm_HmkKMcvh1QsGQ1s7pr_QepoDs7hSj_oXUs_Tc7c9n97sBGwWXfGc4yKLdodqyubc31qdSmHxcU5I8-FEFbbhMWZ3Pk51N7sMOjoYoP2oupL6d5DSf9bCermdQNAm4UElB1aVY0l_rmg282JhxP_5qBCLAml8xhGHT6JxMHAKA8H6MpNBusSbcfPmfuO_px6ipSnKE9ZsmY4y8_nbMZUaIotKED5NJn9YUlKDgrI58R050HTSbv-zy-rBVpLRwFhk2aC5dhYSyxVc6QxEYIrKvnVyouumm4bvB9xankZB_A1ESpBrAPszKOoznYh1y72LR8P18o7tOvUWxonfsKbnNeZ92RZOOIeUcaJ6-ZYUVO4Os5DrL4QSTByrikq4hdsABQHu89omt_6iS2t_salQ6eVrGQLI9E9yHzoDTVzWsYf8DZ9NbpI9OHMBFB7nJ8DDVm7XXvpgKtRezh2JVu9C3BZHLswq_0dJrE2hH30U1_pXy4mAOjddNCVt4%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"512"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"iIl8PJ7gVyQ=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: POST /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025838618409","endTimeUnixNano":"1735335025838948772","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=djBzMm5teGFzNDQxSlorL2ZYVUN2Nk1XVHAraVFnSjhtU01IRGRlU0czSm8zRUlsM296bjlEc0o4VVhXSHFPOTd3QnlDVEhtSlp1dElXS3ExOWx5dVE9PXwxNzM1MzM1MDI1fPYv0au-VRFGwhaPnlvMz_uEJ22N0pzR8VwFreItQnBEo4C5PRNtMUFyKPuWzxHL67dfMAFNXNaYKrpHwE-RshBnJbE4oxQd6W705zHKAwm2lJsMMMHJkHZmBUk_Tghkuj7AsIlCKQHAG4Dd3NuGBjqd3Kqs9kwin5JnZat8aa2-5KX5Lk4QUFo_sRMebvIARsqK2QubYU114RUM62nzm_HmkKMcvh1QsGQ1s7pr_QepoDs7hSj_oXUs_Tc7c9n97sBGwWXfGc4yKLdodqyubc31qdSmHxcU5I8-FEFbbhMWZ3Pk51N7sMOjoYoP2oupL6d5DSf9bCermdQNAm4UElB1aVY0l_rmg282JhxP_5qBCLAml8xhGHT6JxMHAKA8H6MpNBusSbcfPmfuO_px6ipSnKE9ZsmY4y8_nbMZUaIotKED5NJn9YUlKDgrI58R050HTSbv-zy-rBVpLRwFhk2aC5dhYSyxVc6QxEYIrKvnVyouumm4bvB9xankZB_A1ESpBrAPszKOoznYh1y72LR8P18o7tOvUWxonfsKbnNeZ92RZOOIeUcaJ6-ZYUVO4Os5DrL4QSTByrikq4hdsABQHu89omt_6iS2t_salQ6eVrGQLI9E9yHzoDTVzWsYf8DZ9NbpI9OHMBFB7nJ8DDVm7XXvpgKtRezh2JVu9C3BZHLswq_0dJrE2hH30U1_pXy4mAOjddNCVt4%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Z7eWk7+izdA=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oauth2/callback","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025838961582","endTimeUnixNano":"1735335025842657227","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=djBzMm5teGFzNDQxSlorL2ZYVUN2Nk1XVHAraVFnSjhtU01IRGRlU0czSm8zRUlsM296bjlEc0o4VVhXSHFPOTd3QnlDVEhtSlp1dElXS3ExOWx5dVE9PXwxNzM1MzM1MDI1fPYv0au-VRFGwhaPnlvMz_uEJ22N0pzR8VwFreItQnBEo4C5PRNtMUFyKPuWzxHL67dfMAFNXNaYKrpHwE-RshBnJbE4oxQd6W705zHKAwm2lJsMMMHJkHZmBUk_Tghkuj7AsIlCKQHAG4Dd3NuGBjqd3Kqs9kwin5JnZat8aa2-5KX5Lk4QUFo_sRMebvIARsqK2QubYU114RUM62nzm_HmkKMcvh1QsGQ1s7pr_QepoDs7hSj_oXUs_Tc7c9n97sBGwWXfGc4yKLdodqyubc31qdSmHxcU5I8-FEFbbhMWZ3Pk51N7sMOjoYoP2oupL6d5DSf9bCermdQNAm4UElB1aVY0l_rmg282JhxP_5qBCLAml8xhGHT6JxMHAKA8H6MpNBusSbcfPmfuO_px6ipSnKE9ZsmY4y8_nbMZUaIotKED5NJn9YUlKDgrI58R050HTSbv-zy-rBVpLRwFhk2aC5dhYSyxVc6QxEYIrKvnVyouumm4bvB9xankZB_A1ESpBrAPszKOoznYh1y72LR8P18o7tOvUWxonfsKbnNeZ92RZOOIeUcaJ6-ZYUVO4Os5DrL4QSTByrikq4hdsABQHu89omt_6iS2t_salQ6eVrGQLI9E9yHzoDTVzWsYf8DZ9NbpI9OHMBFB7nJ8DDVm7XXvpgKtRezh2JVu9C3BZHLswq_0dJrE2hH30U1_pXy4mAOjddNCVt4%3D"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Wmt35B+x3Uo=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /foo","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025846420282","endTimeUnixNano":"1735335025848745650","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"net.peer.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}},{"key":"http.response_content_length","value":{"intValue":"1450"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"yYgi/sxRPQ8=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /.pomerium/sign_in","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025848760220","endTimeUnixNano":"1735335025851122848","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=KuYlGzyljFFgiYixx3bBF6bdlA-O1S065hn1lQuRHgM%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-6e31980f6e94316e-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5a6b77e41fb1dd4a-01"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"yabq+U/JonU=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025851145408","endTimeUnixNano":"1735335025851369911","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=K05oR213S1ZvdUlZMlh0TDFaRlU3SUlZdmc4OGIxOUFoanBDaXkvNlpZc3ZUeklnc0VQMm1CYjFGYkYrK3ZYdXpnNkNtYS9MZUtleU9DY3NMN0VNUUE9PXwxNzM1MzM1MDI1fF-scOL_CXRp0kDbE4ZjdGwJnPC2MQY02ucYYTtu54IwwWu6QgQgYZ-L1nC0_8TPDlspFKyI_65zC2CAiEPwji6gGkC65hOD0mDcjWFyaMY8NgZJYvackeVgm0tLWY0KPOyJmJEk3oGmXGYlPoPJ_sz8ywot5peE8tV3_7FUgL2k61nEBRmqffg_ICtN21fxrk63NHmQZBw5E6_fj_DSwMT0XeLiu3I2sJSMP8gAXgZ-9KNAU34h9NmBwAqkyCJ5HP7N0cYwc0wCWoZVO8kgI_MrasJvIyvf6UKnp9oBDp8Ro4pTlrukgQgcj1-Wu2x6pCl9xpBT32Ftwl4Vuvr_zI7V6MoqZMKywDBNfsApHUkZTUedX0ZLK9A29RXEVkNqr9OevFVyVsMyOvTZ_tfS7_5hfwzHjyOnHq7q2COA1ocRzWCMaXtIeSk8WwL_K6hCf9vNrpSi0_agf2JWLbefcjy8AUAWjpp6jt4My9JhdRZ-qBmnL973LPUmyiIbYkC0Km1nIe71rItMLqx2SZCwuc8-9jq7DshSiSNzNqMD35PGdiiEawIhMFJwhWRWKCDMPBCYeAkYIODnfJWGBbOrD7kFReqvfO9Jm2zbuaw9tPw6HCHWY8XwIyAougw7FP15F9Vqtzc4imuTmsvkOYJNTvI8lz8Sk2ALwlPo63QbI9DFjlE_uDuGb-HqiUqe0eoIx4EqP9Ni_Qj_AkY%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"512"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"zFaBQK23WvM=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: POST /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025851442912","endTimeUnixNano":"1735335025851703335","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=K05oR213S1ZvdUlZMlh0TDFaRlU3SUlZdmc4OGIxOUFoanBDaXkvNlpZc3ZUeklnc0VQMm1CYjFGYkYrK3ZYdXpnNkNtYS9MZUtleU9DY3NMN0VNUUE9PXwxNzM1MzM1MDI1fF-scOL_CXRp0kDbE4ZjdGwJnPC2MQY02ucYYTtu54IwwWu6QgQgYZ-L1nC0_8TPDlspFKyI_65zC2CAiEPwji6gGkC65hOD0mDcjWFyaMY8NgZJYvackeVgm0tLWY0KPOyJmJEk3oGmXGYlPoPJ_sz8ywot5peE8tV3_7FUgL2k61nEBRmqffg_ICtN21fxrk63NHmQZBw5E6_fj_DSwMT0XeLiu3I2sJSMP8gAXgZ-9KNAU34h9NmBwAqkyCJ5HP7N0cYwc0wCWoZVO8kgI_MrasJvIyvf6UKnp9oBDp8Ro4pTlrukgQgcj1-Wu2x6pCl9xpBT32Ftwl4Vuvr_zI7V6MoqZMKywDBNfsApHUkZTUedX0ZLK9A29RXEVkNqr9OevFVyVsMyOvTZ_tfS7_5hfwzHjyOnHq7q2COA1ocRzWCMaXtIeSk8WwL_K6hCf9vNrpSi0_agf2JWLbefcjy8AUAWjpp6jt4My9JhdRZ-qBmnL973LPUmyiIbYkC0Km1nIe71rItMLqx2SZCwuc8-9jq7DshSiSNzNqMD35PGdiiEawIhMFJwhWRWKCDMPBCYeAkYIODnfJWGBbOrD7kFReqvfO9Jm2zbuaw9tPw6HCHWY8XwIyAougw7FP15F9Vqtzc4imuTmsvkOYJNTvI8lz8Sk2ALwlPo63QbI9DFjlE_uDuGb-HqiUqe0eoIx4EqP9Ni_Qj_AkY%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"xa5A0dmcNOo=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oauth2/callback","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025851714055","endTimeUnixNano":"1735335025856269410","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=K05oR213S1ZvdUlZMlh0TDFaRlU3SUlZdmc4OGIxOUFoanBDaXkvNlpZc3ZUeklnc0VQMm1CYjFGYkYrK3ZYdXpnNkNtYS9MZUtleU9DY3NMN0VNUUE9PXwxNzM1MzM1MDI1fF-scOL_CXRp0kDbE4ZjdGwJnPC2MQY02ucYYTtu54IwwWu6QgQgYZ-L1nC0_8TPDlspFKyI_65zC2CAiEPwji6gGkC65hOD0mDcjWFyaMY8NgZJYvackeVgm0tLWY0KPOyJmJEk3oGmXGYlPoPJ_sz8ywot5peE8tV3_7FUgL2k61nEBRmqffg_ICtN21fxrk63NHmQZBw5E6_fj_DSwMT0XeLiu3I2sJSMP8gAXgZ-9KNAU34h9NmBwAqkyCJ5HP7N0cYwc0wCWoZVO8kgI_MrasJvIyvf6UKnp9oBDp8Ro4pTlrukgQgcj1-Wu2x6pCl9xpBT32Ftwl4Vuvr_zI7V6MoqZMKywDBNfsApHUkZTUedX0ZLK9A29RXEVkNqr9OevFVyVsMyOvTZ_tfS7_5hfwzHjyOnHq7q2COA1ocRzWCMaXtIeSk8WwL_K6hCf9vNrpSi0_agf2JWLbefcjy8AUAWjpp6jt4My9JhdRZ-qBmnL973LPUmyiIbYkC0Km1nIe71rItMLqx2SZCwuc8-9jq7DshSiSNzNqMD35PGdiiEawIhMFJwhWRWKCDMPBCYeAkYIODnfJWGBbOrD7kFReqvfO9Jm2zbuaw9tPw6HCHWY8XwIyAougw7FP15F9Vqtzc4imuTmsvkOYJNTvI8lz8Sk2ALwlPo63QbI9DFjlE_uDuGb-HqiUqe0eoIx4EqP9Ni_Qj_AkY%3D"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"iwk4YAWR/kU=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /foo","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025868112882","endTimeUnixNano":"1735335025870819804","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"net.peer.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}},{"key":"http.response_content_length","value":{"intValue":"1450"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ChvrQ2kjOv4=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /.pomerium/sign_in","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025870834974","endTimeUnixNano":"1735335025873259603","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=NfuC-nJofCctq_72Lvl602hVWbnwI3s-MmMAYICg8RM%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-9690e8cd7247b772-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-8b0938600591fe45-01"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"mOp99KCsZuE=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025873277763","endTimeUnixNano":"1735335025873525176","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=NlpEbWhsdDRlbm1nNEVtSWR0U1BsY3RWU3kyRTNvR09DK3o1NnRLUEtGNCtCNUk5NmE0dUE2N01KM0xkdnk2WGgwTjN1eGQ2cG1rLzdweE4wc1JCbFE9PXwxNzM1MzM1MDI1fNPhIc3ToLHpvS_Gvc8aQRaU88cZYSsnQK8fP27HBXGJKzuRaASTXkCUMwIy2bfmT5GrsBxx8-MmupZH8PE_9xtv96G9GEbRgrrFdukuhrQei_80gU6OgUoTwTtCC9jC8T37myP2Oy72fpFGDoXxzEzRGhSyMAHQJy_HBv3PLX-UxHuResc5NH_Yj7aDx16xDkDAWmc-S1VeeUW0zqT8HgnzfXFYdZqf2_tnV12NUp-CLGXLRsU-3XJIsSwy5CRy1mfIGspxD-SxBHxPaRjKMY9FMxc4CsDmR4eiF28glstJrheqExh9wDhG0TFmgwLtEJ1Vq_hLTmV3RZO8jx8ZtdV1id1slzj8fwzRXH-IVKgTJ1uLcA28LLI1ZzJzD2PrTkfISjg6-5nUJv-OYQcQlQI21pxL1s4ICAa0l3KzE5i5_yeymQaGeAfSLIJ3-5RTTwBMrx5zgMlco96WRrJZ604XqQSOgiGoxSLA9aHu__QOK9OU7GFdga92Y67T3vkKwvWXeX3w2EAF__tzDF5dy--oGborqKGcv0fkKeMgRyfxkxJ-E1w0Z7ppcwL6LL1q78zKI8JcLzgV8MyVDcVnHRrkIToVZ5YYavQnbXK_7lfo_XbKyifs4ZqPmwNY0WyGZ0j_ZZTOLRo8lQR2y5NOlTZRWCz3sqkmOJ3SeAoGrHiQ0tZDfFgvO99ucbtQj1_PCyO6u2-JaIpH6fg%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"512"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"xm+TiiRVATY=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: POST /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025873587166","endTimeUnixNano":"1735335025873885981","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=NlpEbWhsdDRlbm1nNEVtSWR0U1BsY3RWU3kyRTNvR09DK3o1NnRLUEtGNCtCNUk5NmE0dUE2N01KM0xkdnk2WGgwTjN1eGQ2cG1rLzdweE4wc1JCbFE9PXwxNzM1MzM1MDI1fNPhIc3ToLHpvS_Gvc8aQRaU88cZYSsnQK8fP27HBXGJKzuRaASTXkCUMwIy2bfmT5GrsBxx8-MmupZH8PE_9xtv96G9GEbRgrrFdukuhrQei_80gU6OgUoTwTtCC9jC8T37myP2Oy72fpFGDoXxzEzRGhSyMAHQJy_HBv3PLX-UxHuResc5NH_Yj7aDx16xDkDAWmc-S1VeeUW0zqT8HgnzfXFYdZqf2_tnV12NUp-CLGXLRsU-3XJIsSwy5CRy1mfIGspxD-SxBHxPaRjKMY9FMxc4CsDmR4eiF28glstJrheqExh9wDhG0TFmgwLtEJ1Vq_hLTmV3RZO8jx8ZtdV1id1slzj8fwzRXH-IVKgTJ1uLcA28LLI1ZzJzD2PrTkfISjg6-5nUJv-OYQcQlQI21pxL1s4ICAa0l3KzE5i5_yeymQaGeAfSLIJ3-5RTTwBMrx5zgMlco96WRrJZ604XqQSOgiGoxSLA9aHu__QOK9OU7GFdga92Y67T3vkKwvWXeX3w2EAF__tzDF5dy--oGborqKGcv0fkKeMgRyfxkxJ-E1w0Z7ppcwL6LL1q78zKI8JcLzgV8MyVDcVnHRrkIToVZ5YYavQnbXK_7lfo_XbKyifs4ZqPmwNY0WyGZ0j_ZZTOLRo8lQR2y5NOlTZRWCz3sqkmOJ3SeAoGrHiQ0tZDfFgvO99ucbtQj1_PCyO6u2-JaIpH6fg%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"DtfBvkQHXZs=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oauth2/callback","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025873931101","endTimeUnixNano":"1735335025877578894","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=NlpEbWhsdDRlbm1nNEVtSWR0U1BsY3RWU3kyRTNvR09DK3o1NnRLUEtGNCtCNUk5NmE0dUE2N01KM0xkdnk2WGgwTjN1eGQ2cG1rLzdweE4wc1JCbFE9PXwxNzM1MzM1MDI1fNPhIc3ToLHpvS_Gvc8aQRaU88cZYSsnQK8fP27HBXGJKzuRaASTXkCUMwIy2bfmT5GrsBxx8-MmupZH8PE_9xtv96G9GEbRgrrFdukuhrQei_80gU6OgUoTwTtCC9jC8T37myP2Oy72fpFGDoXxzEzRGhSyMAHQJy_HBv3PLX-UxHuResc5NH_Yj7aDx16xDkDAWmc-S1VeeUW0zqT8HgnzfXFYdZqf2_tnV12NUp-CLGXLRsU-3XJIsSwy5CRy1mfIGspxD-SxBHxPaRjKMY9FMxc4CsDmR4eiF28glstJrheqExh9wDhG0TFmgwLtEJ1Vq_hLTmV3RZO8jx8ZtdV1id1slzj8fwzRXH-IVKgTJ1uLcA28LLI1ZzJzD2PrTkfISjg6-5nUJv-OYQcQlQI21pxL1s4ICAa0l3KzE5i5_yeymQaGeAfSLIJ3-5RTTwBMrx5zgMlco96WRrJZ604XqQSOgiGoxSLA9aHu__QOK9OU7GFdga92Y67T3vkKwvWXeX3w2EAF__tzDF5dy--oGborqKGcv0fkKeMgRyfxkxJ-E1w0Z7ppcwL6LL1q78zKI8JcLzgV8MyVDcVnHRrkIToVZ5YYavQnbXK_7lfo_XbKyifs4ZqPmwNY0WyGZ0j_ZZTOLRo8lQR2y5NOlTZRWCz3sqkmOJ3SeAoGrHiQ0tZDfFgvO99ucbtQj1_PCyO6u2-JaIpH6fg%3D"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"YEKY6t1fm8A=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /foo","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025887526993","endTimeUnixNano":"1735335025889954213","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"net.peer.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}},{"key":"http.response_content_length","value":{"intValue":"1450"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"X170o5Ll3/I=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /.pomerium/sign_in","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025889968233","endTimeUnixNano":"1735335025892715716","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=dqScDf0iAzuxtV8kOVFuK6LkRdrngMgT7ZF5WZSWxrM%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-75518e62ed861ee4-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-604298eadd5f9bc0-01"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"CMjIXZ4ROSY=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025892732676","endTimeUnixNano":"1735335025892959438","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=eE9BRU5YQUZpTzFaV0pPK28rTmF0NU1xempEV1NzU3hhM1lDL1I2Zm5KNFRkM0NPd3RQY2wxZDAvVVFJaVB1MTN6enlwa1h1NDFaZmRHZGFIdFQxVlE9PXwxNzM1MzM1MDI1fEw2htjJsKXWR0ubmlecrlxxyWaM5lyrfQKVov8UW14H8_5YI7twjd0qGPWd8uLPpHaUpE5zR5PVk37RnPpkNC2J5NteJv5IN51QbBerrsuGuDZ650Sbqr9-p9FXOKmBBcoKRPW_0N-B7PWMWkQIsY5rVbjYeXgfOjuadl0PNkCA0awKZQhOwfJwr8waX88ypxsRszpU26pXB75zl5tO-AOtrROKkjsv_Y0i-Vvvhj7VGlRPsJ2xvNqK9DuJLbdjmyKrSutz1IsGwhs6NWyd82yaid7x7qCob3VkStVsNeLhjcniP9xjD22Hbi50itMD1HyXCTu_hAVyEYw2SHC842nad-4tP1EEvQkB05PBwTT5G324_lbicp6lFCSzGT9rS2MPEuz12m8Uq9hdJEj18am1pZM7vBXgevH6K0mwlpb4eIgqceOwxbgSHeGw07op-0jdeGnbz1REdKGGui4pWW5PmxcCjDiAG6g6kP6DrXUhPvAVXPK3GhpK4lxuLfU84zuQr81Xo0EmYOxQGjvdcYepBDn9anS1tEghjCk7iptxM6zO11Q7vMFI9wcVHrNl3sSCAM2k21BFmPEyWcnWvPFsQMsegbSznjD2X52WjB7k19H406-bHN-d5Py0dcMo185Y1aAIZ3m2CyXNMgPlrBPT-5XDSvf8113VVpTk8Yazr1rx0DzKlmLhWaUo0Nxm9mPwxK17XRvpdSg%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"512"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"HWuw3R1htnc=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: POST /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025893017369","endTimeUnixNano":"1735335025893320422","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=eE9BRU5YQUZpTzFaV0pPK28rTmF0NU1xempEV1NzU3hhM1lDL1I2Zm5KNFRkM0NPd3RQY2wxZDAvVVFJaVB1MTN6enlwa1h1NDFaZmRHZGFIdFQxVlE9PXwxNzM1MzM1MDI1fEw2htjJsKXWR0ubmlecrlxxyWaM5lyrfQKVov8UW14H8_5YI7twjd0qGPWd8uLPpHaUpE5zR5PVk37RnPpkNC2J5NteJv5IN51QbBerrsuGuDZ650Sbqr9-p9FXOKmBBcoKRPW_0N-B7PWMWkQIsY5rVbjYeXgfOjuadl0PNkCA0awKZQhOwfJwr8waX88ypxsRszpU26pXB75zl5tO-AOtrROKkjsv_Y0i-Vvvhj7VGlRPsJ2xvNqK9DuJLbdjmyKrSutz1IsGwhs6NWyd82yaid7x7qCob3VkStVsNeLhjcniP9xjD22Hbi50itMD1HyXCTu_hAVyEYw2SHC842nad-4tP1EEvQkB05PBwTT5G324_lbicp6lFCSzGT9rS2MPEuz12m8Uq9hdJEj18am1pZM7vBXgevH6K0mwlpb4eIgqceOwxbgSHeGw07op-0jdeGnbz1REdKGGui4pWW5PmxcCjDiAG6g6kP6DrXUhPvAVXPK3GhpK4lxuLfU84zuQr81Xo0EmYOxQGjvdcYepBDn9anS1tEghjCk7iptxM6zO11Q7vMFI9wcVHrNl3sSCAM2k21BFmPEyWcnWvPFsQMsegbSznjD2X52WjB7k19H406-bHN-d5Py0dcMo185Y1aAIZ3m2CyXNMgPlrBPT-5XDSvf8113VVpTk8Yazr1rx0DzKlmLhWaUo0Nxm9mPwxK17XRvpdSg%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"BDPsEOd6+nI=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oauth2/callback","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025893331733","endTimeUnixNano":"1735335025896943046","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=eE9BRU5YQUZpTzFaV0pPK28rTmF0NU1xempEV1NzU3hhM1lDL1I2Zm5KNFRkM0NPd3RQY2wxZDAvVVFJaVB1MTN6enlwa1h1NDFaZmRHZGFIdFQxVlE9PXwxNzM1MzM1MDI1fEw2htjJsKXWR0ubmlecrlxxyWaM5lyrfQKVov8UW14H8_5YI7twjd0qGPWd8uLPpHaUpE5zR5PVk37RnPpkNC2J5NteJv5IN51QbBerrsuGuDZ650Sbqr9-p9FXOKmBBcoKRPW_0N-B7PWMWkQIsY5rVbjYeXgfOjuadl0PNkCA0awKZQhOwfJwr8waX88ypxsRszpU26pXB75zl5tO-AOtrROKkjsv_Y0i-Vvvhj7VGlRPsJ2xvNqK9DuJLbdjmyKrSutz1IsGwhs6NWyd82yaid7x7qCob3VkStVsNeLhjcniP9xjD22Hbi50itMD1HyXCTu_hAVyEYw2SHC842nad-4tP1EEvQkB05PBwTT5G324_lbicp6lFCSzGT9rS2MPEuz12m8Uq9hdJEj18am1pZM7vBXgevH6K0mwlpb4eIgqceOwxbgSHeGw07op-0jdeGnbz1REdKGGui4pWW5PmxcCjDiAG6g6kP6DrXUhPvAVXPK3GhpK4lxuLfU84zuQr81Xo0EmYOxQGjvdcYepBDn9anS1tEghjCk7iptxM6zO11Q7vMFI9wcVHrNl3sSCAM2k21BFmPEyWcnWvPFsQMsegbSznjD2X52WjB7k19H406-bHN-d5Py0dcMo185Y1aAIZ3m2CyXNMgPlrBPT-5XDSvf8113VVpTk8Yazr1rx0DzKlmLhWaUo0Nxm9mPwxK17XRvpdSg%3D"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"YelxH0HhuCs=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /foo","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025932967647","endTimeUnixNano":"1735335025935240514","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"net.peer.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}},{"key":"http.response_content_length","value":{"intValue":"1450"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"NEEa6gf13GY=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /.pomerium/sign_in","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025935254354","endTimeUnixNano":"1735335025937528801","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=1DclRi_1pUHTYKNXCgmLO7xIgxsFHQ6PnZo6vg2QNpc%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-e310a150afa55100-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-61e9711f41e1b82b-01"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"rjRgfziAtxI=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025937538181","endTimeUnixNano":"1735335025937811784","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=Vld4RWw1Q2pKNkZ2NXNST3pzVTdFMi9qSTNMY0E3bm91bDNYUDNyNWVJbUMrekFzSW5WejIySEtxclJscnBvUkkvVWY1RStubmcrT1g3S1llcklSUWc9PXwxNzM1MzM1MDI1fCZPSA2y7mrm6xOOpuWELdtqzK8AmnUFMhpMijy4u4IE5SwEboZafPLVOuG6XLO-hnn5Xf2lTXraFGd6UpqWGelZT7VfHlpWPhDdf2uCYIAj1u6TYRgPS3_hMilmsO6AvSWmX37-XsPL4I78VhCqbL2FoEKz_CgyCAVhowdooxX6INcvZ7-G8BCJX16jnZKu7o3gX8xFl0mrHYKV1EAPY1V2YsxRTGRS2ptkcqEuxXw9YVOQIIsC6OsphlbN91V6qHGhzBRcLX8CZ-8E_2D9Osgiy4-C-koOCtzRLLFJy9S6oq-8Z1c3PK_wGhTu-syftdKFKjKjdRo97DWgjVz_lnKWeYC2usX32gaExGKkVkMERCaQ6JKoi-TXo4kY1g-AywEfn_gehBqAqzlXQ5VHEDdCth5C2BHjOhgUKcDv7W4Oqmp_93_U28elEz4Q72JUpSaDDe1fKA_eArNUrQBJTJ6N6bABEwL52xkh0ifRwWcERMCwnc1bOlAIDs2T1yWW27hdYrpSOZbAwdKuSji2x1ut0-DqgnGoDMMqzgS98MUPsC9I3PNltfu_DgC959VjH7H6_HmE2q1nPELeZUXRCGzAZTOFE23bD0ks6uyipS3jaVb1qxVClG3VMuSLf9SdV2ESH7MWJaXGyvhN8c1KBsiZp5L-7gqCnsZOMYQrJNQIsbdic-DkeRlmQYbs8iZykp-cADl2uTKek-0%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"512"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"AxuIpomNTe4=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: POST /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025939701887","endTimeUnixNano":"1735335025941702821","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=Vld4RWw1Q2pKNkZ2NXNST3pzVTdFMi9qSTNMY0E3bm91bDNYUDNyNWVJbUMrekFzSW5WejIySEtxclJscnBvUkkvVWY1RStubmcrT1g3S1llcklSUWc9PXwxNzM1MzM1MDI1fCZPSA2y7mrm6xOOpuWELdtqzK8AmnUFMhpMijy4u4IE5SwEboZafPLVOuG6XLO-hnn5Xf2lTXraFGd6UpqWGelZT7VfHlpWPhDdf2uCYIAj1u6TYRgPS3_hMilmsO6AvSWmX37-XsPL4I78VhCqbL2FoEKz_CgyCAVhowdooxX6INcvZ7-G8BCJX16jnZKu7o3gX8xFl0mrHYKV1EAPY1V2YsxRTGRS2ptkcqEuxXw9YVOQIIsC6OsphlbN91V6qHGhzBRcLX8CZ-8E_2D9Osgiy4-C-koOCtzRLLFJy9S6oq-8Z1c3PK_wGhTu-syftdKFKjKjdRo97DWgjVz_lnKWeYC2usX32gaExGKkVkMERCaQ6JKoi-TXo4kY1g-AywEfn_gehBqAqzlXQ5VHEDdCth5C2BHjOhgUKcDv7W4Oqmp_93_U28elEz4Q72JUpSaDDe1fKA_eArNUrQBJTJ6N6bABEwL52xkh0ifRwWcERMCwnc1bOlAIDs2T1yWW27hdYrpSOZbAwdKuSji2x1ut0-DqgnGoDMMqzgS98MUPsC9I3PNltfu_DgC959VjH7H6_HmE2q1nPELeZUXRCGzAZTOFE23bD0ks6uyipS3jaVb1qxVClG3VMuSLf9SdV2ESH7MWJaXGyvhN8c1KBsiZp5L-7gqCnsZOMYQrJNQIsbdic-DkeRlmQYbs8iZykp-cADl2uTKek-0%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Sn6DR8I82JI=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oauth2/callback","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025941731151","endTimeUnixNano":"1735335025945621198","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=Vld4RWw1Q2pKNkZ2NXNST3pzVTdFMi9qSTNMY0E3bm91bDNYUDNyNWVJbUMrekFzSW5WejIySEtxclJscnBvUkkvVWY1RStubmcrT1g3S1llcklSUWc9PXwxNzM1MzM1MDI1fCZPSA2y7mrm6xOOpuWELdtqzK8AmnUFMhpMijy4u4IE5SwEboZafPLVOuG6XLO-hnn5Xf2lTXraFGd6UpqWGelZT7VfHlpWPhDdf2uCYIAj1u6TYRgPS3_hMilmsO6AvSWmX37-XsPL4I78VhCqbL2FoEKz_CgyCAVhowdooxX6INcvZ7-G8BCJX16jnZKu7o3gX8xFl0mrHYKV1EAPY1V2YsxRTGRS2ptkcqEuxXw9YVOQIIsC6OsphlbN91V6qHGhzBRcLX8CZ-8E_2D9Osgiy4-C-koOCtzRLLFJy9S6oq-8Z1c3PK_wGhTu-syftdKFKjKjdRo97DWgjVz_lnKWeYC2usX32gaExGKkVkMERCaQ6JKoi-TXo4kY1g-AywEfn_gehBqAqzlXQ5VHEDdCth5C2BHjOhgUKcDv7W4Oqmp_93_U28elEz4Q72JUpSaDDe1fKA_eArNUrQBJTJ6N6bABEwL52xkh0ifRwWcERMCwnc1bOlAIDs2T1yWW27hdYrpSOZbAwdKuSji2x1ut0-DqgnGoDMMqzgS98MUPsC9I3PNltfu_DgC959VjH7H6_HmE2q1nPELeZUXRCGzAZTOFE23bD0ks6uyipS3jaVb1qxVClG3VMuSLf9SdV2ESH7MWJaXGyvhN8c1KBsiZp5L-7gqCnsZOMYQrJNQIsbdic-DkeRlmQYbs8iZykp-cADl2uTKek-0%3D"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"VDHmxoPFG3g=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /foo","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025979196630","endTimeUnixNano":"1735335025983688133","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"net.peer.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}},{"key":"http.response_content_length","value":{"intValue":"1450"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"V4KDwU2b7No=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /.pomerium/sign_in","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025983702733","endTimeUnixNano":"1735335025986526338","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=ZnX70Wyz8dCAymop0VMwrObf9dt9EeGcdvdSJMWerio%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-a0e19ab5c251cf8a-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5431e6c683c51b78-01"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"zKjX115N+9k=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025986536548","endTimeUnixNano":"1735335025986763250","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=TlVucmJGSVM1TTkxQjdVeFpFTndpblBPMUpqdjBoOTBCTkN1NVBPa1RWWGkzcC9YNE1Td3RYc3IyOHZQS05HSVA5am9EbngyT0pNdzBzdEQ4Kzhrbmc9PXwxNzM1MzM1MDI1fGFr6SjNR6oY3LbAckTb1zOQwi4DF7qp73tBPnxNCM9uAN-E7benf9dM4rTw1MXWLEUpA1XTw-NdsI5DXa5mG-guOz-4MjnubljhndMmKy6H-mdPjrJwqpirLHSjGNIOMh6prOpDlt1i-RnnOB_wtIY1wxUCPntd-aCiBGfViIgB0aCMQq6YcqcJYohYyPYYILz6IczkvpPa5cevuvnCJ20J7X69_6AJ0tYzwo9PVmYrGFyTJULaNfw-2G-nUAFuP4-FHv4PexN-NB_Qn0kM4KsX6482RPpNetBM8hHfHbdlr6J_018NOc7Jpv092cxDmMpwzeTQ7mKtEgntBj8WrClM6VTrkecRpdJ8yiZhLvgVhjK76JUFXlXNzYItTt-NvBKuXZ-869TmVrWADw6579M1IP2uCvQlE88v4e__LelTsipYfTzM_kN_Zl7hoDur57rTfzsbkipm5-szZ1W4wEksgtPybfnauIOJY3AtTQ1u6HSciCbGULA6N024tcnpDAJ2mOwYuYbMHlvVYpTv9AJ7P6WOuxaD9GQP5jDJb_AcmhzylwX6r7TQrHnPtbRfratJUOP85A9-5vgaKdLJ4QIPgVqPh4eyTR6xRWWkFnkJdUnJ8m8gshI0ZZI1KzM-ORzBPl8TrEFfqrHHVq1ujN7JQknxUgbrWQOU-jzncZnCYk6TBMbyFX7SdKMG8rgbLaDdnANGnoVCiuA%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"512"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Jz4teHxKmjI=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: POST /oidc/auth","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025986818651","endTimeUnixNano":"1735335025987117384","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=TlVucmJGSVM1TTkxQjdVeFpFTndpblBPMUpqdjBoOTBCTkN1NVBPa1RWWGkzcC9YNE1Td3RYc3IyOHZQS05HSVA5am9EbngyT0pNdzBzdEQ4Kzhrbmc9PXwxNzM1MzM1MDI1fGFr6SjNR6oY3LbAckTb1zOQwi4DF7qp73tBPnxNCM9uAN-E7benf9dM4rTw1MXWLEUpA1XTw-NdsI5DXa5mG-guOz-4MjnubljhndMmKy6H-mdPjrJwqpirLHSjGNIOMh6prOpDlt1i-RnnOB_wtIY1wxUCPntd-aCiBGfViIgB0aCMQq6YcqcJYohYyPYYILz6IczkvpPa5cevuvnCJ20J7X69_6AJ0tYzwo9PVmYrGFyTJULaNfw-2G-nUAFuP4-FHv4PexN-NB_Qn0kM4KsX6482RPpNetBM8hHfHbdlr6J_018NOc7Jpv092cxDmMpwzeTQ7mKtEgntBj8WrClM6VTrkecRpdJ8yiZhLvgVhjK76JUFXlXNzYItTt-NvBKuXZ-869TmVrWADw6579M1IP2uCvQlE88v4e__LelTsipYfTzM_kN_Zl7hoDur57rTfzsbkipm5-szZ1W4wEksgtPybfnauIOJY3AtTQ1u6HSciCbGULA6N024tcnpDAJ2mOwYuYbMHlvVYpTv9AJ7P6WOuxaD9GQP5jDJb_AcmhzylwX6r7TQrHnPtbRfratJUOP85A9-5vgaKdLJ4QIPgVqPh4eyTR6xRWWkFnkJdUnJ8m8gshI0ZZI1KzM-ORzBPl8TrEFfqrHHVq1ujN7JQknxUgbrWQOU-jzncZnCYk6TBMbyFX7SdKMG8rgbLaDdnANGnoVCiuA%3D"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"X0cwWL6jdNw=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /oauth2/callback","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025987145055","endTimeUnixNano":"1735335025993611052","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=TlVucmJGSVM1TTkxQjdVeFpFTndpblBPMUpqdjBoOTBCTkN1NVBPa1RWWGkzcC9YNE1Td3RYc3IyOHZQS05HSVA5am9EbngyT0pNdzBzdEQ4Kzhrbmc9PXwxNzM1MzM1MDI1fGFr6SjNR6oY3LbAckTb1zOQwi4DF7qp73tBPnxNCM9uAN-E7benf9dM4rTw1MXWLEUpA1XTw-NdsI5DXa5mG-guOz-4MjnubljhndMmKy6H-mdPjrJwqpirLHSjGNIOMh6prOpDlt1i-RnnOB_wtIY1wxUCPntd-aCiBGfViIgB0aCMQq6YcqcJYohYyPYYILz6IczkvpPa5cevuvnCJ20J7X69_6AJ0tYzwo9PVmYrGFyTJULaNfw-2G-nUAFuP4-FHv4PexN-NB_Qn0kM4KsX6482RPpNetBM8hHfHbdlr6J_018NOc7Jpv092cxDmMpwzeTQ7mKtEgntBj8WrClM6VTrkecRpdJ8yiZhLvgVhjK76JUFXlXNzYItTt-NvBKuXZ-869TmVrWADw6579M1IP2uCvQlE88v4e__LelTsipYfTzM_kN_Zl7hoDur57rTfzsbkipm5-szZ1W4wEksgtPybfnauIOJY3AtTQ1u6HSciCbGULA6N024tcnpDAJ2mOwYuYbMHlvVYpTv9AJ7P6WOuxaD9GQP5jDJb_AcmhzylwX6r7TQrHnPtbRfratJUOP85A9-5vgaKdLJ4QIPgVqPh4eyTR6xRWWkFnkJdUnJ8m8gshI0ZZI1KzM-ORzBPl8TrEFfqrHHVq1ujN7JQknxUgbrWQOU-jzncZnCYk6TBMbyFX7SdKMG8rgbLaDdnANGnoVCiuA%3D"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"4mu5mTDooas=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /.pomerium/sign_in","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025993631222","endTimeUnixNano":"1735335025996706079","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:36063/.pomerium/sign_in?pomerium_expiry=1735335325\u0026pomerium_idp_id=M2Pt9LfWA3cI5EVZ85tXmQi7G1zsg7gftJXCyQTqtWK\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_signature=ZnX70Wyz8dCAymop0VMwrObf9dt9EeGcdvdSJMWerio%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-a0e19ab5c251cf8a-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5431e6c683c51b78-01"}},{"key":"net.peer.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"fUtVfh0g50A=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /.pomerium/callback/","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025996719560","endTimeUnixNano":"1735335025998707173","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/.pomerium/callback/?pomerium_expiry=1735335325\u0026pomerium_issued=1735335025\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A36063%2Ffoo\u0026pomerium_session_encrypted=gnpoJSWHmBQQkVfvjiOomyk9_F63PJnoFtYhIeue3g62S-0NF5d6BOpeOkgOwptEfJSQ_kKjqSYQSDT6kkqtKF1QDaX1fRCjLC9LHcOeioxnfLQ2rglnkAbs6XFscaRpZFstZ3TlJXUXf4oL9C-DwpXx96tlbHS9RzPLPijhEMnnLi3_zjw0hlHFmk8a6laRLJFuf-n9vtjh8TVnhmeiE7hic2GNbQjGfu8dqWjOvWDhN1XwmMzKGdp3RosO8xLozBy4i9CP1C35UgXeuoPvS8JPa4Y2d3V3YazDKutWLoMdmtZVicgWb80MJRuR9NX4vlscb12wNk2QpO-qxXeLn6L8UennLI8v2RM-G-TPmK7GDQ2Ihfd8ifmIdRNCHIIKpTpNGHuc6uwWmc86lQt6VP7yyn2-6F0lUtMRLSFZMmfaqcEM2I1C1XULxqi-R8W5aQgO5Wb1sFA4quUUNQy2lHanVtFceM8lOVHeIBuOaieLxFvKmf0TILOdlaYDoeNKMLk9Z_gPBEUyngeDwfD-2RimJTkeUJxQUpjCuMtfc_0aWfKg8PpeFrwnQhSBBhdasBEbplMrRZ-YwcC_IBHeLRqANNaW8pTpVaimIsTCDoMq0WQu5UXhGWT_Xbc2BQ0vNZVCKylD47DijPzYP8fQd5ibiJtElOmxwuG_hl_rPEcPQKMnoByUd6MtVvFdkfik-iGqMEEGbYfixzO3D4q5ahd13lWCkaFsajMJS9Mapde3c2V-DwS53stVswOYFnOetvMYt4koEE5mqg27zdXEUTNImTExWvpo5swL9JgGbWlGwJIrTsSLpWcD7cfb\u0026pomerium_signature=ylPA2kChpnYPWI9WUZDIU5mnENgGeREsecrSWxxxN9g%3D\u0026pomerium_traceparent=00-9827f4d83b31fdd3f0d0f5a17584e3dc-a0e19ab5c251cf8a-01\u0026pomerium_tracestate=pomerium.traceparent%3D00-9827f4d83b31fdd3f0d0f5a17584e3dc-5431e6c683c51b78-01"}},{"key":"net.peer.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"302"}},{"key":"http.response_content_length","value":{"intValue":"66"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"fxjsKYWTWL4=","parentSpanId":"X5SXMIPajt8=","flags":256,"name":"Client: GET /foo","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025998722713","endTimeUnixNano":"1735335026004195719","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"net.peer.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"36063"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"2"}}],"status":{}}]},{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"X5SXMIPajt8=","parentSpanId":"s/6I32KmWeU=","flags":256,"name":"httpUpstream.Do","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025791973471","endTimeUnixNano":"1735335026004164859","attributes":[{"key":"method","value":{"stringValue":"GET"}},{"key":"url","value":{"stringValue":"https://foo.localhost.pomerium.io:36063/foo"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/upstreams/http.go:288"}},{"key":"http.request.resend_count","value":{"intValue":"7"}}],"events":[{"timeUnixNano":"1735335025791981701","name":"creating new http client"},{"timeUnixNano":"1735335025814881765","name":"submitting form","attributes":[{"key":"location","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=bVhDeVhqYXBxRzNLS0F5dm1LVXRzUTZrUHVtSDgrcjdhQSswdmZsT3o2eE81OGJsaEgvOEY4UUVZbFV6em95elFySUNmeFJYelJ4Y0RkRWErUVdtWnc9PXwxNzM1MzM1MDI1fFr-RTRGNqnLn2IMEbk7FiyJdpSLrfG2toEU04qhst7IcWtWqDDKpQf-rvb4AOVFEyb_DCJImrMZ_JKA0yiP0S_U54-vwXFi6BfICA4m3NAFaFycuWP-DbKAQAfk9Uba9ichigqepuu4l8w3EGZwlpOR_H98gkVp8i0wUsRSi0QvJExUdEYP2R6hvWCaEF24kJZGzEgc3K8tFLuKKXF51oZzS8uTXWxZKboeoN_QAE7OJ5s2oD_oUJeQW1mU7W5l1lslB214Zv24ft7fLAbVQYJBd14DDoHfg4WZpQE8v3qIuCCMNcgCod69B1Fa-h4Glz138FEfNbnOgFjiCkW63brqiutwbtIn_Voy034oAAK95qCQvwc0B5RCf2peV7-rb3IrZk6h_ABEeNeRZEkKZtsixVns0ew7rEzfUlcPFre-3D7RK0HfV66arIKuNr4qnMLBuIiDO9PuucWyY4Rft3VAp8Onv-ENIWwLVkrOGvnDFtqubfCnutmW1cqXzF1oVuckiEOwmG0k0vUCk7xR8LLEkNBkh3s3yF3F8LvZLlfesQIl-dNXAWWKIonfCyb_fLiciYxlROetfJXFkMr6LKvgCUHB4AAkx_X--KpHabCpBVnFrEIPv2E59D-eLo7J9Sf4Yhd6nDzdAgj5B4BgDPKSLWvAvjRe5-kD2AT2AGcG-pIBVuKZoI7avNO0GJM23k0-b6_djF13Cg8%3D"}}]},{"timeUnixNano":"1735335025819520690","name":"Retrying on 5xx error","attributes":[{"key":"status","value":{"stringValue":"500 Internal Server Error"}}]},{"timeUnixNano":"1735335025819540450","name":"backing off for 1.122949ms..."},{"timeUnixNano":"1735335025826246421","name":"submitting form","attributes":[{"key":"location","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=TUQrZ3dFSUpycGJ5TFp4c09pdGkvMDB3RWhsUkdLVEZ4Ni9OYmZOMWJuTG5xTlI3OE4vNjdQd0I4cGFSUU1QOUFTWXVqOEs4Z3lMenJhaks4ejRIdVE9PXwxNzM1MzM1MDI1fNlsbQS7GploSTVB0WfdtdH7gi0aLdGkm-Zw-kVVwqJF9rJZSPHGDEYpgyyhScfl32qa-0GvpPpCEPxXfroMBfIx0VSczgLJty1vJ_7-Wvuc_6EdlHR9x544l5p1a7zUqdftP975Xmdcj7tkJwsje3ttJ5uVESkFOy6sN0PAyQihwcYPd0aCvVnLPgVqroEbrbkPcVavCRI8sVE0Awx0X2YzkgBPWzKHmH43iAie-kM0uMZYFPFRtEFxFvD43PscwW-prc3qShMzp4C5cNJt2TRQCRJlXcUr_ccnecAjs3jdys3da_nFHMsbhpDgDfHaYsnL86zq_QVdjWJZFMpIEHOXQUVi2Q7PTm_n9iymHah7FPcWDaAo5BRsKr7fM4_JaHT_BsnrvLMRSV8c61gtzpBoDnIKrrvr7hzua4i1OFkR5XNRSOIy0gh1Re2ZpBFUo0H8miVsqD8Hqx7N1rxFopl3R-spO3a9Arin74L-5gZu1DsgkxVRa9oF5ntJ9ZigAKCnFFRgrmHynjxD4QMxGKR0tFBrlLBYiZFlWpxMYD-bmq6jccxwA8491DQAePeeyWbLIL4-9OIsBoZqIRmJac1gou6EqxJKsECNxxLCCreNKPYcNJKWdY6ZERWT2rVYlzaYowFnfV9KDvQAoaGV_D7rg3Dw7XGnQvo5NHAYGXv5sW7LB-vlPU_5CuVjV2bwmwcy6Ybez22aD1g%3D"}}]},{"timeUnixNano":"1735335025830902106","name":"Retrying on 5xx error","attributes":[{"key":"status","value":{"stringValue":"500 Internal Server Error"}}]},{"timeUnixNano":"1735335025830940047","name":"backing off for 2.414787ms..."},{"timeUnixNano":"1735335025838599218","name":"submitting form","attributes":[{"key":"location","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=djBzMm5teGFzNDQxSlorL2ZYVUN2Nk1XVHAraVFnSjhtU01IRGRlU0czSm8zRUlsM296bjlEc0o4VVhXSHFPOTd3QnlDVEhtSlp1dElXS3ExOWx5dVE9PXwxNzM1MzM1MDI1fPYv0au-VRFGwhaPnlvMz_uEJ22N0pzR8VwFreItQnBEo4C5PRNtMUFyKPuWzxHL67dfMAFNXNaYKrpHwE-RshBnJbE4oxQd6W705zHKAwm2lJsMMMHJkHZmBUk_Tghkuj7AsIlCKQHAG4Dd3NuGBjqd3Kqs9kwin5JnZat8aa2-5KX5Lk4QUFo_sRMebvIARsqK2QubYU114RUM62nzm_HmkKMcvh1QsGQ1s7pr_QepoDs7hSj_oXUs_Tc7c9n97sBGwWXfGc4yKLdodqyubc31qdSmHxcU5I8-FEFbbhMWZ3Pk51N7sMOjoYoP2oupL6d5DSf9bCermdQNAm4UElB1aVY0l_rmg282JhxP_5qBCLAml8xhGHT6JxMHAKA8H6MpNBusSbcfPmfuO_px6ipSnKE9ZsmY4y8_nbMZUaIotKED5NJn9YUlKDgrI58R050HTSbv-zy-rBVpLRwFhk2aC5dhYSyxVc6QxEYIrKvnVyouumm4bvB9xankZB_A1ESpBrAPszKOoznYh1y72LR8P18o7tOvUWxonfsKbnNeZ92RZOOIeUcaJ6-ZYUVO4Os5DrL4QSTByrikq4hdsABQHu89omt_6iS2t_salQ6eVrGQLI9E9yHzoDTVzWsYf8DZ9NbpI9OHMBFB7nJ8DDVm7XXvpgKtRezh2JVu9C3BZHLswq_0dJrE2hH30U1_pXy4mAOjddNCVt4%3D"}}]},{"timeUnixNano":"1735335025842671447","name":"Retrying on 5xx error","attributes":[{"key":"status","value":{"stringValue":"500 Internal Server Error"}}]},{"timeUnixNano":"1735335025842692477","name":"backing off for 3.385134ms..."},{"timeUnixNano":"1735335025851420141","name":"submitting form","attributes":[{"key":"location","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=K05oR213S1ZvdUlZMlh0TDFaRlU3SUlZdmc4OGIxOUFoanBDaXkvNlpZc3ZUeklnc0VQMm1CYjFGYkYrK3ZYdXpnNkNtYS9MZUtleU9DY3NMN0VNUUE9PXwxNzM1MzM1MDI1fF-scOL_CXRp0kDbE4ZjdGwJnPC2MQY02ucYYTtu54IwwWu6QgQgYZ-L1nC0_8TPDlspFKyI_65zC2CAiEPwji6gGkC65hOD0mDcjWFyaMY8NgZJYvackeVgm0tLWY0KPOyJmJEk3oGmXGYlPoPJ_sz8ywot5peE8tV3_7FUgL2k61nEBRmqffg_ICtN21fxrk63NHmQZBw5E6_fj_DSwMT0XeLiu3I2sJSMP8gAXgZ-9KNAU34h9NmBwAqkyCJ5HP7N0cYwc0wCWoZVO8kgI_MrasJvIyvf6UKnp9oBDp8Ro4pTlrukgQgcj1-Wu2x6pCl9xpBT32Ftwl4Vuvr_zI7V6MoqZMKywDBNfsApHUkZTUedX0ZLK9A29RXEVkNqr9OevFVyVsMyOvTZ_tfS7_5hfwzHjyOnHq7q2COA1ocRzWCMaXtIeSk8WwL_K6hCf9vNrpSi0_agf2JWLbefcjy8AUAWjpp6jt4My9JhdRZ-qBmnL973LPUmyiIbYkC0Km1nIe71rItMLqx2SZCwuc8-9jq7DshSiSNzNqMD35PGdiiEawIhMFJwhWRWKCDMPBCYeAkYIODnfJWGBbOrD7kFReqvfO9Jm2zbuaw9tPw6HCHWY8XwIyAougw7FP15F9Vqtzc4imuTmsvkOYJNTvI8lz8Sk2ALwlPo63QbI9DFjlE_uDuGb-HqiUqe0eoIx4EqP9Ni_Qj_AkY%3D"}}]},{"timeUnixNano":"1735335025856282990","name":"Retrying on 5xx error","attributes":[{"key":"status","value":{"stringValue":"500 Internal Server Error"}}]},{"timeUnixNano":"1735335025856302840","name":"backing off for 11.041868ms..."},{"timeUnixNano":"1735335025873572446","name":"submitting form","attributes":[{"key":"location","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=NlpEbWhsdDRlbm1nNEVtSWR0U1BsY3RWU3kyRTNvR09DK3o1NnRLUEtGNCtCNUk5NmE0dUE2N01KM0xkdnk2WGgwTjN1eGQ2cG1rLzdweE4wc1JCbFE9PXwxNzM1MzM1MDI1fNPhIc3ToLHpvS_Gvc8aQRaU88cZYSsnQK8fP27HBXGJKzuRaASTXkCUMwIy2bfmT5GrsBxx8-MmupZH8PE_9xtv96G9GEbRgrrFdukuhrQei_80gU6OgUoTwTtCC9jC8T37myP2Oy72fpFGDoXxzEzRGhSyMAHQJy_HBv3PLX-UxHuResc5NH_Yj7aDx16xDkDAWmc-S1VeeUW0zqT8HgnzfXFYdZqf2_tnV12NUp-CLGXLRsU-3XJIsSwy5CRy1mfIGspxD-SxBHxPaRjKMY9FMxc4CsDmR4eiF28glstJrheqExh9wDhG0TFmgwLtEJ1Vq_hLTmV3RZO8jx8ZtdV1id1slzj8fwzRXH-IVKgTJ1uLcA28LLI1ZzJzD2PrTkfISjg6-5nUJv-OYQcQlQI21pxL1s4ICAa0l3KzE5i5_yeymQaGeAfSLIJ3-5RTTwBMrx5zgMlco96WRrJZ604XqQSOgiGoxSLA9aHu__QOK9OU7GFdga92Y67T3vkKwvWXeX3w2EAF__tzDF5dy--oGborqKGcv0fkKeMgRyfxkxJ-E1w0Z7ppcwL6LL1q78zKI8JcLzgV8MyVDcVnHRrkIToVZ5YYavQnbXK_7lfo_XbKyifs4ZqPmwNY0WyGZ0j_ZZTOLRo8lQR2y5NOlTZRWCz3sqkmOJ3SeAoGrHiQ0tZDfFgvO99ucbtQj1_PCyO6u2-JaIpH6fg%3D"}}]},{"timeUnixNano":"1735335025877583754","name":"Retrying on 5xx error","attributes":[{"key":"status","value":{"stringValue":"500 Internal Server Error"}}]},{"timeUnixNano":"1735335025877603214","name":"backing off for 8.858289ms..."},{"timeUnixNano":"1735335025892996459","name":"submitting form","attributes":[{"key":"location","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=eE9BRU5YQUZpTzFaV0pPK28rTmF0NU1xempEV1NzU3hhM1lDL1I2Zm5KNFRkM0NPd3RQY2wxZDAvVVFJaVB1MTN6enlwa1h1NDFaZmRHZGFIdFQxVlE9PXwxNzM1MzM1MDI1fEw2htjJsKXWR0ubmlecrlxxyWaM5lyrfQKVov8UW14H8_5YI7twjd0qGPWd8uLPpHaUpE5zR5PVk37RnPpkNC2J5NteJv5IN51QbBerrsuGuDZ650Sbqr9-p9FXOKmBBcoKRPW_0N-B7PWMWkQIsY5rVbjYeXgfOjuadl0PNkCA0awKZQhOwfJwr8waX88ypxsRszpU26pXB75zl5tO-AOtrROKkjsv_Y0i-Vvvhj7VGlRPsJ2xvNqK9DuJLbdjmyKrSutz1IsGwhs6NWyd82yaid7x7qCob3VkStVsNeLhjcniP9xjD22Hbi50itMD1HyXCTu_hAVyEYw2SHC842nad-4tP1EEvQkB05PBwTT5G324_lbicp6lFCSzGT9rS2MPEuz12m8Uq9hdJEj18am1pZM7vBXgevH6K0mwlpb4eIgqceOwxbgSHeGw07op-0jdeGnbz1REdKGGui4pWW5PmxcCjDiAG6g6kP6DrXUhPvAVXPK3GhpK4lxuLfU84zuQr81Xo0EmYOxQGjvdcYepBDn9anS1tEghjCk7iptxM6zO11Q7vMFI9wcVHrNl3sSCAM2k21BFmPEyWcnWvPFsQMsegbSznjD2X52WjB7k19H406-bHN-d5Py0dcMo185Y1aAIZ3m2CyXNMgPlrBPT-5XDSvf8113VVpTk8Yazr1rx0DzKlmLhWaUo0Nxm9mPwxK17XRvpdSg%3D"}}]},{"timeUnixNano":"1735335025896949436","name":"Retrying on 5xx error","attributes":[{"key":"status","value":{"stringValue":"500 Internal Server Error"}}]},{"timeUnixNano":"1735335025896969616","name":"backing off for 35.491179ms..."},{"timeUnixNano":"1735335025939610306","name":"submitting form","attributes":[{"key":"location","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=Vld4RWw1Q2pKNkZ2NXNST3pzVTdFMi9qSTNMY0E3bm91bDNYUDNyNWVJbUMrekFzSW5WejIySEtxclJscnBvUkkvVWY1RStubmcrT1g3S1llcklSUWc9PXwxNzM1MzM1MDI1fCZPSA2y7mrm6xOOpuWELdtqzK8AmnUFMhpMijy4u4IE5SwEboZafPLVOuG6XLO-hnn5Xf2lTXraFGd6UpqWGelZT7VfHlpWPhDdf2uCYIAj1u6TYRgPS3_hMilmsO6AvSWmX37-XsPL4I78VhCqbL2FoEKz_CgyCAVhowdooxX6INcvZ7-G8BCJX16jnZKu7o3gX8xFl0mrHYKV1EAPY1V2YsxRTGRS2ptkcqEuxXw9YVOQIIsC6OsphlbN91V6qHGhzBRcLX8CZ-8E_2D9Osgiy4-C-koOCtzRLLFJy9S6oq-8Z1c3PK_wGhTu-syftdKFKjKjdRo97DWgjVz_lnKWeYC2usX32gaExGKkVkMERCaQ6JKoi-TXo4kY1g-AywEfn_gehBqAqzlXQ5VHEDdCth5C2BHjOhgUKcDv7W4Oqmp_93_U28elEz4Q72JUpSaDDe1fKA_eArNUrQBJTJ6N6bABEwL52xkh0ifRwWcERMCwnc1bOlAIDs2T1yWW27hdYrpSOZbAwdKuSji2x1ut0-DqgnGoDMMqzgS98MUPsC9I3PNltfu_DgC959VjH7H6_HmE2q1nPELeZUXRCGzAZTOFE23bD0ks6uyipS3jaVb1qxVClG3VMuSLf9SdV2ESH7MWJaXGyvhN8c1KBsiZp5L-7gqCnsZOMYQrJNQIsbdic-DkeRlmQYbs8iZykp-cADl2uTKek-0%3D"}}]},{"timeUnixNano":"1735335025945633478","name":"Retrying on 5xx error","attributes":[{"key":"status","value":{"stringValue":"500 Internal Server Error"}}]},{"timeUnixNano":"1735335025945650428","name":"backing off for 33.510651ms..."},{"timeUnixNano":"1735335025986804741","name":"submitting form","attributes":[{"key":"location","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/oidc/auth?client_id=CLIENT_ID\u0026redirect_uri=https%3A%2F%2Fauthenticate.localhost.pomerium.io%3A36063%2Foauth2%2Fcallback\u0026response_type=code\u0026scope=openid+email+profile\u0026state=TlVucmJGSVM1TTkxQjdVeFpFTndpblBPMUpqdjBoOTBCTkN1NVBPa1RWWGkzcC9YNE1Td3RYc3IyOHZQS05HSVA5am9EbngyT0pNdzBzdEQ4Kzhrbmc9PXwxNzM1MzM1MDI1fGFr6SjNR6oY3LbAckTb1zOQwi4DF7qp73tBPnxNCM9uAN-E7benf9dM4rTw1MXWLEUpA1XTw-NdsI5DXa5mG-guOz-4MjnubljhndMmKy6H-mdPjrJwqpirLHSjGNIOMh6prOpDlt1i-RnnOB_wtIY1wxUCPntd-aCiBGfViIgB0aCMQq6YcqcJYohYyPYYILz6IczkvpPa5cevuvnCJ20J7X69_6AJ0tYzwo9PVmYrGFyTJULaNfw-2G-nUAFuP4-FHv4PexN-NB_Qn0kM4KsX6482RPpNetBM8hHfHbdlr6J_018NOc7Jpv092cxDmMpwzeTQ7mKtEgntBj8WrClM6VTrkecRpdJ8yiZhLvgVhjK76JUFXlXNzYItTt-NvBKuXZ-869TmVrWADw6579M1IP2uCvQlE88v4e__LelTsipYfTzM_kN_Zl7hoDur57rTfzsbkipm5-szZ1W4wEksgtPybfnauIOJY3AtTQ1u6HSciCbGULA6N024tcnpDAJ2mOwYuYbMHlvVYpTv9AJ7P6WOuxaD9GQP5jDJb_AcmhzylwX6r7TQrHnPtbRfratJUOP85A9-5vgaKdLJ4QIPgVqPh4eyTR6xRWWkFnkJdUnJ8m8gshI0ZZI1KzM-ORzBPl8TrEFfqrHHVq1ujN7JQknxUgbrWQOU-jzncZnCYk6TBMbyFX7SdKMG8rgbLaDdnANGnoVCiuA%3D"}}]}],"status":{"code":"STATUS_CODE_OK"}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, - {"timestamp":"2024-12-27T21:30:29.754253667Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/server.go:85"}},{"key":"service.name","value":{"stringValue":"Control Plane"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"v60d668rsFI=","parentSpanId":"sQT1yEWt9Cs=","flags":256,"name":"envoyconfig.Builder.BuildClusters","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024752937074","endTimeUnixNano":"1735335024753567272","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"1yZo4AZ6R6s=","parentSpanId":"sQT1yEWt9Cs=","flags":256,"name":"envoyconfig.Builder.BuildRouteConfigurations","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024752902344","endTimeUnixNano":"1735335024753657353","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"hwlquCtBU6I=","parentSpanId":"sQT1yEWt9Cs=","flags":256,"name":"envoyconfig.Builder.BuildListeners","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024752986625","endTimeUnixNano":"1735335024756961172","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"sQT1yEWt9Cs=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"controlplane.Server.buildDiscoveryResources","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024752828213","endTimeUnixNano":"1735335024757486459","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/xds.go:23"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"uUk5i2AWBI8=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"controlplane.Server.OnConfigChange","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024757502610","endTimeUnixNano":"1735335024757514710","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/server.go:255"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"m0X4qa9wV2c=","parentSpanId":"ULuo/pZ48II=","flags":256,"name":"envoyconfig.Builder.BuildClusters","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024981645690","endTimeUnixNano":"1735335024981731451","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"G1RZ+p9n3wg=","parentSpanId":"ULuo/pZ48II=","flags":256,"name":"envoyconfig.Builder.BuildRouteConfigurations","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024981640880","endTimeUnixNano":"1735335024981808162","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"lEwSFwmbsuk=","parentSpanId":"ULuo/pZ48II=","flags":256,"name":"envoyconfig.Builder.BuildListeners","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024981666531","endTimeUnixNano":"1735335024982687002","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"ULuo/pZ48II=","parentSpanId":"6TmKHMpFyR8=","flags":256,"name":"controlplane.Server.buildDiscoveryResources","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024981577879","endTimeUnixNano":"1735335024982789743","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/xds.go:23"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"6TmKHMpFyR8=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"controlplane.Server.update","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024979194300","endTimeUnixNano":"1735335024982795253","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/server.go:279"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Sq0l7QhDyGw=","parentSpanId":"Wh6E3dnkilQ=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025810188179","endTimeUnixNano":"1735335025812103302","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"IFI+Jjnxj/k=","parentSpanId":"6gWh1J+1CTs=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025816575605","endTimeUnixNano":"1735335025818844242","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"374SofEc5m0=","parentSpanId":"KCsnd9DyaCk=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025824689532","endTimeUnixNano":"1735335025825267179","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"nMxMJeHK43k=","parentSpanId":"n9+R8Z02ndA=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025827930971","endTimeUnixNano":"1735335025830137867","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"MZXyiOTh9Eg=","parentSpanId":"QAxhOEu2VQU=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025837118560","endTimeUnixNano":"1735335025837672777","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"HAIPnqctQV0=","parentSpanId":"LUc3vGeZwMU=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025840145587","endTimeUnixNano":"1735335025842017740","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"niDmR26iJ04=","parentSpanId":"kFGSZ3gb9bk=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025849973685","endTimeUnixNano":"1735335025850509481","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"SKktgOzuqY8=","parentSpanId":"yp5RxlvZiO8=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025852883819","endTimeUnixNano":"1735335025855193017","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"NrLjbmN+dFg=","parentSpanId":"skvhJjM1/7g=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025872067819","endTimeUnixNano":"1735335025872665856","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"+1faJLPAIcA=","parentSpanId":"KS2iwg6Ulrk=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025875076544","endTimeUnixNano":"1735335025877003927","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"1wVqzZAVYVo=","parentSpanId":"UvKTIVQP9uU=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025891138737","endTimeUnixNano":"1735335025892143109","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"74NUTYEb5Bg=","parentSpanId":"tqKzCJwTic4=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025894506047","endTimeUnixNano":"1735335025896375430","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Lg6KRq4pqLI=","parentSpanId":"KmOo7wve0b8=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025936456229","endTimeUnixNano":"1735335025936996765","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"6ssvTNGwuSI=","parentSpanId":"V/8X7Bvg6MQ=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025943015547","endTimeUnixNano":"1735335025945046211","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"81+d+tfVjk8=","parentSpanId":"b8TIlWwstzc=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025985231592","endTimeUnixNano":"1735335025985897359","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"gxVvt4Qw3j4=","parentSpanId":"NPwFGMaPhF0=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025988379350","endTimeUnixNano":"1735335025992807653","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Sakh5mjAWmc=","parentSpanId":"4uLVZ3lujSQ=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025994883677","endTimeUnixNano":"1735335025996036772","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"c61OVvg7XN8=","parentSpanId":"/Z4/vZfyMIg=","flags":768,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025791087760","endTimeUnixNano":"1735335025791265402","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52360"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"qxEQ86FHF6s=","parentSpanId":"P2zcRg+jsFU=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025801616696","endTimeUnixNano":"1735335025804016115","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52362"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"82uwbqPPsyw=","parentSpanId":"RaoMfsRKjao=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025808896723","endTimeUnixNano":"1735335025809411459","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"+Esu9QHJMcINMk5i34yYkQ==","spanId":"agu8Rnp2lTY=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025812994992","endTimeUnixNano":"1735335025813461367","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"heQjAAl6dfU=","parentSpanId":"tjexoDV7/Hw=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025815769595","endTimeUnixNano":"1735335025816184490","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"7w6Ze9HZfEVHHUuDVdMu6A==","spanId":"YScrBlT6wM8=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025819523040","endTimeUnixNano":"1735335025819673762","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"3Y8oPmEgblM=","parentSpanId":"moqlVoGI3u8=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025821290081","endTimeUnixNano":"1735335025822550546","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52362"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"ahfKuiWUwf4UL01+76Wbzw==","spanId":"kF6eC5N4U6Q=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025823305055","endTimeUnixNano":"1735335025823437187","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52388"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"PiagOqYTpmM=","parentSpanId":"d+7biQunPZg=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025823865962","endTimeUnixNano":"1735335025824300178","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"PI8a76D3GRaQcVMVvKuhxA==","spanId":"aLFJpoBTHWU=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025825930037","endTimeUnixNano":"1735335025826102579","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"KKk10SNGrmc=","parentSpanId":"12NNbneNbUM=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025827104331","endTimeUnixNano":"1735335025827553596","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"hvwtVhD28xiIg6jGUsGaUA==","spanId":"zmCkOSt/uoU=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025830854846","endTimeUnixNano":"1735335025831018477","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"9g8u2zSWvxM=","parentSpanId":"Cw7qX+Mq85E=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025834037494","endTimeUnixNano":"1735335025835353929","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52362"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"+MYYvMbuVt4g7WuhUNPW/g==","spanId":"dz5uWpV6YNA=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025835827455","endTimeUnixNano":"1735335025836006738","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52388"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"86mKxrPrdQQ=","parentSpanId":"aQWRcZQDEL0=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025836369052","endTimeUnixNano":"1735335025836705056","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"TUv6UTv0VXh0kfLsYrzVKQ==","spanId":"4o3f+Kc15eg=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025838285165","endTimeUnixNano":"1735335025838413676","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"VchSVDx2hio=","parentSpanId":"zZCpocublXw=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025839411308","endTimeUnixNano":"1735335025839735711","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mpWzym14/n5SACh89xs85A==","spanId":"nzL1FznBWHc=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025842625817","endTimeUnixNano":"1735335025842773378","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"04nIoKZ7Rn8=","parentSpanId":"pMCRYOIpDEU=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025846889047","endTimeUnixNano":"1735335025848342995","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52362"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ZaPF80+dmj0=","parentSpanId":"ix5QFxi6p84=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025849258506","endTimeUnixNano":"1735335025849554229","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"BmZa5YHnHlYQiR+IrkT47Q==","spanId":"A/QKUn1/j7Y=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025851141728","endTimeUnixNano":"1735335025851285560","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"4k00P8wSf5A=","parentSpanId":"RjOOCHrl7Pg=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025852180531","endTimeUnixNano":"1735335025852530975","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"r24gxKcPFjoUmOrv5DxJhw==","spanId":"Fx9e2eE/2tk=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025856341321","endTimeUnixNano":"1735335025856488572","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"AWRkklChgtM=","parentSpanId":"AxcQF3bUAPA=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025868616247","endTimeUnixNano":"1735335025870334218","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52362"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"FV+HlSQF4YQXvthFn+vPRA==","spanId":"zqQnLUpZJCY=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025870825104","endTimeUnixNano":"1735335025871009936","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52388"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"egVsvBSrdgA=","parentSpanId":"zprk0rw2m8Q=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025871292749","endTimeUnixNano":"1735335025871653633","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"Sxyz3wluzqbxA7WEDH0jmQ==","spanId":"JmqOMx1Ebj0=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025873224052","endTimeUnixNano":"1735335025873363724","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"QdUTNSa6qh0=","parentSpanId":"zsr1/Xcbq2A=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025874360096","endTimeUnixNano":"1735335025874704660","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"vWoVZuYHE7KlLY8HQtSswg==","spanId":"QDkd/nq3ZF0=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025877585554","endTimeUnixNano":"1735335025877748386","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"gaEA+BN+3kI=","parentSpanId":"VlLCvKW20c8=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025887963709","endTimeUnixNano":"1735335025889467206","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52362"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"C+l5gQuBOqvHCrVNu+ScZQ==","spanId":"GqwwAwMakis=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025889952343","endTimeUnixNano":"1735335025890103245","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52388"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"tUL29k86aA0=","parentSpanId":"TnTIhQSeD8s=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025890415708","endTimeUnixNano":"1735335025890740692","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"1ZAKVivbLPa8QTyRp/cohA==","spanId":"GjA6x3sP32c=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025892627675","endTimeUnixNano":"1735335025892774436","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ojzIzoRpdkE=","parentSpanId":"fpe5wrNxals=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025893747717","endTimeUnixNano":"1735335025894140383","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"5qSLUDyjtQlRUAUnOy376Q==","spanId":"MVnTad8KraA=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025896950346","endTimeUnixNano":"1735335025897145708","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"iI6JC+zkuuI=","parentSpanId":"HG6gDV9Mh1I=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025933475133","endTimeUnixNano":"1735335025934864520","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52362"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"jozir3kDgsM=","parentSpanId":"x4/fCcrN2l0=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025935692199","endTimeUnixNano":"1735335025936067784","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"u6fr0Pm2yZsJdFtLxvYY6A==","spanId":"I/1Y/o3LJKE=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025937522891","endTimeUnixNano":"1735335025937665513","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"HSPi+PbWXh0=","parentSpanId":"Ktn4PSE/Eio=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025942261708","endTimeUnixNano":"1735335025942572452","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"qcVMFqgVWF0zPlPHBnnkTw==","spanId":"u4M218qJeLE=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025945601248","endTimeUnixNano":"1735335025945778270","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"vZpvKbrCQqQ=","parentSpanId":"xC/VB6ObsY8=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025980994651","endTimeUnixNano":"1735335025981239534","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52390"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"2yTflfwGhOI=","parentSpanId":"zQwYJxI53tM=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025981996074","endTimeUnixNano":"1735335025982197246","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52390"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"FtTGqe5YvtE=","parentSpanId":"SRBjmOgMIao=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025979676595","endTimeUnixNano":"1735335025983226488","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52362"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"e3JgaULd+gGFz/8A4w5e8g==","spanId":"+HmVp1qzIts=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025983732184","endTimeUnixNano":"1735335025983996588","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52388"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"owAwqwUWHX4=","parentSpanId":"U+zO7kgs2kw=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025984297291","endTimeUnixNano":"1735335025984776537","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"fjpGU6n1puaDxNOl3amsAw==","spanId":"LEmR4prBl5E=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025986530558","endTimeUnixNano":"1735335025986712400","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ayYTEGRLA/4=","parentSpanId":"pqDfjSA5zY8=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025987598210","endTimeUnixNano":"1735335025988006776","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"cifxyFmv4dY=","parentSpanId":"Hh70Fz863sw=","flags":768,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025990548786","endTimeUnixNano":"1735335025990652057","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52402"}},{"key":"rpc.grpc.status_code","value":{"intValue":"5"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"vP37Rm86XmY=","parentSpanId":"JywV84LM8IA=","flags":768,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025991322805","endTimeUnixNano":"1735335025991453716","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52402"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"3ijdqATnWA0=","parentSpanId":"8GtaEDC3sm8=","flags":768,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025992226656","endTimeUnixNano":"1735335025992349167","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52402"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"ubxh4ifDIe2okDP6OtRYFA==","spanId":"ESGoMZpHB88=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025993673843","endTimeUnixNano":"1735335025993818894","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"2cTcRlCzz7A=","parentSpanId":"IwEGClsNoWA=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025994191250","endTimeUnixNano":"1735335025994511013","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52368"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"WgzfdQgrifQ=","parentSpanId":"TjF0fvq4Ahs=","flags":768,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025995387233","endTimeUnixNano":"1735335025995508385","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52402"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"BPwmhsyPx+Hs4Aq/WNwlfw==","spanId":"QPU+lU7U14g=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025996637579","endTimeUnixNano":"1735335025996759610","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52382"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"jn2jF/UGEwI=","parentSpanId":"g8gyWKtu1KQ=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025997215675","endTimeUnixNano":"1735335025997552799","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52362"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"wBTyiupIuPiO4BLtsY0qUA==","spanId":"D+teV707Wvg=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025998741624","endTimeUnixNano":"1735335025998851195","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52388"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"AWyn5ABOBS0=","parentSpanId":"qo+2ljnLo3s=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025999800756","endTimeUnixNano":"1735335025999986049","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52390"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"QBjGZUW54F0=","parentSpanId":"niGr/qWcz4g=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335026000647497","endTimeUnixNano":"1735335026000798758","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52390"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"yHJAjGq7Cgg=","parentSpanId":"yo8tJLvB7F8=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335026001480356","endTimeUnixNano":"1735335026001645488","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52390"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"DC1wSzzAIIQ=","parentSpanId":"GZGIXC4RfsY=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335026002212135","endTimeUnixNano":"1735335026002391097","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52390"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"kKNQsnG+JKs=","parentSpanId":"Vh7x9OYP6sg=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025999199329","endTimeUnixNano":"1735335026003181657","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52362"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"3JQRBybzzOfTYzQQxN38pg==","spanId":"U1wYLMi29Kw=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335026004202599","endTimeUnixNano":"1735335026004326451","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52388"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"CGVc3FYZu4Q=","parentSpanId":"TGyFsg9jcEc=","flags":768,"name":"registry.Registry/Report","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335026622400462","endTimeUnixNano":"1735335026622594285","attributes":[{"key":"rpc.service","value":{"stringValue":"registry.Registry"}},{"key":"rpc.method","value":{"stringValue":"Report"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52414"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp","version":"0.57.0"},"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Wh6E3dnkilQ=","parentSpanId":"dq09RB/npOA=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025810161779","endTimeUnixNano":"1735335025812111242","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1109"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"6gWh1J+1CTs=","parentSpanId":"dMDfBRWWNfg=","flags":768,"name":"Server: GET /oauth2/callback","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025816557095","endTimeUnixNano":"1735335025818859832","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/oauth2/callback"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1286"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"KCsnd9DyaCk=","parentSpanId":"K5Za89EMcFE=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025824673852","endTimeUnixNano":"1735335025825279409","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1109"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"n9+R8Z02ndA=","parentSpanId":"ETof3b34RNg=","flags":768,"name":"Server: GET /oauth2/callback","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025827892501","endTimeUnixNano":"1735335025830155978","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/oauth2/callback"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1286"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"QAxhOEu2VQU=","parentSpanId":"JMTzABiGm6s=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025837101150","endTimeUnixNano":"1735335025837680307","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1109"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"LUc3vGeZwMU=","parentSpanId":"rLKtQLC36X0=","flags":768,"name":"Server: GET /oauth2/callback","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025840115597","endTimeUnixNano":"1735335025842033750","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/oauth2/callback"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1286"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"kFGSZ3gb9bk=","parentSpanId":"UBrFUNOcDBc=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025849955795","endTimeUnixNano":"1735335025850520191","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1109"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"yp5RxlvZiO8=","parentSpanId":"B7oHC/7Ojus=","flags":768,"name":"Server: GET /oauth2/callback","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025852864029","endTimeUnixNano":"1735335025855209937","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/oauth2/callback"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1286"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"skvhJjM1/7g=","parentSpanId":"fOxEGb9Booo=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025872050999","endTimeUnixNano":"1735335025872673316","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1109"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"KS2iwg6Ulrk=","parentSpanId":"rid7A7oKsBM=","flags":768,"name":"Server: GET /oauth2/callback","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025875060084","endTimeUnixNano":"1735335025877018588","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/oauth2/callback"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1286"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"UvKTIVQP9uU=","parentSpanId":"jh13uEWyq54=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025891112586","endTimeUnixNano":"1735335025892150419","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1109"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"tqKzCJwTic4=","parentSpanId":"zaz8bXwr3x0=","flags":768,"name":"Server: GET /oauth2/callback","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025894483737","endTimeUnixNano":"1735335025896388580","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/oauth2/callback"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1286"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"KmOo7wve0b8=","parentSpanId":"8j7gcdiyUN4=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025936439719","endTimeUnixNano":"1735335025937002515","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1109"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"V/8X7Bvg6MQ=","parentSpanId":"PbLOU7xbMgE=","flags":768,"name":"Server: GET /oauth2/callback","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025942995277","endTimeUnixNano":"1735335025945061772","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/oauth2/callback"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1286"}},{"key":"http.status_code","value":{"intValue":"500"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"b8TIlWwstzc=","parentSpanId":"C27ROXjc3Mg=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025985214352","endTimeUnixNano":"1735335025985923170","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1109"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"NPwFGMaPhF0=","parentSpanId":"rn0P4iWyrGg=","flags":768,"name":"Server: GET /oauth2/callback","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025988363160","endTimeUnixNano":"1735335025992825383","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/oauth2/callback"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"545"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"4uLVZ3lujSQ=","parentSpanId":"HLHEc+bHits=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025994865237","endTimeUnixNano":"1735335025996042312","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34922"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1334"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"rGSqePbCvq8=","parentSpanId":"c4K33/UOGkY=","flags":768,"name":"Server: GET /","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025998120747","endTimeUnixNano":"1735335025998238578","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"36063"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"34938"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/callback/"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"66"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, - {"timestamp":"2024-12-27T21:30:29.961108701Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/authenticate.go:54"}},{"key":"service.name","value":{"stringValue":"Authenticate"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp","version":"0.57.0"},"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"68TkOnpSKwY=","parentSpanId":"Sq0l7QhDyGw=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025810351341","endTimeUnixNano":"1735335025811858969","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"7uQdjOjS7cI=","parentSpanId":"2OM054Y6ekI=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025816661726","endTimeUnixNano":"1735335025816945089","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"PGcngHRgqCX4iaoO2fEC5g==","spanId":"ZsmRwKi3Ki4=","flags":256,"name":"OAuth2 Client: GET /.well-known/jwks.json","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025817551966","endTimeUnixNano":"1735335025817837991","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/jwks.json"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"237"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"uMSJ2tNZc4o=","parentSpanId":"374SofEc5m0=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025824809913","endTimeUnixNano":"1735335025825054246","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"d/S+GCMxW/g=","parentSpanId":"keERcjTx5f8=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025828015602","endTimeUnixNano":"1735335025828350076","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"Cm2eM1rGKwOPUvNrZ3JnIQ==","spanId":"2vnGWHtaRpY=","flags":256,"name":"OAuth2 Client: GET /.well-known/jwks.json","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025828864322","endTimeUnixNano":"1735335025829104825","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/jwks.json"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"237"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"STVwwUHTbwU=","parentSpanId":"MZXyiOTh9Eg=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025837271002","endTimeUnixNano":"1735335025837536305","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"rhwZXUEWDec=","parentSpanId":"+q/dL49IE64=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025840238718","endTimeUnixNano":"1735335025840538652","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"Ng1V9c7eduVtG8yBTNSUdw==","spanId":"uAH2dcySq2g=","flags":256,"name":"OAuth2 Client: GET /.well-known/jwks.json","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025841008297","endTimeUnixNano":"1735335025841249770","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/jwks.json"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"237"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"KVfrvuRAMxg=","parentSpanId":"niDmR26iJ04=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025850109416","endTimeUnixNano":"1735335025850425100","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Ua0wEfIKbXg=","parentSpanId":"Gt2TeC1aqPg=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025853004610","endTimeUnixNano":"1735335025853297644","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"1F2INkD1DM1Gd2zZRmjgeQ==","spanId":"TOYv2v4FgWY=","flags":256,"name":"OAuth2 Client: GET /.well-known/jwks.json","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025853745289","endTimeUnixNano":"1735335025853992803","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/jwks.json"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"237"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"FBXEc0lb4OY=","parentSpanId":"NrLjbmN+dFg=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025872225141","endTimeUnixNano":"1735335025872507544","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"u5Nt8T6Ejo0=","parentSpanId":"DIniBfO6UmU=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025875180866","endTimeUnixNano":"1735335025875464629","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"3QpZE6RgWdOi5jhjakP3iw==","spanId":"ieD2I+EcHco=","flags":256,"name":"OAuth2 Client: GET /.well-known/jwks.json","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025875966256","endTimeUnixNano":"1735335025876204678","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/jwks.json"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"237"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Vb1M0/gTV54=","parentSpanId":"1wVqzZAVYVo=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025891274498","endTimeUnixNano":"1735335025891560921","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"E3uSnGr1nuo=","parentSpanId":"OrWTQ/P6cUg=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025894580388","endTimeUnixNano":"1735335025894879101","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"PgqwQu60TK/M1HnlytJ0OA==","spanId":"bO5cO4l7/ZY=","flags":256,"name":"OAuth2 Client: GET /.well-known/jwks.json","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025895363437","endTimeUnixNano":"1735335025895629050","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/jwks.json"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"237"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"s5LlPWRVkt0=","parentSpanId":"Lg6KRq4pqLI=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025936568330","endTimeUnixNano":"1735335025936827293","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"MN31qbGJoAI=","parentSpanId":"Dtl5Ryn+L04=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025943121088","endTimeUnixNano":"1735335025943398532","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"Taxuplp6DB+VUFbngmAAPw==","spanId":"75tyAErilUw=","flags":256,"name":"OAuth2 Client: GET /.well-known/jwks.json","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025943941569","endTimeUnixNano":"1735335025944189992","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/jwks.json"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"237"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"11tLsq+ze/k=","parentSpanId":"81+d+tfVjk8=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025985356943","endTimeUnixNano":"1735335025985697997","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"EqhoNIgJKpM=","parentSpanId":"JmRWK9vBQms=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025988459931","endTimeUnixNano":"1735335025988721994","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"u+VskjE7NJ5A/WdVWZgX0g==","spanId":"eNxd/O1kD1M=","flags":256,"name":"OAuth2 Client: GET /.well-known/jwks.json","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025989222180","endTimeUnixNano":"1735335025989419262","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:33771/.well-known/jwks.json"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"33771"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"237"}}],"status":{}}]},{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"fwpT5InD5fo=","parentSpanId":"Sq0l7QhDyGw=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025810285030","endTimeUnixNano":"1735335025812091962","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"2OM054Y6ekI=","parentSpanId":"IFI+Jjnxj/k=","flags":256,"name":"authenticate.getOAuthCallback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025816621876","endTimeUnixNano":"1735335025818676870","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:327"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"rfYpAPj+Z5Y=","parentSpanId":"374SofEc5m0=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025824756963","endTimeUnixNano":"1735335025825255419","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"keERcjTx5f8=","parentSpanId":"nMxMJeHK43k=","flags":256,"name":"authenticate.getOAuthCallback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025827976542","endTimeUnixNano":"1735335025829830903","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:327"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"mLi3S4al7/Q=","parentSpanId":"MZXyiOTh9Eg=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025837219552","endTimeUnixNano":"1735335025837669477","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"+q/dL49IE64=","parentSpanId":"HAIPnqctQV0=","flags":256,"name":"authenticate.getOAuthCallback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025840195818","endTimeUnixNano":"1735335025841862858","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:327"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"TXUsos5kBAU=","parentSpanId":"niDmR26iJ04=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025850054386","endTimeUnixNano":"1735335025850506761","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Gt2TeC1aqPg=","parentSpanId":"SKktgOzuqY8=","flags":256,"name":"authenticate.getOAuthCallback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025852956420","endTimeUnixNano":"1735335025854634200","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:327"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"B/8GRqmLiD0=","parentSpanId":"NrLjbmN+dFg=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025872170720","endTimeUnixNano":"1735335025872662356","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"DIniBfO6UmU=","parentSpanId":"+1faJLPAIcA=","flags":256,"name":"authenticate.getOAuthCallback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025875137845","endTimeUnixNano":"1735335025876866526","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:327"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"qM9apFAszkU=","parentSpanId":"1wVqzZAVYVo=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025891216407","endTimeUnixNano":"1735335025892125099","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"OrWTQ/P6cUg=","parentSpanId":"74NUTYEb5Bg=","flags":256,"name":"authenticate.getOAuthCallback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025894542748","endTimeUnixNano":"1735335025896299149","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:327"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"v29TkK6tEd8=","parentSpanId":"Lg6KRq4pqLI=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025936521940","endTimeUnixNano":"1735335025936988485","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Dtl5Ryn+L04=","parentSpanId":"6ssvTNGwuSI=","flags":256,"name":"authenticate.getOAuthCallback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025943072718","endTimeUnixNano":"1735335025944817659","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:327"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"2ZQdrlp4PQA=","parentSpanId":"81+d+tfVjk8=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025985306413","endTimeUnixNano":"1735335025985888449","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"JmRWK9vBQms=","parentSpanId":"gxVvt4Qw3j4=","flags":256,"name":"authenticate.getOAuthCallback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025988424971","endTimeUnixNano":"1735335025992705111","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:327"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"/vivTqGooJY=","parentSpanId":"P8FQuKylCa4=","flags":256,"name":"authenticate.SignIn","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025995755978","endTimeUnixNano":"1735335025996025632","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:162"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"P8FQuKylCa4=","parentSpanId":"Sakh5mjAWmc=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025995020979","endTimeUnixNano":"1735335025996034412","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"TBBw9HlazRs=","parentSpanId":"2OM054Y6ekI=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025818473748","endTimeUnixNano":"1735335025818497298","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"S28y0oRNdMQ=","parentSpanId":"2OM054Y6ekI=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025818637590","endTimeUnixNano":"1735335025818656600","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"RAQ9e4c2I1o=","parentSpanId":"keERcjTx5f8=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025829704651","endTimeUnixNano":"1735335025829733222","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"YoI+tVmeBU8=","parentSpanId":"keERcjTx5f8=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025829802983","endTimeUnixNano":"1735335025829819013","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"qpHhwhJ2jAg=","parentSpanId":"+q/dL49IE64=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025841747185","endTimeUnixNano":"1735335025841773156","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"TUjoaJC9HOc=","parentSpanId":"+q/dL49IE64=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025841839637","endTimeUnixNano":"1735335025841853178","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"hyl5oOC3YzM=","parentSpanId":"Gt2TeC1aqPg=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025854524469","endTimeUnixNano":"1735335025854547439","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"TMGWDzqOHlQ=","parentSpanId":"Gt2TeC1aqPg=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025854606140","endTimeUnixNano":"1735335025854625060","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ZU5kNBztLjY=","parentSpanId":"DIniBfO6UmU=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025876733454","endTimeUnixNano":"1735335025876758805","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"k7H+ovBso5U=","parentSpanId":"DIniBfO6UmU=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025876834766","endTimeUnixNano":"1735335025876854736","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"PSwEM35MNTk=","parentSpanId":"OrWTQ/P6cUg=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025896197728","endTimeUnixNano":"1735335025896218918","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"oOGdHZ1iuAA=","parentSpanId":"OrWTQ/P6cUg=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025896277748","endTimeUnixNano":"1735335025896290689","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"oXhIZii2q6U=","parentSpanId":"Dtl5Ryn+L04=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025944684407","endTimeUnixNano":"1735335025944716418","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"6qc4irrqh+0=","parentSpanId":"Dtl5Ryn+L04=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025944789678","endTimeUnixNano":"1735335025944804859","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Hh70Fz863sw=","parentSpanId":"JmRWK9vBQms=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025989899467","endTimeUnixNano":"1735335025990944830","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"5"}}],"status":{"message":"record not found","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"JywV84LM8IA=","parentSpanId":"JmRWK9vBQms=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025991011171","endTimeUnixNano":"1735335025991662729","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"8GtaEDC3sm8=","parentSpanId":"JmRWK9vBQms=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025991930462","endTimeUnixNano":"1735335025992593130","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"TjF0fvq4Ahs=","parentSpanId":"P8FQuKylCa4=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025995100900","endTimeUnixNano":"1735335025995725367","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, - {"timestamp":"2024-12-27T21:30:29.966934441Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/authorize/authorize.go:48"}},{"key":"service.name","value":{"stringValue":"Authorize"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"+QZw+I9tpmY=","parentSpanId":"9zbnaSilf64=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025802086372","endTimeUnixNano":"1735335025802275004","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"K5lGkBkLfGA=","parentSpanId":"5PoR/9GtyZ8=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025802546737","endTimeUnixNano":"1735335025802838961","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"WVzMc/GgbKk=","parentSpanId":"5PoR/9GtyZ8=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025802869671","endTimeUnixNano":"1735335025803008213","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Ht0evUW5Oi8=","parentSpanId":"5PoR/9GtyZ8=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025803026513","endTimeUnixNano":"1735335025803153644","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"5PoR/9GtyZ8=","parentSpanId":"9zbnaSilf64=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025802107732","endTimeUnixNano":"1735335025803553979","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"9zbnaSilf64=","parentSpanId":"CfdXOWhYtlc=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025802041861","endTimeUnixNano":"1735335025803563059","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"rDn9JOsOTU4=","parentSpanId":"CfdXOWhYtlc=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025803784751","endTimeUnixNano":"1735335025803813032","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"CfdXOWhYtlc=","parentSpanId":"qxEQ86FHF6s=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025802002621","endTimeUnixNano":"1735335025803821102","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ZYwpZQe8Ypc=","parentSpanId":"AavMnl5RXEU=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025809118426","endTimeUnixNano":"1735335025809252097","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"AavMnl5RXEU=","parentSpanId":"gsWnxw7YYZ4=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025809084905","endTimeUnixNano":"1735335025809265737","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"P7rFWZXQDtU=","parentSpanId":"gsWnxw7YYZ4=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025809271637","endTimeUnixNano":"1735335025809305098","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"gsWnxw7YYZ4=","parentSpanId":"82uwbqPPsyw=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025809017044","endTimeUnixNano":"1735335025809311668","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"uS0c/bpC1QE=","parentSpanId":"89ZZ3/tdVtE=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025815968698","endTimeUnixNano":"1735335025816070779","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"89ZZ3/tdVtE=","parentSpanId":"vMeSVkU8tvE=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025815939228","endTimeUnixNano":"1735335025816085939","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"sMMZrzVKBwo=","parentSpanId":"vMeSVkU8tvE=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025816091679","endTimeUnixNano":"1735335025816137810","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"vMeSVkU8tvE=","parentSpanId":"heQjAAl6dfU=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025815832295","endTimeUnixNano":"1735335025816145190","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"IsHgfMSwOFc=","parentSpanId":"BjvlEOqqoYQ=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025821396242","endTimeUnixNano":"1735335025821479763","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"trf8MoiBLtw=","parentSpanId":"6j75o2UhjsM=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025821599005","endTimeUnixNano":"1735335025821762686","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"l0zVW4Qbs5Q=","parentSpanId":"6j75o2UhjsM=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025821775127","endTimeUnixNano":"1735335025821865809","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"uKeJNfsaBgU=","parentSpanId":"6j75o2UhjsM=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025821877069","endTimeUnixNano":"1735335025821985160","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"6j75o2UhjsM=","parentSpanId":"BjvlEOqqoYQ=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025821413642","endTimeUnixNano":"1735335025822306794","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"BjvlEOqqoYQ=","parentSpanId":"mF2wVQWgFZ0=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025821370152","endTimeUnixNano":"1735335025822315724","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"WB1ZRIGSTkY=","parentSpanId":"mF2wVQWgFZ0=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025822479446","endTimeUnixNano":"1735335025822512356","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"mF2wVQWgFZ0=","parentSpanId":"3Y8oPmEgblM=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025821345842","endTimeUnixNano":"1735335025822520676","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"en/si12hPAA=","parentSpanId":"955DafiANbI=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025824068025","endTimeUnixNano":"1735335025824176806","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"955DafiANbI=","parentSpanId":"xhQqyPAaa3s=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025824035004","endTimeUnixNano":"1735335025824191396","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"JgVQu1FL8Xg=","parentSpanId":"xhQqyPAaa3s=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025824196046","endTimeUnixNano":"1735335025824227327","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"xhQqyPAaa3s=","parentSpanId":"PiagOqYTpmM=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025823978244","endTimeUnixNano":"1735335025824229907","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"dQH9gToiFwg=","parentSpanId":"LbjnoYA5zpo=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025827339963","endTimeUnixNano":"1735335025827462485","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"LbjnoYA5zpo=","parentSpanId":"WyPW5d+ni+k=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025827307573","endTimeUnixNano":"1735335025827476625","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"BO5fc2gjmE4=","parentSpanId":"WyPW5d+ni+k=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025827482245","endTimeUnixNano":"1735335025827510355","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"WyPW5d+ni+k=","parentSpanId":"KKk10SNGrmc=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025827222892","endTimeUnixNano":"1735335025827517415","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"WCM+K0l/Fh8=","parentSpanId":"JNt2DRr7/Bg=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025834189906","endTimeUnixNano":"1735335025834295387","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"p2l78TY+CMQ=","parentSpanId":"nGIqcHMvUqs=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025834402658","endTimeUnixNano":"1735335025834580890","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"CrmdWPLP2Ss=","parentSpanId":"nGIqcHMvUqs=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025834594621","endTimeUnixNano":"1735335025834742182","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ZtLWldKyeFY=","parentSpanId":"nGIqcHMvUqs=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025834753742","endTimeUnixNano":"1735335025834837043","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"nGIqcHMvUqs=","parentSpanId":"JNt2DRr7/Bg=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025834221616","endTimeUnixNano":"1735335025835139387","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"JNt2DRr7/Bg=","parentSpanId":"D2FLhaWG8No=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025834158286","endTimeUnixNano":"1735335025835148707","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"pjk3dOwVGKU=","parentSpanId":"D2FLhaWG8No=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025835289348","endTimeUnixNano":"1735335025835314379","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"D2FLhaWG8No=","parentSpanId":"9g8u2zSWvxM=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025834117995","endTimeUnixNano":"1735335025835322209","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"1bOsLY65o2g=","parentSpanId":"QSVgmuuQlyk=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025836527134","endTimeUnixNano":"1735335025836617435","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"QSVgmuuQlyk=","parentSpanId":"hf9k8TOUFdQ=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025836499553","endTimeUnixNano":"1735335025836631825","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"w9ssqcECh8E=","parentSpanId":"hf9k8TOUFdQ=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025836637165","endTimeUnixNano":"1735335025836663395","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"hf9k8TOUFdQ=","parentSpanId":"86mKxrPrdQQ=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025836444483","endTimeUnixNano":"1735335025836665285","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"m3ofV4AgdzU=","parentSpanId":"zd/6MXSPyNg=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025839567369","endTimeUnixNano":"1735335025839665131","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"zd/6MXSPyNg=","parentSpanId":"gpBYkLZrl4c=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025839541359","endTimeUnixNano":"1735335025839679011","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"IHKu0mcP9wE=","parentSpanId":"gpBYkLZrl4c=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025839683671","endTimeUnixNano":"1735335025839707591","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"gpBYkLZrl4c=","parentSpanId":"VchSVDx2hio=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025839471088","endTimeUnixNano":"1735335025839709111","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"hlJITdy5GbM=","parentSpanId":"nmW6dmDbCZ0=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025847066409","endTimeUnixNano":"1735335025847203041","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"YKX2PFK9kUo=","parentSpanId":"RTL/QJCpqKo=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025847364843","endTimeUnixNano":"1735335025847548445","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"2tmsOuk2trk=","parentSpanId":"RTL/QJCpqKo=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025847564305","endTimeUnixNano":"1735335025847650146","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"q4oGWBE7P2o=","parentSpanId":"RTL/QJCpqKo=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025847661786","endTimeUnixNano":"1735335025847729647","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"RTL/QJCpqKo=","parentSpanId":"nmW6dmDbCZ0=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025847087100","endTimeUnixNano":"1735335025848095482","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"nmW6dmDbCZ0=","parentSpanId":"bjGYD26UMW4=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025847024189","endTimeUnixNano":"1735335025848104842","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Z2cxpjgDXKo=","parentSpanId":"bjGYD26UMW4=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025848274294","endTimeUnixNano":"1735335025848301155","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"bjGYD26UMW4=","parentSpanId":"04nIoKZ7Rn8=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025846988918","endTimeUnixNano":"1735335025848309595","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"RhqhM2Lw8Cw=","parentSpanId":"Jpo9gKiZZbA=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025849399137","endTimeUnixNano":"1735335025849478718","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Jpo9gKiZZbA=","parentSpanId":"AuMbefBUznA=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025849372607","endTimeUnixNano":"1735335025849492258","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Nf4cA//0tWc=","parentSpanId":"AuMbefBUznA=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025849496388","endTimeUnixNano":"1735335025849519459","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"AuMbefBUznA=","parentSpanId":"ZaPF80+dmj0=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025849324586","endTimeUnixNano":"1735335025849520989","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"CU1kvqJd8pY=","parentSpanId":"qv941ZWIXdc=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025852347363","endTimeUnixNano":"1735335025852451264","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"qv941ZWIXdc=","parentSpanId":"AkCsHLC/QWw=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025852318633","endTimeUnixNano":"1735335025852466974","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"M8M06S5Ax0U=","parentSpanId":"AkCsHLC/QWw=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025852471954","endTimeUnixNano":"1735335025852501705","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"AkCsHLC/QWw=","parentSpanId":"4k00P8wSf5A=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025852241682","endTimeUnixNano":"1735335025852503465","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Esv4dxLKSzg=","parentSpanId":"KTbmH0YYMbs=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025868750089","endTimeUnixNano":"1735335025868853830","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"iPrUuthF+So=","parentSpanId":"r65t/8B85Eo=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025869072113","endTimeUnixNano":"1735335025869308355","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"15r71FlGjP0=","parentSpanId":"r65t/8B85Eo=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025869326245","endTimeUnixNano":"1735335025869420507","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ZFDnmObzkoY=","parentSpanId":"r65t/8B85Eo=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025869433287","endTimeUnixNano":"1735335025869522318","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"r65t/8B85Eo=","parentSpanId":"KTbmH0YYMbs=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025868781859","endTimeUnixNano":"1735335025870036635","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"KTbmH0YYMbs=","parentSpanId":"lpDozXJHt3I=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025868718519","endTimeUnixNano":"1735335025870047135","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"dGyTLkjazfI=","parentSpanId":"lpDozXJHt3I=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025870231497","endTimeUnixNano":"1735335025870272947","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"lpDozXJHt3I=","parentSpanId":"AWRkklChgtM=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025868684858","endTimeUnixNano":"1735335025870289748","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ukceDzC3mkU=","parentSpanId":"kzEZGsLQ6Kk=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025871459001","endTimeUnixNano":"1735335025871560392","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"kzEZGsLQ6Kk=","parentSpanId":"JBX+Bty4YDA=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025871436931","endTimeUnixNano":"1735335025871576662","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Dqi6tJi5ct4=","parentSpanId":"JBX+Bty4YDA=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025871581612","endTimeUnixNano":"1735335025871611953","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"JBX+Bty4YDA=","parentSpanId":"egVsvBSrdgA=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025871373260","endTimeUnixNano":"1735335025871613623","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Hbe4hwbmNH8=","parentSpanId":"N3lmTyy+O4Q=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025874539408","endTimeUnixNano":"1735335025874617299","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"N3lmTyy+O4Q=","parentSpanId":"3Nk2CzroXhE=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025874497358","endTimeUnixNano":"1735335025874630559","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"V9Iz75CIWkI=","parentSpanId":"3Nk2CzroXhE=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025874641509","endTimeUnixNano":"1735335025874666730","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"3Nk2CzroXhE=","parentSpanId":"QdUTNSa6qh0=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025874420637","endTimeUnixNano":"1735335025874672780","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"baJUDiZjW24=","parentSpanId":"1/N7WxCrg+A=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025888077280","endTimeUnixNano":"1735335025888180602","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"PdGcA51RQic=","parentSpanId":"feitCe313qQ=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025888339613","endTimeUnixNano":"1735335025888562636","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"UovJyUrDUj8=","parentSpanId":"feitCe313qQ=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025888581186","endTimeUnixNano":"1735335025888663237","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"GkjNF0cAMyw=","parentSpanId":"feitCe313qQ=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025888674147","endTimeUnixNano":"1735335025888761718","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"feitCe313qQ=","parentSpanId":"1/N7WxCrg+A=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025888088331","endTimeUnixNano":"1735335025889199503","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"1/N7WxCrg+A=","parentSpanId":"dVGOYu2GHuQ=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025888050010","endTimeUnixNano":"1735335025889209203","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"qZB8lZItX/8=","parentSpanId":"dVGOYu2GHuQ=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025889399106","endTimeUnixNano":"1735335025889425796","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"dVGOYu2GHuQ=","parentSpanId":"gaEA+BN+3kI=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025888027330","endTimeUnixNano":"1735335025889434086","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"X6t41m2jxuQ=","parentSpanId":"UQgm8R8CPu4=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025890572690","endTimeUnixNano":"1735335025890655181","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"UQgm8R8CPu4=","parentSpanId":"6j4JzApql/8=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025890531350","endTimeUnixNano":"1735335025890668221","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"mYGGhy9JPXM=","parentSpanId":"6j4JzApql/8=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025890678251","endTimeUnixNano":"1735335025890701492","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"6j4JzApql/8=","parentSpanId":"tUL29k86aA0=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025890483229","endTimeUnixNano":"1735335025890707882","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"0p9gZYL0WMw=","parentSpanId":"Ak/AhdqIt1Q=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025893972201","endTimeUnixNano":"1735335025894060642","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Ak/AhdqIt1Q=","parentSpanId":"mM6hh4zyNaY=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025893943961","endTimeUnixNano":"1735335025894074652","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"2Pjk6OMh7QY=","parentSpanId":"mM6hh4zyNaY=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025894079212","endTimeUnixNano":"1735335025894103792","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"mM6hh4zyNaY=","parentSpanId":"ojzIzoRpdkE=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025893823268","endTimeUnixNano":"1735335025894105402","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Ph5DuK/hPIs=","parentSpanId":"VDjrSWqMTyo=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025933615094","endTimeUnixNano":"1735335025933712166","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Jk23ZxyiZAI=","parentSpanId":"Pv/OUxSoLt0=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025933842257","endTimeUnixNano":"1735335025934053800","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"22nudVC7nbI=","parentSpanId":"Pv/OUxSoLt0=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025934070661","endTimeUnixNano":"1735335025934172422","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"tE5jOpi7xag=","parentSpanId":"Pv/OUxSoLt0=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025934185102","endTimeUnixNano":"1735335025934273633","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Pv/OUxSoLt0=","parentSpanId":"VDjrSWqMTyo=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025933641165","endTimeUnixNano":"1735335025934632597","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"VDjrSWqMTyo=","parentSpanId":"4xChUK+lUQA=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025933581614","endTimeUnixNano":"1735335025934642447","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"LeP3Wa8uaUQ=","parentSpanId":"4xChUK+lUQA=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025934788779","endTimeUnixNano":"1735335025934814629","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"4xChUK+lUQA=","parentSpanId":"iI6JC+zkuuI=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025933548594","endTimeUnixNano":"1735335025934830159","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"UQmJPJkHcto=","parentSpanId":"ooJJdyUu9E0=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025935853461","endTimeUnixNano":"1735335025935972563","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ooJJdyUu9E0=","parentSpanId":"jDMcjJG1E0g=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025935821601","endTimeUnixNano":"1735335025935985504","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"tRutovykdVQ=","parentSpanId":"jDMcjJG1E0g=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025935990094","endTimeUnixNano":"1735335025936014614","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"jDMcjJG1E0g=","parentSpanId":"jozir3kDgsM=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025935760090","endTimeUnixNano":"1735335025936016204","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Xx9BJ7LoYww=","parentSpanId":"YSKgfHDwYVU=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025942413670","endTimeUnixNano":"1735335025942497731","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"YSKgfHDwYVU=","parentSpanId":"wytgul5s1Bw=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025942395640","endTimeUnixNano":"1735335025942511661","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"8fSOqkwi94o=","parentSpanId":"wytgul5s1Bw=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025942515701","endTimeUnixNano":"1735335025942543032","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"wytgul5s1Bw=","parentSpanId":"HSPi+PbWXh0=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025942324049","endTimeUnixNano":"1735335025942544302","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"PlKDELGMRME=","parentSpanId":"FDNGCAVYL6s=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025979828997","endTimeUnixNano":"1735335025979952199","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"A5Eu+VH/j/k=","parentSpanId":"zqN5at4EmC8=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025980145902","endTimeUnixNano":"1735335025981619709","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Hr6dVe1Xl7k=","parentSpanId":"zqN5at4EmC8=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025981641269","endTimeUnixNano":"1735335025982429099","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"oaUyaHxddFI=","parentSpanId":"zqN5at4EmC8=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025982455469","endTimeUnixNano":"1735335025982481589","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"zqN5at4EmC8=","parentSpanId":"FDNGCAVYL6s=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025979848637","endTimeUnixNano":"1735335025982943405","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"FDNGCAVYL6s=","parentSpanId":"oOGatcJRz4o=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025979790987","endTimeUnixNano":"1735335025982953885","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"/7q/6uvS7PQ=","parentSpanId":"oOGatcJRz4o=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025983155277","endTimeUnixNano":"1735335025983184887","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"oOGatcJRz4o=","parentSpanId":"FtTGqe5YvtE=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025979755566","endTimeUnixNano":"1735335025983192618","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ddsiV4yPi00=","parentSpanId":"EVCUlNnGnZA=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025984529224","endTimeUnixNano":"1735335025984663775","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"EVCUlNnGnZA=","parentSpanId":"CaypUioEVro=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025984489183","endTimeUnixNano":"1735335025984678796","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"j4auuJdL6Fc=","parentSpanId":"CaypUioEVro=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025984685216","endTimeUnixNano":"1735335025984723086","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"CaypUioEVro=","parentSpanId":"owAwqwUWHX4=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025984410812","endTimeUnixNano":"1735335025984729996","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"C3GISENbQC4=","parentSpanId":"nKI+9NglJjs=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025987822973","endTimeUnixNano":"1735335025987930624","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"nKI+9NglJjs=","parentSpanId":"WYB/uQwJqxw=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025987793652","endTimeUnixNano":"1735335025987943574","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"9RtAyEpXyF8=","parentSpanId":"WYB/uQwJqxw=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025987948154","endTimeUnixNano":"1735335025987974775","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"WYB/uQwJqxw=","parentSpanId":"ayYTEGRLA/4=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025987708481","endTimeUnixNano":"1735335025987977925","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"AWsYWu+Xc0I=","parentSpanId":"Ze5RueXjySs=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025994341641","endTimeUnixNano":"1735335025994428252","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Ze5RueXjySs=","parentSpanId":"Od0H5CjLwYU=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025994316841","endTimeUnixNano":"1735335025994441292","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"MN1XjKdi+4U=","parentSpanId":"Od0H5CjLwYU=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025994445612","endTimeUnixNano":"1735335025994471673","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Od0H5CjLwYU=","parentSpanId":"2cTcRlCzz7A=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025994259650","endTimeUnixNano":"1735335025994477013","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"0y4T4EjPqoA=","parentSpanId":"P3sUoSO2QW8=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025997382847","endTimeUnixNano":"1735335025997472788","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"P3sUoSO2QW8=","parentSpanId":"9xP3tf6dYZc=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025997355987","endTimeUnixNano":"1735335025997486558","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"YMIGvR9FWUI=","parentSpanId":"9xP3tf6dYZc=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025997491278","endTimeUnixNano":"1735335025997516819","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"9xP3tf6dYZc=","parentSpanId":"jn2jF/UGEwI=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025997284916","endTimeUnixNano":"1735335025997519729","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"1DHjc96hDgo=","parentSpanId":"fKTN7FMA9bA=","flags":256,"name":"authorize.getDataBrokerSessionOrServiceAccount","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025999408481","endTimeUnixNano":"1735335026000304333","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/databroker.go:64"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"A+TRUlQLrHs=","parentSpanId":"fKTN7FMA9bA=","flags":256,"name":"authorize.getDataBrokerUser","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026000307673","endTimeUnixNano":"1735335026001064951","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/databroker.go:97"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"qPBApgBMYDo=","parentSpanId":"H3n1ziGO7oI=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026001326014","endTimeUnixNano":"1735335026001992103","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"815f0b5e-a8a0-48d0-9ece-e1a8a1ed9e03\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"sC9Owg0pKCg=","parentSpanId":"H3n1ziGO7oI=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026002013393","endTimeUnixNano":"1735335026002274936","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"815f0b5e-a8a0-48d0-9ece-e1a8a1ed9e03\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"KBPImvf4CCY=","parentSpanId":"H3n1ziGO7oI=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026002369397","endTimeUnixNano":"1735335026002500159","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"815f0b5e-a8a0-48d0-9ece-e1a8a1ed9e03\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"STC1CzycLEI=","parentSpanId":"H3n1ziGO7oI=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026002553109","endTimeUnixNano":"1735335026002670921","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.User\""}},{"key":"record_id","value":{"stringValue":"\"623709e6-f5b8-445c-b93b-641d72e148a6\""}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"/7zHyVfkJXI=","parentSpanId":"rU3EQcGjQag=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026001109672","endTimeUnixNano":"1735335026002707651","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"H3n1ziGO7oI=","parentSpanId":"rU3EQcGjQag=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026001117002","endTimeUnixNano":"1735335026003087055","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"rU3EQcGjQag=","parentSpanId":"fKTN7FMA9bA=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026001079121","endTimeUnixNano":"1735335026003097476","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"L+E5uS0Pmj8=","parentSpanId":"fKTN7FMA9bA=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026003101526","endTimeUnixNano":"1735335026003142976","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:27"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"fKTN7FMA9bA=","parentSpanId":"kKNQsnG+JKs=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025999274240","endTimeUnixNano":"1735335026003151306","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"N+v2AAwKVfI=","parentSpanId":"K5lGkBkLfGA=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025802655949","endTimeUnixNano":"1735335025802690289","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"E93VUaFf2Js=","parentSpanId":"WVzMc/GgbKk=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025802953572","endTimeUnixNano":"1735335025802978622","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"Z7SxNTRbIlI=","parentSpanId":"Ht0evUW5Oi8=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025803094304","endTimeUnixNano":"1735335025803117944","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"9DQy+AMvVdU=","parentSpanId":"trf8MoiBLtw=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025821664945","endTimeUnixNano":"1735335025821682636","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ivezjXbKh00=","parentSpanId":"l0zVW4Qbs5Q=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025821824757","endTimeUnixNano":"1735335025821841718","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"+Y+OAZu4k5E=","parentSpanId":"uKeJNfsaBgU=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025821946920","endTimeUnixNano":"1735335025821961990","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"yP28QGB/eIQ=","parentSpanId":"p2l78TY+CMQ=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025834474299","endTimeUnixNano":"1735335025834498179","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"3gTVWLuOnUA=","parentSpanId":"CrmdWPLP2Ss=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025834698192","endTimeUnixNano":"1735335025834715032","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"kgdUpDdt3Zo=","parentSpanId":"ZtLWldKyeFY=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025834802283","endTimeUnixNano":"1735335025834817913","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"VsDbNMPWFyo=","parentSpanId":"YKX2PFK9kUo=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025847440004","endTimeUnixNano":"1735335025847459274","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"NpHyYPqtEcw=","parentSpanId":"2tmsOuk2trk=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025847614826","endTimeUnixNano":"1735335025847629396","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"iimzfcbaGXw=","parentSpanId":"q4oGWBE7P2o=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025847704227","endTimeUnixNano":"1735335025847717107","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"NG0e34bILr0=","parentSpanId":"iPrUuthF+So=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025869171194","endTimeUnixNano":"1735335025869191354","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"WR68p7UO4YI=","parentSpanId":"15r71FlGjP0=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025869382826","endTimeUnixNano":"1735335025869402586","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"GWfj0lLpLIo=","parentSpanId":"ZFDnmObzkoY=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025869475787","endTimeUnixNano":"1735335025869497057","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"o/4RdylU04s=","parentSpanId":"PdGcA51RQic=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025888424664","endTimeUnixNano":"1735335025888452865","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"vV8NB5ruoOU=","parentSpanId":"UovJyUrDUj8=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025888627447","endTimeUnixNano":"1735335025888640797","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"h2MSFkLt9p4=","parentSpanId":"GkjNF0cAMyw=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025888727348","endTimeUnixNano":"1735335025888745088","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"2CeZdqym8xg=","parentSpanId":"Jk23ZxyiZAI=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025933940429","endTimeUnixNano":"1735335025933958539","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"TDCb5mGNTys=","parentSpanId":"22nudVC7nbI=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025934122361","endTimeUnixNano":"1735335025934147932","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"C42DkoQ/1Ic=","parentSpanId":"tE5jOpi7xag=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025934234913","endTimeUnixNano":"1735335025934254703","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:41075: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"xC/VB6ObsY8=","parentSpanId":"A5Eu+VH/j/k=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025980218182","endTimeUnixNano":"1735335025981570378","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"zQwYJxI53tM=","parentSpanId":"Hr6dVe1Xl7k=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025981702279","endTimeUnixNano":"1735335025982399588","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"qo+2ljnLo3s=","parentSpanId":"1DHjc96hDgo=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025999465152","endTimeUnixNano":"1735335026000214772","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"niGr/qWcz4g=","parentSpanId":"A+TRUlQLrHs=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335026000369583","endTimeUnixNano":"1735335026001014421","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"yo8tJLvB7F8=","parentSpanId":"/7zHyVfkJXI=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335026001200583","endTimeUnixNano":"1735335026001827350","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"GZGIXC4RfsY=","parentSpanId":"/7zHyVfkJXI=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335026001960572","endTimeUnixNano":"1735335026002569590","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, - {"timestamp":"2024-12-27T21:30:29.976009599Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/databroker/cache.go:94"}},{"key":"service.name","value":{"stringValue":"Data Broker"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"ba01ZDTHdtk=","parentSpanId":"wASe7nky7+s=","flags":256,"name":"databroker.grpc.AcquireLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024979877729","endTimeUnixNano":"1735335024979946490","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:83"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"BpM7OEx5blE=","parentSpanId":"PZh68vG8OFw=","flags":256,"name":"databroker.grpc.SyncLatest","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024980693738","endTimeUnixNano":"1735335024980859720","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:393"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"uSSBr2vDr+k=","parentSpanId":"gmAAOaMKlhw=","flags":256,"name":"databroker.grpc.SyncLatest","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024980710859","endTimeUnixNano":"1735335024980930291","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:393"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"JwezNJcX+Sw=","parentSpanId":"hcyPk5piJ2w=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025481032712","endTimeUnixNano":"1735335025481045612","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"Rt5U410X8d8=","parentSpanId":"c61OVvg7XN8=","flags":256,"name":"databroker.grpc.SyncLatest","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025791172121","endTimeUnixNano":"1735335025791253352","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:393"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"Gq0XIVz0V8w=","parentSpanId":"iVo7IHe6x6A=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025980612787","endTimeUnixNano":"1735335025980623447","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"6X2Mt+eDjks=","parentSpanId":"vZpvKbrCQqQ=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025981082192","endTimeUnixNano":"1735335025981188064","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"FbAKR5t/VF8=","parentSpanId":"2yTflfwGhOI=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025982084235","endTimeUnixNano":"1735335025982169766","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"uRXsqpb8sac=","parentSpanId":"cifxyFmv4dY=","flags":256,"name":"databroker.grpc.Get","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025990608827","endTimeUnixNano":"1735335025990630187","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:110"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"F3yNlNUGRtw=","parentSpanId":"vP37Rm86XmY=","flags":256,"name":"databroker.grpc.Put","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025991377145","endTimeUnixNano":"1735335025991412456","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:207"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"ymstrLvsdeM=","parentSpanId":"3ijdqATnWA0=","flags":256,"name":"databroker.grpc.Put","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025992282787","endTimeUnixNano":"1735335025992317287","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:207"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"JS26p672Ew8=","parentSpanId":"WgzfdQgrifQ=","flags":256,"name":"databroker.grpc.Get","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025995441274","endTimeUnixNano":"1735335025995463904","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:110"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"7yYzlt2jIZw=","parentSpanId":"AWyn5ABOBS0=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025999878447","endTimeUnixNano":"1735335025999959508","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"TinKSNbNkto=","parentSpanId":"QBjGZUW54F0=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026000717487","endTimeUnixNano":"1735335026000775288","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"mX+fLnOfT9c=","parentSpanId":"yHJAjGq7Cgg=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026001565227","endTimeUnixNano":"1735335026001621198","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"63xkmInB/OQ=","parentSpanId":"DC1wSzzAIIQ=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026002316647","endTimeUnixNano":"1735335026002366377","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"uGIKXrnoZgI=","parentSpanId":"48tM9guCB7U=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026481483556","endTimeUnixNano":"1735335026481500596","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"kUBB4g/3mLw=","parentSpanId":"CGVc3FYZu4Q=","flags":256,"name":"databroker.grpc.Report","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026622495634","endTimeUnixNano":"1735335026622541525","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/registry.go:27"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"AFuw9JL5Xes=","parentSpanId":"IPKDPNoyYVI=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026981558406","endTimeUnixNano":"1735335026981568306","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"ONbNFwPfwZM=","parentSpanId":"534wdGGTsVw=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335027481416914","endTimeUnixNano":"1735335027481433145","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"QJOnObIqIf8=","parentSpanId":"Kt6HdK0x1SI=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335027981632535","endTimeUnixNano":"1735335027981643605","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"GRV9N0Uu6WI=","parentSpanId":"s/79OOiIyW8=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335028481339059","endTimeUnixNano":"1735335028481354230","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"YpZ4gH4pj1s=","parentSpanId":"pNRMXAPmD0o=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335028981482941","endTimeUnixNano":"1735335028981492081","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"/BP6SqKMjSo=","parentSpanId":"u3CptJkv/oQ=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335029481050441","endTimeUnixNano":"1735335029481060561","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"wASe7nky7+s=","parentSpanId":"RExgtixuvIo=","flags":768,"name":"databroker.DataBrokerService/AcquireLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335024979764968","endTimeUnixNano":"1735335024980042561","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"AcquireLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"RExgtixuvIo=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/AcquireLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335024979268002","endTimeUnixNano":"1735335024980212473","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"AcquireLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"PZh68vG8OFw=","parentSpanId":"yecKEUDPiwE=","flags":768,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335024980602177","endTimeUnixNano":"1735335024980873570","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"gmAAOaMKlhw=","parentSpanId":"uTXSTu1o/4w=","flags":768,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335024980622358","endTimeUnixNano":"1735335024980940641","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"yecKEUDPiwE=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335024980357704","endTimeUnixNano":"1735335024981013972","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"uTXSTu1o/4w=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335024980347634","endTimeUnixNano":"1735335024981035892","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"hcyPk5piJ2w=","parentSpanId":"Jf7MJl1DrwM=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025480950041","endTimeUnixNano":"1735335025481084252","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"Jf7MJl1DrwM=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025480640167","endTimeUnixNano":"1735335025481266504","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"iVo7IHe6x6A=","parentSpanId":"bcj0AK8Y/vs=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025980551466","endTimeUnixNano":"1735335025980651767","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"bcj0AK8Y/vs=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025980403345","endTimeUnixNano":"1735335025980758629","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"48tM9guCB7U=","parentSpanId":"P8qyVdH0S0I=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335026481393325","endTimeUnixNano":"1735335026481539477","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"P8qyVdH0S0I=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335026481185033","endTimeUnixNano":"1735335026481632248","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"IPKDPNoyYVI=","parentSpanId":"0saA2tMcvZk=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335026981468595","endTimeUnixNano":"1735335026981600897","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"0saA2tMcvZk=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335026981228112","endTimeUnixNano":"1735335026981720128","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"534wdGGTsVw=","parentSpanId":"XoADz2djL1c=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335027481298093","endTimeUnixNano":"1735335027481487875","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"XoADz2djL1c=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335027481013320","endTimeUnixNano":"1735335027481626977","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"Kt6HdK0x1SI=","parentSpanId":"e04LxDH4n8U=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335027981534604","endTimeUnixNano":"1735335027981689146","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"e04LxDH4n8U=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335027981301780","endTimeUnixNano":"1735335027981813137","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"s/79OOiIyW8=","parentSpanId":"Ok+FyABLV8k=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335028481247548","endTimeUnixNano":"1735335028481393530","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"Ok+FyABLV8k=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335028481041156","endTimeUnixNano":"1735335028481531892","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"pNRMXAPmD0o=","parentSpanId":"aSHOPBsOYTU=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335028981413940","endTimeUnixNano":"1735335028981515991","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"aSHOPBsOYTU=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335028981164807","endTimeUnixNano":"1735335028981576862","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"u3CptJkv/oQ=","parentSpanId":"gxf2r5UfW94=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335029480982791","endTimeUnixNano":"1735335029481085622","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"gxf2r5UfW94=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335029480740818","endTimeUnixNano":"1735335029481166113","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, - {"timestamp":"2024-12-27T21:30:30.080468097Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"mCf02Dsx/dPw0PWhdYTj3A==","spanId":"hKGR0btH9xI=","parentSpanId":"fxjsKYWTWL4=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025998816000","endTimeUnixNano":"1735335026004007000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"d9cf85d0-6ce9-93d2-83f4-31e616491757"}},{"key":"http.url","value":{"stringValue":"https://127.0.0.1:43211/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"2"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-29d1ce2f2cda20fb"}},{"key":"http.status_code","value":{"stringValue":"200"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.external-parent-span","value":{"stringValue":"7f18ec29859358be"}}]}]}]}]}}, - {"timestamp":"2024-12-27T21:30:30.193443488Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:377"}},{"key":"service.name","value":{"stringValue":"Test Environment"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"FRYjgKjC2fg=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335025791678197","endTimeUnixNano":"1735335030170817557","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"41075"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"error reading from server: EOF","code":"STATUS_CODE_ERROR"}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"eTl2DY0T39Y=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335030177708480","endTimeUnixNano":"1735335030180906089","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"error reading server preface: read tcp 127.0.0.1:33216-\u003e127.0.0.1:41075: read: connection reset by peer\"","code":"STATUS_CODE_ERROR"}}]},{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"wX7NySRpuEk=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"State Change Callback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335026004298180","endTimeUnixNano":"1735335030193020283","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:932"}},{"key":"state","value":{"stringValue":"Stopping"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"sXmaUcEg8Zo=","flags":256,"name":"TestOTLPTracing","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024644383476","endTimeUnixNano":"1735335030193209966","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:379"}}],"events":[{"timeUnixNano":"1735335024645032905","name":"Add: *scenarios.OTLPTraceReceiver from /home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:66"},{"timeUnixNano":"1735335024645076815","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735335024645146246","name":"AddUpstream: *upstreams.grpcUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:71"},{"timeUnixNano":"1735335024645174036","name":"Add: *upstreams.grpcUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:71"},{"timeUnixNano":"1735335024645180996","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735335024645195116","name":"AddTask: *upstreams.grpcUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:71"},{"timeUnixNano":"1735335024645213057","name":"AddUpstream: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:72"},{"timeUnixNano":"1735335024645222787","name":"Add: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:72"},{"timeUnixNano":"1735335024645229277","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735335024645240347","name":"AddTask: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:72"},{"timeUnixNano":"1735335024645326608","name":"Add: *scenarios.IDP from /home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:72"},{"timeUnixNano":"1735335024645338678","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735335024645456839","name":"AddUpstream: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/mock_idp.go:97"},{"timeUnixNano":"1735335024645490060","name":"Add: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/mock_idp.go:97"},{"timeUnixNano":"1735335024645496940","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735335024645508010","name":"AddTask: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/mock_idp.go:97"},{"timeUnixNano":"1735335024645584241","name":"AddUpstream: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:84"},{"timeUnixNano":"1735335024645606441","name":"Add: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:84"},{"timeUnixNano":"1735335024645613641","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735335024645624181","name":"Add: *testenv.PolicyRoute from /home/ubuntu/pomerium/internal/testenv/types.go:125"},{"timeUnixNano":"1735335024645635562","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735335024645645462","name":"AddTask: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:84"},{"timeUnixNano":"1735335024645664252","name":"Start()"},{"timeUnixNano":"1735335024645670342","name":"state NotRunning -\u003e Starting"},{"timeUnixNano":"1735335024645684982","name":"temp dir: /tmp/TestOTLPTracing380461907/001"},{"timeUnixNano":"1735335024646028336","name":"AddTask: *testenv.taskFunc from /home/ubuntu/pomerium/internal/testenv/environment.go:608"},{"timeUnixNano":"1735335024646118027","name":"state Starting -\u003e Running"},{"timeUnixNano":"1735335024738510112","name":"envoy profiling not available"},{"timeUnixNano":"1735335026004215959","name":"stop: Stop() called manually"},{"timeUnixNano":"1735335026004266750","name":"state Running -\u003e Stopping"},{"timeUnixNano":"1735335026004276110","name":"notifying 3 listeners of state change"},{"timeUnixNano":"1735335026004308931","name":"invoking state change callback (caller: /home/ubuntu/pomerium/internal/testenv/environment.go:659)"},{"timeUnixNano":"1735335026004339961","name":"invoking state change callback (caller: /home/ubuntu/pomerium/internal/testenv/snippets/shutdown.go:17)"},{"timeUnixNano":"1735335026004372111","name":"invoking state change callback (caller: /home/ubuntu/pomerium/internal/testenv/snippets/shutdown.go:17)"},{"timeUnixNano":"1735335030192986643","name":"pomerium server shut down without error"},{"timeUnixNano":"1735335030193042454","name":"done notifying state change listeners"}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, - {"timestamp":"2024-12-27T21:30:30.193784142Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/server.go:85"}},{"key":"service.name","value":{"stringValue":"Control Plane"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"Kcsl2I0u90IL3RjX9ECWQw==","spanId":"vdSorNgp3tM=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335030080404696","endTimeUnixNano":"1735335030080571098","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52388"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"c2UxzUHBdbEMq0ms+REjYg==","spanId":"k6YpnvzKBjQ=","flags":768,"name":"envoy.service.accesslog.v3.AccessLogService/StreamAccessLogs","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335026081822587","endTimeUnixNano":"1735335030118825295","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.accesslog.v3.AccessLogService"}},{"key":"rpc.method","value":{"stringValue":"StreamAccessLogs"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}},{"traceId":"z+C0XOvX6j/JSUWh8/UHWA==","spanId":"2Hpg8GvNW0k=","flags":768,"name":"envoy.service.accesslog.v3.AccessLogService/StreamAccessLogs","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025856359311","endTimeUnixNano":"1735335030123684784","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.accesslog.v3.AccessLogService"}},{"key":"rpc.method","value":{"stringValue":"StreamAccessLogs"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"gwpAbVXcwdw=","parentSpanId":"FRYjgKjC2fg=","flags":768,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025792058502","endTimeUnixNano":"1735335030171437535","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}},{"traceId":"1s487gww7zxt1qWc+o35qA==","spanId":"vOl62ewaIDc=","flags":768,"name":"envoy.service.discovery.v3.AggregatedDiscoveryService/DeltaAggregatedResources","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335025036599817","endTimeUnixNano":"1735335030174370480","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.discovery.v3.AggregatedDiscoveryService"}},{"key":"rpc.method","value":{"stringValue":"DeltaAggregatedResources"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"52346"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, - {"timestamp":"2024-12-27T21:30:30.194141156Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/databroker/cache.go:94"}},{"key":"service.name","value":{"stringValue":"Data Broker"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"CLyx0UG93i0=","parentSpanId":"xPK4CiYdt1w=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335029981041359","endTimeUnixNano":"1735335029981050839","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"qG+NVOCrqCo=","parentSpanId":"gwpAbVXcwdw=","flags":256,"name":"databroker.grpc.Sync","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335025792173903","endTimeUnixNano":"1735335030171424275","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:355"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"NS5usMrn7yI=","parentSpanId":"EF1NpNpk1bM=","flags":256,"name":"databroker.grpc.Sync","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024981421088","endTimeUnixNano":"1735335030192340375","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:355"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"1JC9QOassrE=","parentSpanId":"/+qBjKQTDHQ=","flags":256,"name":"databroker.grpc.Sync","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735335024981366457","endTimeUnixNano":"1735335030192345195","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:355"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"xPK4CiYdt1w=","parentSpanId":"6mMToadBqf4=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335029980977879","endTimeUnixNano":"1735335029981082870","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"6mMToadBqf4=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335029980793386","endTimeUnixNano":"1735335029981161621","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"mqSZh+et1KQ=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335024981134203","endTimeUnixNano":"1735335030192211813","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{"message":"context canceled","code":"STATUS_CODE_ERROR"}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"mA2k4+Vryho=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335024981065413","endTimeUnixNano":"1735335030192267074","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"45869"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{"message":"context canceled","code":"STATUS_CODE_ERROR"}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"/+qBjKQTDHQ=","parentSpanId":"mA2k4+Vryho=","flags":768,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335024981276086","endTimeUnixNano":"1735335030192360775","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"EF1NpNpk1bM=","parentSpanId":"mqSZh+et1KQ=","flags":768,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735335024981311536","endTimeUnixNano":"1735335030192363645","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"39040"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"/luU/UTJghg=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/ReleaseLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335030192357005","endTimeUnixNano":"1735335030192453166","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"ReleaseLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"the connection is draining","code":"STATUS_CODE_ERROR"}},{"traceId":"KvxAB8QGyzbvoUw4etvJLQ==","spanId":"mwChaFmJX8g=","parentSpanId":"sXmaUcEg8Zo=","flags":256,"name":"databroker.DataBrokerService/ReleaseLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735335030192555787","endTimeUnixNano":"1735335030192839370","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"ReleaseLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:45869: connect: connection refused\"","code":"STATUS_CODE_ERROR"}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}} -] diff --git a/internal/telemetry/trace/testdata/recording_02_multi_trace.json b/internal/telemetry/trace/testdata/recording_02_multi_trace.json deleted file mode 100644 index a5241ed76..000000000 --- a/internal/telemetry/trace/testdata/recording_02_multi_trace.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - {"timestamp":"2024-12-30T22:09:12.0668664Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"+R7poiK6/239h1nEy5hIrA==","spanId":"7pbeHJTanQw=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552060390000","endTimeUnixNano":"1735596552063277000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"d359b246-91e9-9c73-9975-8fd66089853f"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:35065/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"0"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"+R7poiK6/239h1nEy5hIrA==","spanId":"ynwZv1w6sTA=","parentSpanId":"7pbeHJTanQw=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552060836000","endTimeUnixNano":"1735596552063332000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"2"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"BI0pXOVFSFDjDh4HlytjyA==","spanId":"INSLthkL30A=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552064753000","endTimeUnixNano":"1735596552065999000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"52243523-9ac3-9bc0-bfa0-49407482f874"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:35065/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"0"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, - {"timestamp":"2024-12-30T22:09:12.071787916Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"BI0pXOVFSFDjDh4HlytjyA==","spanId":"7hYMiT5DOHg=","parentSpanId":"INSLthkL30A=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552064892000","endTimeUnixNano":"1735596552066228000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"2"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"nD9dzNOnTKOIJMxmlWZ9bw==","spanId":"1mtI7TODihI=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552070193000","endTimeUnixNano":"1735596552071393000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"b0ee2c5d-dbbb-9e99-a761-52475c57d45b"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:35065/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"0"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"nD9dzNOnTKOIJMxmlWZ9bw==","spanId":"sIew416JLXc=","parentSpanId":"1mtI7TODihI=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552070307000","endTimeUnixNano":"1735596552071417000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"2"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, - {"timestamp":"2024-12-30T22:09:12.088508581Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"b0XQMNrGpRts3otatJ4VkQ==","spanId":"Fry+Em1HWnk=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552077561000","endTimeUnixNano":"1735596552078549000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"e8b5646e-6da4-951e-9e39-79ee2ad7f54f"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:35065/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"0"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"b0XQMNrGpRts3otatJ4VkQ==","spanId":"poak1tCu/Ds=","parentSpanId":"Fry+Em1HWnk=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552077719000","endTimeUnixNano":"1735596552078573000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"2"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"lCsEeGUuNYemOgQ6EAqw/Q==","spanId":"lVjYK8BjGEA=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552087015000","endTimeUnixNano":"1735596552088139000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"2d9063c8-0eb2-9b72-a077-759abcfaeec3"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:35065/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"0"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, - {"timestamp":"2024-12-30T22:09:12.098955033Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"lCsEeGUuNYemOgQ6EAqw/Q==","spanId":"BI5T6olEsIc=","parentSpanId":"lVjYK8BjGEA=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552087136000","endTimeUnixNano":"1735596552088265000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"2"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"69JHBTXAPq7zDqo19nNt3g==","spanId":"/8VC2W2xsyA=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552097531000","endTimeUnixNano":"1735596552098644000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"bc305fba-3cf8-96c4-b6d2-f4210a62866c"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:35065/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"0"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"69JHBTXAPq7zDqo19nNt3g==","spanId":"rlZX67RjnyY=","parentSpanId":"/8VC2W2xsyA=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552097660000","endTimeUnixNano":"1735596552098666000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"2"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, - {"timestamp":"2024-12-30T22:09:12.18635478Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"qGCrQnZvKm9AiV3UG+9y4A==","spanId":"5bcR4af/ywE=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552123774000","endTimeUnixNano":"1735596552124933000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"cfae5ccd-c26f-9e1f-b2e1-1e98200ec5b9"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:35065/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"0"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"qGCrQnZvKm9AiV3UG+9y4A==","spanId":"rNEObiMfLz0=","parentSpanId":"5bcR4af/ywE=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552123869000","endTimeUnixNano":"1735596552124967000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"2"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"oboPZ53IIkU+AOAisyhydQ==","spanId":"Z7HMwkovokw=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552184826000","endTimeUnixNano":"1735596552185961000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"8addfd98-9521-9b15-9f5a-50b3bb8ebe83"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:35065/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"0"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"500"}},{"key":"response_flags","value":{"stringValue":"UAEX"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}}]}]}]}}, - {"timestamp":"2024-12-30T22:09:12.253542922Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"oboPZ53IIkU+AOAisyhydQ==","spanId":"ws87AD1cVDE=","parentSpanId":"Z7HMwkovokw=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552184938000","endTimeUnixNano":"1735596552186090000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"2"}},{"key":"error","value":{"stringValue":"true"}}],"status":{"code":"STATUS_CODE_ERROR"}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"+OVHuklKQMo=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552248874000","endTimeUnixNano":"1735596552253121000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"be91c88e-0c02-9e51-b7ef-8de7c22ab1bb"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:35065/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"1450"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"UAEX"}}]},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"xCIoDfKUEmA=","parentSpanId":"+OVHuklKQMo=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552248989000","endTimeUnixNano":"1735596552253152000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_unauthorized"}}],"status":{}}]}]}]}}, - {"timestamp":"2024-12-30T22:09:12.262728948Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"DfyMBC/GxYWDl+LE0WQyhQ==","spanId":"acI32UNM7h4=","parentSpanId":"j8D1S9l24K0=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552257242000","endTimeUnixNano":"1735596552258519000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"DfyMBC/GxYWDl+LE0WQyhQ==","spanId":"FBPvHWyFn6I=","parentSpanId":"j8D1S9l24K0=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552258384000","endTimeUnixNano":"1735596552262124000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:37809"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:37809"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"DfyMBC/GxYWDl+LE0WQyhQ==","spanId":"j8D1S9l24K0=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552256907000","endTimeUnixNano":"1735596552262163000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"acad727c-3b98-42c2-8ef5-4a344b18be16"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:35065/.pomerium/sign_in?pomerium_expiry=1735596852\u0026pomerium_idp_id=C44yiF1cj9B5ky7OuS1Le2l5g8fulMelHIX6d6E4dCeB\u0026pomerium_issued=1735596552\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A35065%2Ffoo\u0026pomerium_signature=nQpfEBF3IsHFTzzpL1D7sFy-CHxB1sJTeBATVXukUeQ%3D\u0026pomerium_traceparent=00-0deff952685913fe4df3ac83a5efbb83-1dcc8cb5972746ae-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"792"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-0deff952685913fe4df3ac83a5efbb83-1dcc8cb5972746ae-01"}}]}]}]}]}}, - {"timestamp":"2024-12-30T22:09:12.272346001Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"E9PYtbtULsI=","parentSpanId":"QCoXgtwGqF0=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552264909000","endTimeUnixNano":"1735596552265740000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"qZJUemZrh0U=","parentSpanId":"QCoXgtwGqF0=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552265713000","endTimeUnixNano":"1735596552271945000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:37809"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:37809"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"QCoXgtwGqF0=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552264741000","endTimeUnixNano":"1735596552271969000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"ab366edc-1a2f-9ed9-851b-d982a3396ba9"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:35065/oauth2/callback?code=eyJlbWFpbCI6ImZvb0BleGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IkNMSUVOVF9JRCJ9\u0026state=dEVxQThvQ0k4amQwQWc1VExVN2JJS3I0N1FPd3JQc1l4NW5xSC9nSFZaTkFJaEJwN011RVptTm5oK0NVZ1Nya1dkNis1Z1VoUytjZGFlU1dGNFBoSUE9PXwxNzM1NTk2NTUyfDwRWSChr4x26k5xhXcIAFuQ49JDHDh0PW_YYl2LORSWykRnWoJEuD8UwPKxXsP_HF70JYwmOVesuA17GdKHTFWswy_ggJVJaKJgcWNQL2v-GcbmXWUcOnYi80bud98SpVaii44ozEbwJAkKuF8ZiUGGUBXtvqlazXUmyMWUgujOrXEnr8l-xQL5dMuj1b0szz3q8W3SrMc0wV_sSuGuV3d8Vi9KVvVvuJ3vtheLZWHvUtlppZLT2yyRUIwCyO6RCIZxY6huIPa8Sxs3Lgg61jFbpvWxbNNFq12Ux1Z4zIt2bhdVD9TFOv9QoBXhpZPHOYru0w2k7JfED7Iio9MPOjparZCsOZTzgs5j4eDYbpudwuJRdcPQ4tCZP_3M4WpUKzS8X_V56YewQM-tkk_DHqH7YdoU3RcNr4NC9G0kbp7mmtUQEVgZ1ZahtgfPAZKRGWFF4VFM-_O1gyGJUmPU6VW3QgeVKPyEUzlrZAeWgMmDr3SGIxX3RYnr8e0ttGzUzhv5WiVuMQJaAoQQPTJ6nxjBtzV13NFx1AnPDE5clRQEpWEiTr0z45NSWtfhLQE9LN4BkDLRTr0r"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"322"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-0deff952685913fe4df3ac83a5efbb83-1dcc8cb5972746ae-01"}}]}]}]}]}}, - {"timestamp":"2024-12-30T22:09:12.279487054Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"60XX4iML+BmEG3gZjznr1g==","spanId":"lWFmfug8YNY=","parentSpanId":"KE354oTf2M0=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552272538000","endTimeUnixNano":"1735596552273360000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"60XX4iML+BmEG3gZjznr1g==","spanId":"uuvSf1QFkxA=","parentSpanId":"KE354oTf2M0=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552273329000","endTimeUnixNano":"1735596552279103000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:37809"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:37809"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"60XX4iML+BmEG3gZjznr1g==","spanId":"KE354oTf2M0=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552272386000","endTimeUnixNano":"1735596552279120000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"e96483ca-4bae-483e-9240-30e79bc4b0b7"}},{"key":"http.url","value":{"stringValue":"https://authenticate.localhost.pomerium.io:35065/.pomerium/sign_in?pomerium_expiry=1735596852\u0026pomerium_idp_id=C44yiF1cj9B5ky7OuS1Le2l5g8fulMelHIX6d6E4dCeB\u0026pomerium_issued=1735596552\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A35065%2Ffoo\u0026pomerium_signature=nQpfEBF3IsHFTzzpL1D7sFy-CHxB1sJTeBATVXukUeQ%3D\u0026pomerium_traceparent=00-0deff952685913fe4df3ac83a5efbb83-1dcc8cb5972746ae-01"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"941"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-0deff952685913fe4df3ac83a5efbb83-1dcc8cb5972746ae-01"}}]}]}]}]}}, - {"timestamp":"2024-12-30T22:09:12.281856012Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"PAOA0r+dYsQn8fEvj1rlOQ==","spanId":"5ThhIloR5KY=","parentSpanId":"IfligTWwckk=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552279794000","endTimeUnixNano":"1735596552280758000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"PAOA0r+dYsQn8fEvj1rlOQ==","spanId":"LNr+ATKO4qk=","parentSpanId":"IfligTWwckk=","name":"router pomerium-control-plane-http egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552280625000","endTimeUnixNano":"1735596552281367000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:37809"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:37809"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"PAOA0r+dYsQn8fEvj1rlOQ==","spanId":"IfligTWwckk=","name":"internal: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552279582000","endTimeUnixNano":"1735596552281384000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"65383ba8-61d9-4cbb-bdb3-650b70eacedb"}},{"key":"http.url","value":{"stringValue":"https://foo.localhost.pomerium.io:35065/.pomerium/callback/?pomerium_expiry=1735596852\u0026pomerium_issued=1735596552\u0026pomerium_redirect_uri=https%3A%2F%2Ffoo.localhost.pomerium.io%3A35065%2Ffoo\u0026pomerium_session_encrypted=EWV_6baRK5R5-pUgJFqfUJK13XziMrN6yew9hwsfF_MX5DDsbz3LeQrQ7VpyWA4RWATPOqoOObC0Fjqac3VdaiFkpbTuiXsU8OzOU3m2iU01xkG0nGsgaz8gSfOsNH9vtfSlg73G8NjyelubtY6OGRHw64tHdy-BoGwp68m6bocVbrzUQgOHPXbn7vuFnZ5pj9NdAHQwEZOKNouE6kG6fuz2oZL6eLk9E-uPef5G0OV6Ewl6oNz71kR8f0Y-vdPwE9xDEuN6tWQP6yPeMT87I7KhIAhgwhbm6k_keZm6c3Bz6Mo10WevLJYhNLDQdkgJ0tU5sYObYp5gXMAK1w7lMviZ5bzrjC4asP2oeWaU4ZTg_l_mC0lsu7ztlcDlo1x3ILNuucABtMD1jIgE2H2-BWqGjd_oAxF-53SPNu251S-ciO8Sq1E-T3KP3UKAif9Y2qmVxEntQqxAKs3YzbIEaCwUB4mpr9aLY1SnBN9N2O2KpzLQs3OKdcaLHKv0_K675SaJJR9MUcqjSpUxlHv4BM3NQbA9p56wTCqqG7TJD9DTTCuxIDvLSQRimPI1mJsQic8f9ZjFLjkD_CdHnM1jy2ckoVvYwVELOa6-pPFfnlcqy93Gx7tBGYB7K8_OzAarv7YK54QiRykTpNBf-mymLuRtQcLf9hHsFKsF-Gfx_aforvpZhK4L9EqLE3fZZ3Y0LIAlbjbJzxuip0i4JoudSpbZuL7w2roxYj6dkSxJ_fj1HJVj47wbxPoU6TMqEG1Yu5t14oG3SrtRMxoVyBzwPClS2Hv8EnnizDUKr12uVVN9dux7P1xZ7Ve5qhJMrx4%3D\u0026pomerium_si"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"66"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"upstream_cluster.name","value":{"stringValue":"pomerium-control-plane-http"}},{"key":"http.status_code","value":{"stringValue":"302"}},{"key":"response_flags","value":{"stringValue":"-"}},{"key":"pomerium.traceparent","value":{"stringValue":"00-0deff952685913fe4df3ac83a5efbb83-1dcc8cb5972746ae-01"}}]}]}]}]}}, - {"timestamp":"2024-12-30T22:09:12.287783501Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"pomerium.envoy","value":{"stringValue":"true"}},{"key":"service.name","value":{"stringValue":"Envoy"}}]},"scopeSpans":[{"spans":[{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"6LX3bJmVP78=","parentSpanId":"HxQN6e2rYXU=","name":"async envoy.service.auth.v3.Authorization.Check egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552281917000","endTimeUnixNano":"1735596552286888000","attributes":[{"key":"upstream_cluster","value":{"stringValue":"pomerium-authorize"}},{"key":"upstream_address","value":{"stringValue":"pomerium-authorize"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"grpc.status_code","value":{"stringValue":"0"}},{"key":"ext_authz_status","value":{"stringValue":"ext_authz_ok"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"Nnmvh2NOoyE=","parentSpanId":"HxQN6e2rYXU=","name":"router route-8054bb0274b8b3dd egress","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552286739000","endTimeUnixNano":"1735596552287352000","attributes":[{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"upstream_address","value":{"stringValue":"127.0.0.1:44525"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1:44525"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"200"}},{"key":"response_flags","value":{"stringValue":"-"}}]},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"HxQN6e2rYXU=","name":"ingress: ${method} ${host}${path}","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552281793000","endTimeUnixNano":"1735596552287370000","attributes":[{"key":"node_id","value":{"stringValue":"pomerium"}},{"key":"zone","value":{"stringValue":""}},{"key":"guid:x-request-id","value":{"stringValue":"21cb5717-0472-9782-9751-6cac2d39ac3e"}},{"key":"http.url","value":{"stringValue":"https://127.0.0.1:44525/foo"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"downstream_cluster","value":{"stringValue":"-"}},{"key":"user_agent","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.protocol","value":{"stringValue":"HTTP/1.1"}},{"key":"peer.address","value":{"stringValue":"127.0.0.1"}},{"key":"request_size","value":{"stringValue":"0"}},{"key":"response_size","value":{"stringValue":"2"}},{"key":"component","value":{"stringValue":"proxy"}},{"key":"upstream_cluster","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"upstream_cluster.name","value":{"stringValue":"route-8054bb0274b8b3dd"}},{"key":"http.status_code","value":{"stringValue":"200"}},{"key":"response_flags","value":{"stringValue":"-"}}]}]}]}]}}, - {"timestamp":"2024-12-30T22:09:12.400423432Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:377"}},{"key":"service.name","value":{"stringValue":"Test Environment"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"MfzSNZegoFc=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"Start","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596550908732917","endTimeUnixNano":"1735596550909168452","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:543"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"DSJCIaOGrEM=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.config_source.rebuild","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551000633187","endTimeUnixNano":"1735596551001673799","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"6Qd1AEzhdis=","parentSpanId":"Woul7Heu4HE=","flags":256,"name":"envoyconfig.Builder.BuildBootstrapStaticResources","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551223409140","endTimeUnixNano":"1735596551223438210","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"Woul7Heu4HE=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"envoyconfig.Builder.BuildBootstrap","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551223351599","endTimeUnixNano":"1735596551223487891","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"cFxn05+6kBk=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"authorize.newPolicyEvaluator","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551233260585","endTimeUnixNano":"1735596551241016875","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"KTlvZ+eAgZg=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"authorize.newPolicyEvaluator","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551241293388","endTimeUnixNano":"1735596551241567001","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"6QO6cyUuZo8=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"snippets.WaitStartupComplete","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596550909198712","endTimeUnixNano":"1735596552053766546","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"HNOD2Zpqhns=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"State Change Callback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552287806201","endTimeUnixNano":"1735596552287850811","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:932"}},{"key":"state","value":{"stringValue":"Stopping"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"DlBnuxNmVK0=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"State Change Callback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552287824491","endTimeUnixNano":"1735596552287866952","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:932"}},{"key":"state","value":{"stringValue":"Stopping"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"gTtrtr+YbHo=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"State Change Callback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552287796291","endTimeUnixNano":"1735596552399637883","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:932"}},{"key":"state","value":{"stringValue":"Stopping"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"Qkk6itX7wm8=","flags":256,"name":"TestOTLPTracing_TraceCorrelation","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596550907558493","endTimeUnixNano":"1735596552399753564","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/environment.go:379"}}],"events":[{"timeUnixNano":"1735596550908132151","name":"Add: *scenarios.OTLPTraceReceiver from /home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:133"},{"timeUnixNano":"1735596550908190821","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735596550908242292","name":"AddUpstream: *upstreams.grpcUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:71"},{"timeUnixNano":"1735596550908262792","name":"Add: *upstreams.grpcUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:71"},{"timeUnixNano":"1735596550908269672","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735596550908285502","name":"AddTask: *upstreams.grpcUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:71"},{"timeUnixNano":"1735596550908307833","name":"AddUpstream: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:72"},{"timeUnixNano":"1735596550908326843","name":"Add: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:72"},{"timeUnixNano":"1735596550908336333","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735596550908347513","name":"AddTask: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/trace_receiver.go:72"},{"timeUnixNano":"1735596550908432174","name":"Add: *scenarios.IDP from /home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:139"},{"timeUnixNano":"1735596550908442314","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735596550908521745","name":"AddUpstream: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/mock_idp.go:97"},{"timeUnixNano":"1735596550908543275","name":"Add: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/mock_idp.go:97"},{"timeUnixNano":"1735596550908549705","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735596550908566855","name":"AddTask: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/scenarios/mock_idp.go:97"},{"timeUnixNano":"1735596550908674197","name":"AddUpstream: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:151"},{"timeUnixNano":"1735596550908689937","name":"Add: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:151"},{"timeUnixNano":"1735596550908697167","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735596550908707587","name":"Add: *testenv.PolicyRoute from /home/ubuntu/pomerium/internal/testenv/types.go:125"},{"timeUnixNano":"1735596550908714717","name":"Add: state=NotRunning; calling Attach"},{"timeUnixNano":"1735596550908724767","name":"AddTask: *upstreams.httpUpstream from /home/ubuntu/pomerium/internal/testenv/selftests/tracing_test.go:151"},{"timeUnixNano":"1735596550908739827","name":"Start()"},{"timeUnixNano":"1735596550908745937","name":"state NotRunning -\u003e Starting"},{"timeUnixNano":"1735596550908758458","name":"temp dir: /tmp/TestOTLPTracing_TraceCorrelation2182135484/001"},{"timeUnixNano":"1735596550909053601","name":"AddTask: *testenv.taskFunc from /home/ubuntu/pomerium/internal/testenv/environment.go:608"},{"timeUnixNano":"1735596550909130612","name":"state Starting -\u003e Running"},{"timeUnixNano":"1735596551000151162","name":"envoy profiling not available"},{"timeUnixNano":"1735596552287705270","name":"stop: Stop() called manually"},{"timeUnixNano":"1735596552287763410","name":"state Running -\u003e Stopping"},{"timeUnixNano":"1735596552287772900","name":"notifying 3 listeners of state change"},{"timeUnixNano":"1735596552287811151","name":"invoking state change callback (caller: /home/ubuntu/pomerium/internal/testenv/environment.go:659)"},{"timeUnixNano":"1735596552287828861","name":"invoking state change callback (caller: /home/ubuntu/pomerium/internal/testenv/snippets/shutdown.go:17)"},{"timeUnixNano":"1735596552287859241","name":"invoking state change callback (caller: /home/ubuntu/pomerium/internal/testenv/snippets/shutdown.go:17)"},{"timeUnixNano":"1735596552399607632","name":"pomerium server shut down without error"},{"timeUnixNano":"1735596552399645233","name":"done notifying state change listeners"}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"9BbvwqcIDE4=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552051616251","endTimeUnixNano":"1735596552053279621","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"kKNgDiOw0Jw=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"registry.Registry/Report","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552242718946","endTimeUnixNano":"1735596552242750267","attributes":[{"key":"rpc.service","value":{"stringValue":"registry.Registry"}},{"key":"rpc.method","value":{"stringValue":"Report"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"YjiylRy7lCw=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552053323951","endTimeUnixNano":"1735596552315228660","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"error reading from server: EOF","code":"STATUS_CODE_ERROR"}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"mysN7JNNyTY=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552327146689","endTimeUnixNano":"1735596552389290772","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"error reading server preface: read tcp 127.0.0.1:48314-\u003e127.0.0.1:40165: read: connection reset by peer\"","code":"STATUS_CODE_ERROR"}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"2Sqw7gm3daQ=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552397549129","endTimeUnixNano":"1735596552397602099","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"error reading server preface: read tcp 127.0.0.1:48314-\u003e127.0.0.1:40165: read: connection reset by peer\"","code":"STATUS_CODE_ERROR"}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, - {"timestamp":"2024-12-30T22:09:12.400742295Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/upstreams/http.go:223"}},{"key":"service.name","value":{"stringValue":"Upstream"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp","version":"0.57.0"},"spans":[{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"IKp7lJq2AVg=","parentSpanId":"Nnmvh2NOoyE=","flags":768,"name":"Server: GET /foo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552287072633","endTimeUnixNano":"1735596552287102743","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"127.0.0.1"}},{"key":"net.host.port","value":{"intValue":"44525"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"44066"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/foo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"2"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, - {"timestamp":"2024-12-30T22:09:12.401027609Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/testenv/upstreams/http.go:223"}},{"key":"service.name","value":{"stringValue":"IDP"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp","version":"0.57.0"},"spans":[{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"Q1AHuKM97DU=","parentSpanId":"+fYq7JMxmug=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552260527653","endTimeUnixNano":"1735596552260621394","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"43581"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"42722"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"Fd8uhzUZo099GwCYWCNoyQ==","spanId":"rxnESoh2tjw=","flags":256,"name":"Server: GET /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552264061484","endTimeUnixNano":"1735596552264138305","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"43581"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"42726"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"512"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"vtQLJDvERjrZ9z8pGENSYA==","spanId":"hjt1scYLMl8=","flags":256,"name":"Server: POST /oidc/auth","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552264425638","endTimeUnixNano":"1735596552264504769","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"43581"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"42726"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/auth"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"46"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"18piKxRJ7tw=","parentSpanId":"uhrN3Ns9n/w=","flags":768,"name":"Server: GET /.well-known/openid-configuration","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552266194929","endTimeUnixNano":"1735596552266253250","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"43581"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"42722"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/openid-configuration"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"421"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"40s9nB6GeQMtZnV+dJC7AA==","spanId":"oTzLO7CG3g4=","flags":256,"name":"Server: POST /oidc/token","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552266517173","endTimeUnixNano":"1735596552266727745","attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"43581"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"42722"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/token"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.request_content_length","value":{"intValue":"193"}},{"key":"http.response_content_length","value":{"intValue":"673"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"dwMPQOqC63eLDuFdJ/DRFg==","spanId":"WeTERDnkqsw=","parentSpanId":"BHBu4awlz/Y=","flags":768,"name":"Server: GET /.well-known/jwks.json","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552267120060","endTimeUnixNano":"1735596552267200661","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"43581"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"42722"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/.well-known/jwks.json"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"237"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}},{"traceId":"B5szkLtGsj4UnJtqkubX+g==","spanId":"FudIJLl7VtY=","flags":256,"name":"Server: GET /oidc/userinfo","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552267615176","endTimeUnixNano":"1735596552267670417","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"43581"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"42722"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.target","value":{"stringValue":"/oidc/userinfo"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"150"}},{"key":"http.status_code","value":{"intValue":"200"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, - {"timestamp":"2024-12-30T22:09:12.40196375Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/server.go:85"}},{"key":"service.name","value":{"stringValue":"Control Plane"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"x7/JV/qOoQc=","parentSpanId":"AF2VbjcLF8o=","flags":256,"name":"envoyconfig.Builder.BuildClusters","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551016707484","endTimeUnixNano":"1735596551017525873","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"WyZbPrj6kIk=","parentSpanId":"AF2VbjcLF8o=","flags":256,"name":"envoyconfig.Builder.BuildRouteConfigurations","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551016694654","endTimeUnixNano":"1735596551017745616","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"wRoHtbjfaz0=","parentSpanId":"AF2VbjcLF8o=","flags":256,"name":"envoyconfig.Builder.BuildListeners","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551016746665","endTimeUnixNano":"1735596551021080155","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"AF2VbjcLF8o=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"controlplane.Server.buildDiscoveryResources","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551016576533","endTimeUnixNano":"1735596551021622391","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/xds.go:23"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"pbntg72ezVo=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"controlplane.Server.OnConfigChange","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551021643761","endTimeUnixNano":"1735596551021656921","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/server.go:255"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"I5CVUhOnTU4=","parentSpanId":"lzi6B+5Q2nI=","flags":256,"name":"envoyconfig.Builder.BuildClusters","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551249315632","endTimeUnixNano":"1735596551249436133","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"zspRgxWRZco=","parentSpanId":"lzi6B+5Q2nI=","flags":256,"name":"envoyconfig.Builder.BuildRouteConfigurations","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551249235221","endTimeUnixNano":"1735596551249575035","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"lyCOdoUx1+Q=","parentSpanId":"lzi6B+5Q2nI=","flags":256,"name":"envoyconfig.Builder.BuildListeners","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551249249221","endTimeUnixNano":"1735596551250357424","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"lzi6B+5Q2nI=","parentSpanId":"jGekmqeQkJo=","flags":256,"name":"controlplane.Server.buildDiscoveryResources","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551249096239","endTimeUnixNano":"1735596551250474326","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/xds.go:23"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"jGekmqeQkJo=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"controlplane.Server.update","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551244810840","endTimeUnixNano":"1735596551250478946","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/controlplane/server.go:279"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"gHqxUi5kePg=","parentSpanId":"xSJX1oetTdU=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552258874805","endTimeUnixNano":"1735596552261729827","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"WV8SOlkANhI=","parentSpanId":"3jlv7DJBi34=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552265934866","endTimeUnixNano":"1735596552271656323","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"yhyoDzh9RIM=","parentSpanId":"E6AUkZz0o+g=","flags":256,"name":"middleware.SetHeaders","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552273575835","endTimeUnixNano":"1735596552276684431","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"RXv+G4LTsEk=","parentSpanId":"9BbvwqcIDE4=","flags":768,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552052610012","endTimeUnixNano":"1735596552052848066","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33088"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"+R7poiK6/239h1nEy5hIrA==","spanId":"wtBmI0TId5M=","parentSpanId":"ynwZv1w6sTA=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552061633468","endTimeUnixNano":"1735596552062922194","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"2"}}],"status":{"message":"authorize: error evaluating policy.rego: pomerium.policy:95: eval_builtin_error: get_databroker_record: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"BI0pXOVFSFDjDh4HlytjyA==","spanId":"CxZOOB7mGiQ=","parentSpanId":"7hYMiT5DOHg=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552065072369","endTimeUnixNano":"1735596552065842537","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"2"}}],"status":{"message":"authorize: error evaluating policy.rego: pomerium.policy:95: eval_builtin_error: get_databroker_record: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"BO49aXqcRX/rS1MR1IxpsA==","spanId":"2Z6YH7F78Lc=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552066447434","endTimeUnixNano":"1735596552066962551","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33104"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"nD9dzNOnTKOIJMxmlWZ9bw==","spanId":"lQvkuojiNVU=","parentSpanId":"sIew416JLXc=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552070481851","endTimeUnixNano":"1735596552071222610","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"2"}}],"status":{"message":"authorize: error evaluating policy.rego: pomerium.policy:95: eval_builtin_error: get_databroker_record: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"lCAPf0HR197f/cUT53zYCA==","spanId":"ITRzniYDlzA=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552071717826","endTimeUnixNano":"1735596552071903768","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33104"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"b0XQMNrGpRts3otatJ4VkQ==","spanId":"PNkHn7SfYNs=","parentSpanId":"poak1tCu/Ds=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552077900168","endTimeUnixNano":"1735596552078387493","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"2"}}],"status":{"message":"authorize: error evaluating policy.rego: pomerium.policy:95: eval_builtin_error: get_databroker_record: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"lCsEeGUuNYemOgQ6EAqw/Q==","spanId":"Xrh5E+qCpVc=","parentSpanId":"BI5T6olEsIc=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552087339348","endTimeUnixNano":"1735596552087950615","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"2"}}],"status":{"message":"authorize: error evaluating policy.rego: pomerium.policy:95: eval_builtin_error: get_databroker_record: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"oKHSD4aojA+P/UfWs5LSaQ==","spanId":"2zTy638Eq5k=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552088429270","endTimeUnixNano":"1735596552088637543","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33104"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"69JHBTXAPq7zDqo19nNt3g==","spanId":"nqbiIHn9gV8=","parentSpanId":"rlZX67RjnyY=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552097875770","endTimeUnixNano":"1735596552098442626","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"2"}}],"status":{"message":"authorize: error evaluating policy.rego: pomerium.policy:95: eval_builtin_error: get_databroker_record: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"V9c6M7AQm8lZF4prh3VxSA==","spanId":"yjD3ncvuvJs=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552098899903","endTimeUnixNano":"1735596552099043664","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33104"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"qGCrQnZvKm9AiV3UG+9y4A==","spanId":"7A4ZbdDiN2o=","parentSpanId":"rNEObiMfLz0=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552124101095","endTimeUnixNano":"1735596552124765313","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"2"}}],"status":{"message":"authorize: error evaluating policy.rego: pomerium.policy:95: eval_builtin_error: get_databroker_record: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"oboPZ53IIkU+AOAisyhydQ==","spanId":"13OK1T2dwk8=","parentSpanId":"ws87AD1cVDE=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552185178396","endTimeUnixNano":"1735596552185771603","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"2"}}],"status":{"message":"authorize: error evaluating policy.rego: pomerium.policy:95: eval_builtin_error: get_databroker_record: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"aoeEeDz4Wb1FDja90yAkEQ==","spanId":"W/Z9Zzkiag4=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552186283969","endTimeUnixNano":"1735596552186447471","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33104"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"anG2KFI46uE=","parentSpanId":"9BuTh+PJtJ8=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552250425405","endTimeUnixNano":"1735596552250672338","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33112"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"2ACwBEIpGZ8=","parentSpanId":"yOL5QyxGuD8=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552251440318","endTimeUnixNano":"1735596552251721171","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33112"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"XF9zGG11kRE=","parentSpanId":"xCIoDfKUEmA=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552249204022","endTimeUnixNano":"1735596552252813544","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"cOuoTnMUlEZjYEIVsCPbWg==","spanId":"6P4Ui3Adxa0=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552253489362","endTimeUnixNano":"1735596552253650093","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33104"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"BHkNMsuL5iw=","parentSpanId":"acI32UNM7h4=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552257718511","endTimeUnixNano":"1735596552258121035","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33128"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"HiRbcDL3iIRaCg/blwsIog==","spanId":"1jQJ0R0HrV0=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552262659298","endTimeUnixNano":"1735596552262815790","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33132"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"mDALe+Jrgfg=","parentSpanId":"E9PYtbtULsI=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552265119127","endTimeUnixNano":"1735596552265546902","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33128"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"6usly+cZRsM=","parentSpanId":"aovapvAPfh8=","flags":768,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552268677418","endTimeUnixNano":"1735596552268805120","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33136"}},{"key":"rpc.grpc.status_code","value":{"intValue":"5"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"8fH7m7exgCs=","parentSpanId":"suYSEohRyP8=","flags":768,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552269640899","endTimeUnixNano":"1735596552269823792","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33136"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"+JjftiqLXcg=","parentSpanId":"TwqTw1iB4X0=","flags":768,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552270621410","endTimeUnixNano":"1735596552270745532","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33136"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"McVjdOxONSgdrrlkDUeUQw==","spanId":"lfpJen6rCfQ=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552272272740","endTimeUnixNano":"1735596552272444392","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33132"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"DgnnjNfi9WQ=","parentSpanId":"lWFmfug8YNY=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552272779406","endTimeUnixNano":"1735596552273151110","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33128"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"gNVkkaQRfwg=","parentSpanId":"BCaHfYcFoYI=","flags":768,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552274111891","endTimeUnixNano":"1735596552274255303","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33136"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"SeDH8Vfpadn10kMa8kJvgw==","spanId":"bakD1gohUnw=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552279426554","endTimeUnixNano":"1735596552279568575","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33132"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"fBpfW6Ymuv8=","parentSpanId":"5ThhIloR5KY=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552279993020","endTimeUnixNano":"1735596552280404654","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"LZAG0fuKcRwFnH5cEXT4Ug==","spanId":"sWoD9kavf+E=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552281751630","endTimeUnixNano":"1735596552281986783","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33104"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"c1fn98QaXwo=","parentSpanId":"jF06vgXl+rY=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552282882924","endTimeUnixNano":"1735596552283086246","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33112"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"ImsV4SaMVV8=","parentSpanId":"Z0Z1SUx32F8=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552283799684","endTimeUnixNano":"1735596552283989316","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33112"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"cXrgy5S3krM=","parentSpanId":"N7fb6O62Wao=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552284745205","endTimeUnixNano":"1735596552284901788","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33112"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"pbTnTWBDo/c=","parentSpanId":"lTFYsgKhSFo=","flags":768,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552285570725","endTimeUnixNano":"1735596552285821128","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33112"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"Cn7Zb+Z3DFs=","parentSpanId":"6LX3bJmVP78=","flags":768,"name":"envoy.service.auth.v3.Authorization/Check","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552282117174","endTimeUnixNano":"1735596552286492065","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.auth.v3.Authorization"}},{"key":"rpc.method","value":{"stringValue":"Check"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33100"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"L9z/Dey1wDyGsrGbe0xbKw==","spanId":"Qxa3VIfADAY=","flags":768,"name":"opentelemetry.proto.collector.trace.v1.TraceService/Export","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552287723980","endTimeUnixNano":"1735596552287857581","attributes":[{"key":"rpc.service","value":{"stringValue":"opentelemetry.proto.collector.trace.v1.TraceService"}},{"key":"rpc.method","value":{"stringValue":"Export"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33104"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"UPC6q/yfV2k=","parentSpanId":"YjiylRy7lCw=","flags":768,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552053704325","endTimeUnixNano":"1735596552315825317","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}},{"traceId":"eX1+Nwu+QvGR+nz/98XwWw==","spanId":"jYp+vYBUjRs=","flags":768,"name":"envoy.service.accesslog.v3.AccessLogService/StreamAccessLogs","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552364472913","endTimeUnixNano":"1735596552373285326","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.accesslog.v3.AccessLogService"}},{"key":"rpc.method","value":{"stringValue":"StreamAccessLogs"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}},{"traceId":"QHt9ukn/IblV67PlcrHm/w==","spanId":"iyuOnhdATnw=","flags":768,"name":"envoy.service.discovery.v3.AggregatedDiscoveryService/DeltaAggregatedResources","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596551308452351","endTimeUnixNano":"1735596552381932417","attributes":[{"key":"rpc.service","value":{"stringValue":"envoy.service.discovery.v3.AggregatedDiscoveryService"}},{"key":"rpc.method","value":{"stringValue":"DeltaAggregatedResources"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"33086"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp","version":"0.57.0"},"spans":[{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"xSJX1oetTdU=","parentSpanId":"FBPvHWyFn6I=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552258850984","endTimeUnixNano":"1735596552261747438","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"35065"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"59410"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"975"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"3jlv7DJBi34=","parentSpanId":"qZJUemZrh0U=","flags":768,"name":"Server: GET /oauth2/callback","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552265916216","endTimeUnixNano":"1735596552271673263","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"35065"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"59410"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/oauth2/callback"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"443"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"E6AUkZz0o+g=","parentSpanId":"uuvSf1QFkxA=","flags":768,"name":"Server: GET /.pomerium/sign_in","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552273553145","endTimeUnixNano":"1735596552276695981","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"authenticate.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"35065"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"59410"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/sign_in"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"1237"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"FUiwUhbfpO0=","parentSpanId":"LNr+ATKO4qk=","flags":768,"name":"Server: GET /","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552281027002","endTimeUnixNano":"1735596552281141204","attributes":[{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"net.host.name","value":{"stringValue":"foo.localhost.pomerium.io"}},{"key":"net.host.port","value":{"intValue":"35065"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"59418"}},{"key":"user_agent.original","value":{"stringValue":"Go-http-client/1.1"}},{"key":"http.client_ip","value":{"stringValue":"10.200.138.226"}},{"key":"http.target","value":{"stringValue":"/.pomerium/callback/"}},{"key":"net.protocol.version","value":{"stringValue":"1.1"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/handler.go:125"}},{"key":"http.response_content_length","value":{"intValue":"66"}},{"key":"http.status_code","value":{"intValue":"302"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, - {"timestamp":"2024-12-30T22:09:12.402575956Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/authenticate.go:54"}},{"key":"service.name","value":{"stringValue":"Authenticate"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp","version":"0.57.0"},"spans":[{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"+fYq7JMxmug=","parentSpanId":"gHqxUi5kePg=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552259041196","endTimeUnixNano":"1735596552260746875","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:43581/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"43581"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"uhrN3Ns9n/w=","parentSpanId":"L6h65KKHBOM=","flags":256,"name":"OAuth2 Client: GET /.well-known/openid-configuration","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552266052297","endTimeUnixNano":"1735596552266350501","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:43581/.well-known/openid-configuration"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"43581"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"421"}}],"status":{}},{"traceId":"dwMPQOqC63eLDuFdJ/DRFg==","spanId":"BHBu4awlz/Y=","flags":256,"name":"OAuth2 Client: GET /.well-known/jwks.json","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552266984739","endTimeUnixNano":"1735596552267287902","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.57.0/transport.go:107"}},{"key":"http.method","value":{"stringValue":"GET"}},{"key":"http.url","value":{"stringValue":"http://mock-idp.localhost.pomerium.io:43581/.well-known/jwks.json"}},{"key":"net.peer.name","value":{"stringValue":"mock-idp.localhost.pomerium.io"}},{"key":"net.peer.port","value":{"intValue":"43581"}},{"key":"http.status_code","value":{"intValue":"200"}},{"key":"http.response_content_length","value":{"intValue":"237"}}],"status":{}}]},{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"5JQnjvCb/mM=","parentSpanId":"gHqxUi5kePg=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552258982826","endTimeUnixNano":"1735596552261712807","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"L6h65KKHBOM=","parentSpanId":"WV8SOlkANhI=","flags":256,"name":"authenticate.getOAuthCallback","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552265984307","endTimeUnixNano":"1735596552271050666","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:327"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"QVdbJ8Izbi8=","parentSpanId":"uyDfS9VMw2E=","flags":256,"name":"authenticate.SignIn","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552274529026","endTimeUnixNano":"1735596552276674161","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:162"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"uyDfS9VMw2E=","parentSpanId":"yhyoDzh9RIM=","flags":256,"name":"authenticate.VerifySession","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552273718127","endTimeUnixNano":"1735596552276680921","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authenticate/handlers.go:116"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"aovapvAPfh8=","parentSpanId":"L6h65KKHBOM=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552267935169","endTimeUnixNano":"1735596552269118754","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"events":[{"timeUnixNano":"1735596552269116524","name":"status code: NotFound"}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"suYSEohRyP8=","parentSpanId":"L6h65KKHBOM=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552269284836","endTimeUnixNano":"1735596552270049974","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"TwqTw1iB4X0=","parentSpanId":"L6h65KKHBOM=","flags":256,"name":"databroker.DataBrokerService/Put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552270313067","endTimeUnixNano":"1735596552270952515","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Put"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"BCaHfYcFoYI=","parentSpanId":"uyDfS9VMw2E=","flags":256,"name":"databroker.DataBrokerService/Get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552273809458","endTimeUnixNano":"1735596552274482325","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Get"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, - {"timestamp":"2024-12-30T22:09:12.403360776Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/authorize/authorize.go:48"}},{"key":"service.name","value":{"stringValue":"Authorize"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"+R7poiK6/239h1nEy5hIrA==","spanId":"I+5+jfwGURc=","parentSpanId":"oPLxRoHsmJg=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552062084883","endTimeUnixNano":"1735596552062226725","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"+R7poiK6/239h1nEy5hIrA==","spanId":"oHv+S87aeOg=","parentSpanId":"Ypo2QQUZDgs=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552062572019","endTimeUnixNano":"1735596552062849073","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"+R7poiK6/239h1nEy5hIrA==","spanId":"Ypo2QQUZDgs=","parentSpanId":"oPLxRoHsmJg=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552062174334","endTimeUnixNano":"1735596552062874583","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"+R7poiK6/239h1nEy5hIrA==","spanId":"oPLxRoHsmJg=","parentSpanId":"702+hX/l24A=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552062022603","endTimeUnixNano":"1735596552062885453","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"+R7poiK6/239h1nEy5hIrA==","spanId":"702+hX/l24A=","parentSpanId":"wtBmI0TId5M=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552061969882","endTimeUnixNano":"1735596552062891643","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"BI0pXOVFSFDjDh4HlytjyA==","spanId":"PR3eGT+sQgk=","parentSpanId":"PXxYHizXWvY=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552065245471","endTimeUnixNano":"1735596552065372002","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"BI0pXOVFSFDjDh4HlytjyA==","spanId":"pKeK3Ol+An8=","parentSpanId":"ImM7s8/+0MA=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552065533264","endTimeUnixNano":"1735596552065780997","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"BI0pXOVFSFDjDh4HlytjyA==","spanId":"ImM7s8/+0MA=","parentSpanId":"PXxYHizXWvY=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552065273211","endTimeUnixNano":"1735596552065795577","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"BI0pXOVFSFDjDh4HlytjyA==","spanId":"PXxYHizXWvY=","parentSpanId":"rIEVxXFXkKg=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552065202010","endTimeUnixNano":"1735596552065806667","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"BI0pXOVFSFDjDh4HlytjyA==","spanId":"rIEVxXFXkKg=","parentSpanId":"CxZOOB7mGiQ=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552065158500","endTimeUnixNano":"1735596552065812947","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"nD9dzNOnTKOIJMxmlWZ9bw==","spanId":"QF0H6wiWXJM=","parentSpanId":"E9u3b8PmIoI=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552070634144","endTimeUnixNano":"1735596552070784215","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"nD9dzNOnTKOIJMxmlWZ9bw==","spanId":"kZ1uVCaZwk4=","parentSpanId":"PROD01WEVLg=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552070938967","endTimeUnixNano":"1735596552071153419","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"nD9dzNOnTKOIJMxmlWZ9bw==","spanId":"PROD01WEVLg=","parentSpanId":"E9u3b8PmIoI=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552070650504","endTimeUnixNano":"1735596552071170890","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"nD9dzNOnTKOIJMxmlWZ9bw==","spanId":"E9u3b8PmIoI=","parentSpanId":"mCghMmDIxcc=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552070580902","endTimeUnixNano":"1735596552071188160","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"nD9dzNOnTKOIJMxmlWZ9bw==","spanId":"mCghMmDIxcc=","parentSpanId":"lQvkuojiNVU=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552070547872","endTimeUnixNano":"1735596552071194050","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"b0XQMNrGpRts3otatJ4VkQ==","spanId":"QKLoYXMX29E=","parentSpanId":"xmjkj3o7dfk=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552078006669","endTimeUnixNano":"1735596552078093980","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"b0XQMNrGpRts3otatJ4VkQ==","spanId":"sryg5n/tnBQ=","parentSpanId":"4XomFRf7+kk=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552078190671","endTimeUnixNano":"1735596552078339492","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"b0XQMNrGpRts3otatJ4VkQ==","spanId":"4XomFRf7+kk=","parentSpanId":"xmjkj3o7dfk=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552078028279","endTimeUnixNano":"1735596552078349673","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"b0XQMNrGpRts3otatJ4VkQ==","spanId":"xmjkj3o7dfk=","parentSpanId":"/g6I0n43z0I=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552077980928","endTimeUnixNano":"1735596552078357263","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"b0XQMNrGpRts3otatJ4VkQ==","spanId":"/g6I0n43z0I=","parentSpanId":"PNkHn7SfYNs=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552077957258","endTimeUnixNano":"1735596552078362603","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"lCsEeGUuNYemOgQ6EAqw/Q==","spanId":"Zr/GT/Zu/ho=","parentSpanId":"tDxPozCLZ/s=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552087494559","endTimeUnixNano":"1735596552087633661","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"lCsEeGUuNYemOgQ6EAqw/Q==","spanId":"bDyRo8kOqOc=","parentSpanId":"Isdgl5r8KSs=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552087744612","endTimeUnixNano":"1735596552087903974","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"lCsEeGUuNYemOgQ6EAqw/Q==","spanId":"Isdgl5r8KSs=","parentSpanId":"tDxPozCLZ/s=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552087526160","endTimeUnixNano":"1735596552087915934","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"lCsEeGUuNYemOgQ6EAqw/Q==","spanId":"tDxPozCLZ/s=","parentSpanId":"whiBKTR+R7g=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552087458459","endTimeUnixNano":"1735596552087922914","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"lCsEeGUuNYemOgQ6EAqw/Q==","spanId":"whiBKTR+R7g=","parentSpanId":"Xrh5E+qCpVc=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552087426029","endTimeUnixNano":"1735596552087927814","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"69JHBTXAPq7zDqo19nNt3g==","spanId":"OHqb77VeJE4=","parentSpanId":"GlBfgb5ewww=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552098031612","endTimeUnixNano":"1735596552098165633","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"69JHBTXAPq7zDqo19nNt3g==","spanId":"eZVpM/28z1o=","parentSpanId":"UE5vNgfZ9Jk=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552098241214","endTimeUnixNano":"1735596552098396226","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"69JHBTXAPq7zDqo19nNt3g==","spanId":"UE5vNgfZ9Jk=","parentSpanId":"GlBfgb5ewww=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552098053802","endTimeUnixNano":"1735596552098407726","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"69JHBTXAPq7zDqo19nNt3g==","spanId":"GlBfgb5ewww=","parentSpanId":"kqNihdBkVcY=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552097995401","endTimeUnixNano":"1735596552098414976","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"69JHBTXAPq7zDqo19nNt3g==","spanId":"kqNihdBkVcY=","parentSpanId":"nqbiIHn9gV8=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552097959031","endTimeUnixNano":"1735596552098419756","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"qGCrQnZvKm9AiV3UG+9y4A==","spanId":"LFUxOjlfg/8=","parentSpanId":"JgeqIDcDVXc=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552124249976","endTimeUnixNano":"1735596552124354848","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"qGCrQnZvKm9AiV3UG+9y4A==","spanId":"VP53hCeHVAc=","parentSpanId":"pW/aWnlh3S8=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552124480579","endTimeUnixNano":"1735596552124709162","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"qGCrQnZvKm9AiV3UG+9y4A==","spanId":"pW/aWnlh3S8=","parentSpanId":"JgeqIDcDVXc=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552124259696","endTimeUnixNano":"1735596552124725643","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"qGCrQnZvKm9AiV3UG+9y4A==","spanId":"JgeqIDcDVXc=","parentSpanId":"3UY1EGRVKNE=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552124215096","endTimeUnixNano":"1735596552124734763","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"qGCrQnZvKm9AiV3UG+9y4A==","spanId":"3UY1EGRVKNE=","parentSpanId":"7A4ZbdDiN2o=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552124173605","endTimeUnixNano":"1735596552124740303","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"oboPZ53IIkU+AOAisyhydQ==","spanId":"HtUTSubmmrE=","parentSpanId":"ovY10LzBQFY=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552185324238","endTimeUnixNano":"1735596552185421339","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"oboPZ53IIkU+AOAisyhydQ==","spanId":"xyxreo92b3g=","parentSpanId":"QF+l8Wz96YQ=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552185523760","endTimeUnixNano":"1735596552185722773","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"oboPZ53IIkU+AOAisyhydQ==","spanId":"QF+l8Wz96YQ=","parentSpanId":"ovY10LzBQFY=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552185350058","endTimeUnixNano":"1735596552185733963","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"oboPZ53IIkU+AOAisyhydQ==","spanId":"ovY10LzBQFY=","parentSpanId":"Fnf7AryD+WI=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552185290468","endTimeUnixNano":"1735596552185741623","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"oboPZ53IIkU+AOAisyhydQ==","spanId":"Fnf7AryD+WI=","parentSpanId":"13OK1T2dwk8=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552185261027","endTimeUnixNano":"1735596552185746723","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"NUZgjI72agI=","parentSpanId":"3V6E9VZ9lFw=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552249348673","endTimeUnixNano":"1735596552249456885","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"cR6yd0zNkYs=","parentSpanId":"9V8ar0k7EFA=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552249548196","endTimeUnixNano":"1735596552251056403","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"GTgvZBQ068Q=","parentSpanId":"9V8ar0k7EFA=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552251076964","endTimeUnixNano":"1735596552251972654","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"wDgeqdCQypw=","parentSpanId":"9V8ar0k7EFA=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552251988344","endTimeUnixNano":"1735596552252010944","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"\""}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"9V8ar0k7EFA=","parentSpanId":"3V6E9VZ9lFw=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552249386184","endTimeUnixNano":"1735596552252399628","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"3V6E9VZ9lFw=","parentSpanId":"HcyMtZcnRq4=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552249305223","endTimeUnixNano":"1735596552252409978","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"3DSL89VAypc=","parentSpanId":"HcyMtZcnRq4=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552252639201","endTimeUnixNano":"1735596552252687922","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:28"}},{"key":"result.allow","value":{"boolValue":false}},{"key":"result.allow-why-false","value":{"arrayValue":{"values":[{"stringValue":"user-unauthenticated"}]}}},{"key":"result.deny-why-false","value":{"arrayValue":{}}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"HcyMtZcnRq4=","parentSpanId":"XF9zGG11kRE=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552249274383","endTimeUnixNano":"1735596552252697352","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"g2F4uQwPi5I=","parentSpanId":"kVygGFO0Xw0=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552257879962","endTimeUnixNano":"1735596552257960503","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"kVygGFO0Xw0=","parentSpanId":"I0JWe90UcQ8=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552257851872","endTimeUnixNano":"1735596552257973664","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"rlSFOyl7ozs=","parentSpanId":"I0JWe90UcQ8=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552257985584","endTimeUnixNano":"1735596552258019594","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:28"}},{"key":"result.allow","value":{"boolValue":true}},{"key":"result.allow-why-true","value":{"arrayValue":{"values":[{"stringValue":"pomerium-route"}]}}},{"key":"result.deny-why-false","value":{"arrayValue":{}}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"I0JWe90UcQ8=","parentSpanId":"BHkNMsuL5iw=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552257797392","endTimeUnixNano":"1735596552258026874","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"usCxFBdA74o=","parentSpanId":"Oouk7UHsCu4=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552265322909","endTimeUnixNano":"1735596552265450551","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"Oouk7UHsCu4=","parentSpanId":"IAGqRFX/cVs=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552265289779","endTimeUnixNano":"1735596552265464871","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"24K8qH0UQNo=","parentSpanId":"IAGqRFX/cVs=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552265470721","endTimeUnixNano":"1735596552265504361","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:28"}},{"key":"result.allow","value":{"boolValue":true}},{"key":"result.allow-why-true","value":{"arrayValue":{"values":[{"stringValue":"pomerium-route"}]}}},{"key":"result.deny-why-false","value":{"arrayValue":{}}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"IAGqRFX/cVs=","parentSpanId":"mDALe+Jrgfg=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552265228868","endTimeUnixNano":"1735596552265511791","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"TIdzDOPaZa4=","parentSpanId":"JQr00bdtY6g=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552272943178","endTimeUnixNano":"1735596552273045829","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"JQr00bdtY6g=","parentSpanId":"eG//PYnRynA=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552272917778","endTimeUnixNano":"1735596552273059780","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"25vcPv/qqO8=","parentSpanId":"eG//PYnRynA=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552273064050","endTimeUnixNano":"1735596552273110010","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:28"}},{"key":"result.allow","value":{"boolValue":true}},{"key":"result.allow-why-true","value":{"arrayValue":{"values":[{"stringValue":"pomerium-route"}]}}},{"key":"result.deny-why-false","value":{"arrayValue":{}}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"eG//PYnRynA=","parentSpanId":"DgnnjNfi9WQ=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552272856047","endTimeUnixNano":"1735596552273116970","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"2ufRRaRmOpg=","parentSpanId":"yedpQsjuMek=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552280207462","endTimeUnixNano":"1735596552280308813","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"yedpQsjuMek=","parentSpanId":"IDC3HSZb0BE=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552280170692","endTimeUnixNano":"1735596552280322033","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"ze+U8DOBf9o=","parentSpanId":"IDC3HSZb0BE=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552280327954","endTimeUnixNano":"1735596552280359674","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:28"}},{"key":"result.allow","value":{"boolValue":true}},{"key":"result.allow-why-true","value":{"arrayValue":{"values":[{"stringValue":"pomerium-route"}]}}},{"key":"result.deny-why-false","value":{"arrayValue":{}}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"IDC3HSZb0BE=","parentSpanId":"fBpfW6Ymuv8=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552280065711","endTimeUnixNano":"1735596552280366684","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"xgQ5Ms1BPbY=","parentSpanId":"J4x6xb3qh2c=","flags":256,"name":"authorize.getDataBrokerSessionOrServiceAccount","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552282372287","endTimeUnixNano":"1735596552283419570","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/databroker.go:64"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"HbnRXI9BK14=","parentSpanId":"J4x6xb3qh2c=","flags":256,"name":"authorize.getDataBrokerUser","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552283422410","endTimeUnixNano":"1735596552284305360","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/databroker.go:97"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"ZhLEljBrikk=","parentSpanId":"dqbGwdVFBdI=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552284669214","endTimeUnixNano":"1735596552285248751","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.ServiceAccount\""}},{"key":"record_id","value":{"stringValue":"\"737ae166-e045-403d-bceb-7fdc794b69e7\""}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"2KN++KWW5po=","parentSpanId":"dqbGwdVFBdI=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552285276202","endTimeUnixNano":"1735596552285531025","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"737ae166-e045-403d-bceb-7fdc794b69e7\""}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"NLQFh6R6vYM=","parentSpanId":"dqbGwdVFBdI=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552285631146","endTimeUnixNano":"1735596552285755157","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/session.Session\""}},{"key":"record_id","value":{"stringValue":"\"737ae166-e045-403d-bceb-7fdc794b69e7\""}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"YzW0uskC6VA=","parentSpanId":"dqbGwdVFBdI=","flags":256,"name":"rego.get_databroker_record","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552285800058","endTimeUnixNano":"1735596552285901849","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"record_type","value":{"stringValue":"\"type.googleapis.com/user.User\""}},{"key":"record_id","value":{"stringValue":"\"a4f74022-cf7e-4ecf-a7f8-d655185fe980\""}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"6G9M65neegc=","parentSpanId":"xW0iCPJJO4s=","flags":256,"name":"authorize.HeadersEvaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552284351620","endTimeUnixNano":"1735596552286220182","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"dqbGwdVFBdI=","parentSpanId":"xW0iCPJJO4s=","flags":256,"name":"authorize.PolicyEvaluator.evaluateQuery","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552284373821","endTimeUnixNano":"1735596552286388584","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}},{"key":"script_checksum","value":{"stringValue":"fc039e688537fbdf7e2becc31e246743e1a78d7c1c73a331f08daa1b534eeb2a"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"xW0iCPJJO4s=","parentSpanId":"J4x6xb3qh2c=","flags":256,"name":"authorize.Evaluator.Evaluate","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552284317990","endTimeUnixNano":"1735596552286402824","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/telemetry/trace/trace.go:112"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"3HTg2TiSq+E=","parentSpanId":"J4x6xb3qh2c=","flags":256,"name":"authorize.grpc.LogAuthorizeCheck","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552286408794","endTimeUnixNano":"1735596552286448825","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/log.go:28"}},{"key":"result.allow","value":{"boolValue":true}},{"key":"result.allow-why-true","value":{"arrayValue":{"values":[{"stringValue":"email-ok"}]}}},{"key":"result.deny-why-false","value":{"arrayValue":{}}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"J4x6xb3qh2c=","parentSpanId":"Cn7Zb+Z3DFs=","flags":256,"name":"authorize.grpc.Check","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552282210506","endTimeUnixNano":"1735596552286456045","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/authorize/grpc.go:31"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"+R7poiK6/239h1nEy5hIrA==","spanId":"ZitEHOlhYmg=","parentSpanId":"oHv+S87aeOg=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552062721421","endTimeUnixNano":"1735596552062749752","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"BI0pXOVFSFDjDh4HlytjyA==","spanId":"H2Bi92mnl38=","parentSpanId":"pKeK3Ol+An8=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552065647705","endTimeUnixNano":"1735596552065677235","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"nD9dzNOnTKOIJMxmlWZ9bw==","spanId":"Kt4NyLWXBNc=","parentSpanId":"kZ1uVCaZwk4=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552071026898","endTimeUnixNano":"1735596552071051688","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"b0XQMNrGpRts3otatJ4VkQ==","spanId":"U1A1AlTpBik=","parentSpanId":"sryg5n/tnBQ=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552078251041","endTimeUnixNano":"1735596552078268392","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"lCsEeGUuNYemOgQ6EAqw/Q==","spanId":"8TN7lnb8ZZw=","parentSpanId":"bDyRo8kOqOc=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552087812363","endTimeUnixNano":"1735596552087829773","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"69JHBTXAPq7zDqo19nNt3g==","spanId":"MdCipzjfzDA=","parentSpanId":"eZVpM/28z1o=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552098306135","endTimeUnixNano":"1735596552098323105","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"qGCrQnZvKm9AiV3UG+9y4A==","spanId":"bSnSgvrRc9U=","parentSpanId":"VP53hCeHVAc=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552124561060","endTimeUnixNano":"1735596552124582680","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"oboPZ53IIkU+AOAisyhydQ==","spanId":"K0uogfkk0I8=","parentSpanId":"xyxreo92b3g=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552185615441","endTimeUnixNano":"1735596552185636552","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:40165: connect: connection refused\"","code":"STATUS_CODE_ERROR"}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"9BuTh+PJtJ8=","parentSpanId":"cR6yd0zNkYs=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552249646597","endTimeUnixNano":"1735596552251004663","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"yOL5QyxGuD8=","parentSpanId":"GTgvZBQ068Q=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552251131154","endTimeUnixNano":"1735596552251944683","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"jF06vgXl+rY=","parentSpanId":"xgQ5Ms1BPbY=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552282435188","endTimeUnixNano":"1735596552283362919","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"Z0Z1SUx32F8=","parentSpanId":"HbnRXI9BK14=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552283482131","endTimeUnixNano":"1735596552284245699","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"N7fb6O62Wao=","parentSpanId":"6G9M65neegc=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552284432481","endTimeUnixNano":"1735596552285131900","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"lTFYsgKhSFo=","parentSpanId":"6G9M65neegc=","flags":256,"name":"databroker.DataBrokerService/Query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552285250591","endTimeUnixNano":"1735596552286064931","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Query"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"40165"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}}, - {"timestamp":"2024-12-30T22:09:12.404074234Z","request":{"resourceSpans":[{"resource":{"attributes":[{"key":"provider.created_at","value":{"stringValue":"/home/ubuntu/pomerium/databroker/cache.go:94"}},{"key":"service.name","value":{"stringValue":"Data Broker"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.33.0"}}]},"scopeSpans":[{"scope":{"name":"pomerium.io/core"},"spans":[{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"3Jv9+gNiTBE=","parentSpanId":"WBoo1gQHvyI=","flags":256,"name":"databroker.grpc.AcquireLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551247776624","endTimeUnixNano":"1735596551248251260","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:83"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"FnISXIhjzvI=","parentSpanId":"oDKmK3j0BGo=","flags":256,"name":"databroker.grpc.SyncLatest","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551249094219","endTimeUnixNano":"1735596551249277271","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:393"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"uDaOHkb40p4=","parentSpanId":"MpsYB/n8HZU=","flags":256,"name":"databroker.grpc.SyncLatest","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551249161070","endTimeUnixNano":"1735596551249326972","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:393"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"9co6PIOIWWk=","parentSpanId":"oAf8R1Tzhpo=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551749629896","endTimeUnixNano":"1735596551749642886","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"I2+UHD0llFE=","parentSpanId":"RXv+G4LTsEk=","flags":256,"name":"databroker.grpc.SyncLatest","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552052732244","endTimeUnixNano":"1735596552052829356","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:393"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"mexE3YgRss0=","parentSpanId":"uxgVY6jCPOw=","flags":256,"name":"databroker.grpc.RenewLease","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552249206962","endTimeUnixNano":"1735596552249222362","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:307"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"RpB80gUAJnc=","parentSpanId":"anG2KFI46uE=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552250516206","endTimeUnixNano":"1735596552250616108","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"lFfHkwRj0N4=","parentSpanId":"2ACwBEIpGZ8=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552251564179","endTimeUnixNano":"1735596552251685920","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"anVXyj3g0y0=","parentSpanId":"6usly+cZRsM=","flags":256,"name":"databroker.grpc.Get","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552268754989","endTimeUnixNano":"1735596552268775190","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:110"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"XG+fN6jKD2M=","parentSpanId":"8fH7m7exgCs=","flags":256,"name":"databroker.grpc.Put","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552269719780","endTimeUnixNano":"1735596552269761311","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:207"}}],"status":{}},{"traceId":"eVdO9AzPF7M6v8YjNISJIQ==","spanId":"I7vcpKXSaL0=","parentSpanId":"+JjftiqLXcg=","flags":256,"name":"databroker.grpc.Put","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552270683971","endTimeUnixNano":"1735596552270718502","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:207"}}],"status":{}},{"traceId":"De/5UmhZE/5N86yDpe+7gw==","spanId":"WxNK3s1U+kc=","parentSpanId":"gNVkkaQRfwg=","flags":256,"name":"databroker.grpc.Get","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552274169042","endTimeUnixNano":"1735596552274201452","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:110"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"eRRiR8mb/Zk=","parentSpanId":"c1fn98QaXwo=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552282971495","endTimeUnixNano":"1735596552283056846","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"EyiHMzQqpYI=","parentSpanId":"ImsV4SaMVV8=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552283885275","endTimeUnixNano":"1735596552283961596","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"yLyVXa2MkYM=","parentSpanId":"cXrgy5S3krM=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552284821237","endTimeUnixNano":"1735596552284876897","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"5RXk9sVkWH+di1bftPaEIQ==","spanId":"E9KYOI4Enow=","parentSpanId":"pbTnTWBDo/c=","flags":256,"name":"databroker.grpc.Query","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552285707277","endTimeUnixNano":"1735596552285789157","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:154"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"6ZPe6wSwVqs=","parentSpanId":"UPC6q/yfV2k=","flags":256,"name":"databroker.grpc.Sync","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596552053809077","endTimeUnixNano":"1735596552315804667","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:355"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"KIpwWMH3RYw=","parentSpanId":"INZmZYN0zYM=","flags":256,"name":"databroker.grpc.Sync","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551249794687","endTimeUnixNano":"1735596552399213398","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:355"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"5Xc5hfznIok=","parentSpanId":"RK6oHqmnMlg=","flags":256,"name":"databroker.grpc.Sync","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":"1735596551249850358","endTimeUnixNano":"1735596552399227388","attributes":[{"key":"caller","value":{"stringValue":"/home/ubuntu/pomerium/internal/databroker/server.go:355"}}],"status":{}}]},{"scope":{"name":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc","version":"0.57.0"},"spans":[{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"WBoo1gQHvyI=","parentSpanId":"lpAHqgla4lk=","flags":768,"name":"databroker.DataBrokerService/AcquireLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596551247120936","endTimeUnixNano":"1735596551248383661","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"AcquireLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"50506"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"lpAHqgla4lk=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/AcquireLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596551244933821","endTimeUnixNano":"1735596551248572994","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"AcquireLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"35089"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"oDKmK3j0BGo=","parentSpanId":"3NhMqufhs3E=","flags":768,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596551249023349","endTimeUnixNano":"1735596551249292572","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"50506"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"MpsYB/n8HZU=","parentSpanId":"JfRLTLgutnU=","flags":768,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596551249053529","endTimeUnixNano":"1735596551249333762","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"50506"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"3NhMqufhs3E=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596551248774936","endTimeUnixNano":"1735596551249483274","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"35089"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"JfRLTLgutnU=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/SyncLatest","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596551248781656","endTimeUnixNano":"1735596551249500274","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"SyncLatest"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"35089"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"oAf8R1Tzhpo=","parentSpanId":"zZeFpq0lf1U=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596551749484304","endTimeUnixNano":"1735596551749700567","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"50506"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"zZeFpq0lf1U=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596551749149160","endTimeUnixNano":"1735596551749836088","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"35089"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"uxgVY6jCPOw=","parentSpanId":"Nhvnj8qzYVg=","flags":768,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596552249128171","endTimeUnixNano":"1735596552249248462","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"50506"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"Nhvnj8qzYVg=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/RenewLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552248935569","endTimeUnixNano":"1735596552249348583","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"RenewLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"35089"}},{"key":"rpc.grpc.status_code","value":{"intValue":"0"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"LAE5i/Gu+f8=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596551249562874","endTimeUnixNano":"1735596552399071366","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"35089"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{"message":"context canceled","code":"STATUS_CODE_ERROR"}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"erfJ2NXx8k8=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596551249581905","endTimeUnixNano":"1735596552399145197","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"35089"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{"message":"context canceled","code":"STATUS_CODE_ERROR"}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"INZmZYN0zYM=","parentSpanId":"LAE5i/Gu+f8=","flags":768,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596551249734266","endTimeUnixNano":"1735596552399244838","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"50506"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"RK6oHqmnMlg=","parentSpanId":"erfJ2NXx8k8=","flags":768,"name":"databroker.DataBrokerService/Sync","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1735596551249787427","endTimeUnixNano":"1735596552399254019","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"Sync"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:62"}},{"key":"net.sock.peer.addr","value":{"stringValue":"127.0.0.1"}},{"key":"net.sock.peer.port","value":{"intValue":"50506"}},{"key":"rpc.grpc.status_code","value":{"intValue":"1"}}],"status":{}},{"traceId":"IJZtfbsT1kLJj30547VSMg==","spanId":"uzNAHYDfDE4=","parentSpanId":"Qkk6itX7wm8=","flags":256,"name":"databroker.DataBrokerService/ReleaseLease","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1735596552399284209","endTimeUnixNano":"1735596552399551322","attributes":[{"key":"rpc.service","value":{"stringValue":"databroker.DataBrokerService"}},{"key":"rpc.method","value":{"stringValue":"ReleaseLease"}},{"key":"rpc.system","value":{"stringValue":"grpc"}},{"key":"caller","value":{"stringValue":"/home/ubuntu/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.57.0/stats_handler.go:102"}},{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}],"status":{"message":"connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:35089: connect: connection refused\"","code":"STATUS_CODE_ERROR"}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.26.0"}]}} -] diff --git a/internal/telemetry/trace/trace.go b/internal/telemetry/trace/trace.go index d3c973c13..0f03ec07e 100644 --- a/internal/telemetry/trace/trace.go +++ b/internal/telemetry/trace/trace.go @@ -7,7 +7,6 @@ import ( "runtime" "sync" "sync/atomic" - "time" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/exporters/otlp/otlptrace" @@ -35,8 +34,11 @@ func (op Options) NewContext(parent context.Context) context.Context { options: op, tpm: &tracerProviderManager{}, } + if op.DebugFlags.Check(TrackSpanReferences) { + sys.observer = newSpanObserver() + } ctx := context.WithValue(parent, systemContextKey, sys) - sys.exporterServer = NewServer(ctx, op.RemoteClient) + sys.exporterServer = NewServer(ctx) sys.exporterServer.Start(ctx) return ctx } @@ -86,13 +88,14 @@ func NewTracerProvider(ctx context.Context, serviceName string, opts ...sdktrace if sys.options.DebugFlags.Check(TrackSpanCallers) { options = append(options, sdktrace.WithSpanProcessor(&stackTraceProcessor{})) } + if sys.options.DebugFlags.Check(TrackSpanReferences) { + tracker := newSpanTracker(sys.observer, sys.options.DebugFlags) + options = append(options, sdktrace.WithSpanProcessor(tracker)) + } options = append(append(options, sdktrace.WithBatcher(exp), sdktrace.WithResource(r), ), opts...) - for _, proc := range sys.exporterServer.SpanProcessors() { - options = append(options, sdktrace.WithSpanProcessor(proc)) - } tp := sdktrace.NewTracerProvider(options...) sys.tpm.Add(tp) return tp @@ -156,34 +159,6 @@ func RemoteClientFromContext(ctx context.Context) otlptrace.Client { return nil } -func DebugFlagsFromContext(ctx context.Context) DebugFlags { - if sys := systemContextFromContext(ctx); sys != nil { - return sys.options.DebugFlags - } - return 0 -} - -// WaitForSpans will block up to the given max duration and wait for all -// in-flight spans from tracers created with the given context to end. This -// function can be called more than once, and is safe to call from multiple -// goroutines in parallel. -// -// This requires the [TrackSpanReferences] debug flag to have been set with -// [Options.NewContext]. Otherwise, this function is a no-op and will return -// immediately. -// -// If this function blocks for more than 10 seconds, it will print a warning -// to stderr containing a list of span IDs it is waiting for, and the IDs of -// their parents (if known). Additionally, if the [TrackAllSpans] debug flag -// is set, details about parent spans will be displayed, including call site -// and trace ID. -func WaitForSpans(ctx context.Context, maxDuration time.Duration) error { - if sys := systemContextFromContext(ctx); sys != nil { - return sys.exporterServer.spanExportQueue.WaitForSpans(maxDuration) - } - return nil -} - // ForceFlush immediately exports all spans that have not yet been exported for // all tracer providers created using the given context. func ForceFlush(ctx context.Context) error { @@ -204,6 +179,7 @@ var systemContextKey systemContextKeyType type systemContext struct { options Options tpm *tracerProviderManager + observer *spanObserver exporterServer *ExporterServer shutdown atomic.Bool } diff --git a/internal/telemetry/trace/trace_export_test.go b/internal/telemetry/trace/trace_export_test.go index 35360bce0..62f6cdcf0 100644 --- a/internal/telemetry/trace/trace_export_test.go +++ b/internal/telemetry/trace/trace_export_test.go @@ -18,7 +18,7 @@ var ( type XStackTraceProcessor = stackTraceProcessor func (obs *spanObserver) XWait() { - obs.wait(nil, 5*time.Second) + obs.wait(5 * time.Second) } func (obs *spanObserver) XUnobservedIDs() []oteltrace.SpanID { diff --git a/internal/telemetry/trace/util.go b/internal/telemetry/trace/util.go index 9f62beb85..ec1b28437 100644 --- a/internal/telemetry/trace/util.go +++ b/internal/telemetry/trace/util.go @@ -1,85 +1,11 @@ package trace import ( - "encoding/hex" - "fmt" - "net/url" - "strings" "unique" - "go.opentelemetry.io/otel/attribute" oteltrace "go.opentelemetry.io/otel/trace" - commonv1 "go.opentelemetry.io/proto/otlp/common/v1" - tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" ) -func ParseTraceparent(traceparent string) (oteltrace.SpanContext, error) { - parts := strings.Split(traceparent, "-") - if len(parts) != 4 { - return oteltrace.SpanContext{}, fmt.Errorf("malformed traceparent: expected 4 segments, found %d", len(parts)) - } - traceID, err := oteltrace.TraceIDFromHex(parts[1]) - if err != nil { - return oteltrace.SpanContext{}, fmt.Errorf("malformed traceparent: invalid trace ID: %w", err) - } - spanID, err := oteltrace.SpanIDFromHex(parts[2]) - if err != nil { - return oteltrace.SpanContext{}, fmt.Errorf("malformed traceparent: invalid span ID: %w", err) - } - var traceFlags oteltrace.TraceFlags - if flags, err := hex.DecodeString(parts[3]); err != nil { - return oteltrace.SpanContext{}, fmt.Errorf("malformed traceparent: invalid trace flags: %w", err) - } else if len(flags) == 1 { - traceFlags = oteltrace.TraceFlags(flags[0]) - } else { - return oteltrace.SpanContext{}, fmt.Errorf("malformed traceparent: invalid trace flags of size %d", len(flags)) - } - if len(traceID) != 16 { - return oteltrace.SpanContext{}, fmt.Errorf("malformed traceparent: invalid trace ID of size %d", len(traceID)) - } - if len(spanID) != 8 { - return oteltrace.SpanContext{}, fmt.Errorf("malformed traceparent: invalid span ID of size %d", len(spanID)) - } - return oteltrace.NewSpanContext(oteltrace.SpanContextConfig{ - TraceID: traceID, - SpanID: spanID, - TraceFlags: traceFlags, - }), nil -} - -// WithTraceFromSpanContext returns a copy of traceparent with the trace ID -// (2nd segment) and trace flags (4th segment) replaced with the corresponding -// values from spanContext. -func WithTraceFromSpanContext(traceparent string, spanContext oteltrace.SpanContext) string { - parts := strings.Split(traceparent, "-") - if len(parts) != 4 { - return traceparent - } - parts[1] = spanContext.TraceID().String() - parts[3] = spanContext.TraceFlags().String() - return strings.Join(parts, "-") -} - -func FormatSpanName(span *tracev1.Span) { - hasVariables := strings.Contains(span.GetName(), "${") - if hasVariables { - replacements := make([]string, 0, 6) - for _, attr := range span.Attributes { - switch attr.Key { - case "http.url": - u, _ := url.Parse(attr.Value.GetStringValue()) - replacements = append(replacements, - "${path}", u.Path, - "${host}", u.Host, - ) - case "http.method": - replacements = append(replacements, "${method}", attr.Value.GetStringValue()) - } - } - span.Name = strings.NewReplacer(replacements...).Replace(span.Name) - } -} - var ( zeroSpanID oteltrace.SpanID zeroTraceID = unique.Make(oteltrace.TraceID([16]byte{})) @@ -104,33 +30,3 @@ func ToTraceID(bytes []byte) (unique.Handle[oteltrace.TraceID], bool) { } return zeroTraceID, false } - -func NewAttributeSet(kvs ...*commonv1.KeyValue) attribute.Set { - attrs := make([]attribute.KeyValue, len(kvs)) - for i, kv := range kvs { - var value attribute.Value - switch v := kv.Value.Value.(type) { - case *commonv1.AnyValue_BoolValue: - value = attribute.BoolValue(v.BoolValue) - case *commonv1.AnyValue_BytesValue: - value = attribute.StringValue(string(v.BytesValue)) - case *commonv1.AnyValue_DoubleValue: - value = attribute.Float64Value(v.DoubleValue) - case *commonv1.AnyValue_IntValue: - value = attribute.Int64Value(v.IntValue) - case *commonv1.AnyValue_StringValue: - value = attribute.StringValue(v.StringValue) - case *commonv1.AnyValue_ArrayValue: - panic("unimplemented") - case *commonv1.AnyValue_KvlistValue: - panic("unimplemented") - default: - panic(fmt.Sprintf("unexpected v1.isAnyValue_Value: %#v", v)) - } - attrs[i] = attribute.KeyValue{ - Key: attribute.Key(kv.Key), - Value: value, - } - } - return attribute.NewSet(attrs...) -} diff --git a/internal/testenv/environment.go b/internal/testenv/environment.go index 00744b9e8..e4ef2c4ab 100644 --- a/internal/testenv/environment.go +++ b/internal/testenv/environment.go @@ -286,7 +286,7 @@ const StandardTraceDebugFlags = trace.TrackSpanCallers | trace.WarnOnIncompleteSpans | trace.WarnOnIncompleteTraces | trace.WarnOnUnresolvedReferences | - trace.LogTraceIDMappingsOnWarn | + trace.LogTraceIDsOnWarn | trace.LogAllSpansOnWarn func WithTraceDebugFlags(flags trace.DebugFlags) EnvironmentOption { diff --git a/internal/testenv/scenarios/trace_receiver.go b/internal/testenv/scenarios/trace_receiver.go index dfa7671eb..cebdf8962 100644 --- a/internal/testenv/scenarios/trace_receiver.go +++ b/internal/testenv/scenarios/trace_receiver.go @@ -13,7 +13,7 @@ import ( "github.com/pomerium/pomerium/internal/testenv" "github.com/pomerium/pomerium/internal/testenv/upstreams" "github.com/pomerium/pomerium/internal/testenv/values" - "github.com/pomerium/pomerium/internal/testutil" + "github.com/pomerium/pomerium/internal/testutil/tracetest" "go.opentelemetry.io/otel/exporters/otlp/otlptrace" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" @@ -136,7 +136,7 @@ func (rec *OTLPTraceReceiver) PeekResourceSpans() []*tracev1.ResourceSpans { } func (rec *OTLPTraceReceiver) peekResourceSpansLocked() []*tracev1.ResourceSpans { - return testutil.FlattenExportRequests(rec.receivedRequests) + return tracetest.FlattenExportRequests(rec.receivedRequests) } func (rec *OTLPTraceReceiver) FlushResourceSpans() []*tracev1.ResourceSpans { diff --git a/internal/testenv/selftests/tracing_test.go b/internal/testenv/selftests/tracing_test.go index 324ea5700..28e9e293f 100644 --- a/internal/testenv/selftests/tracing_test.go +++ b/internal/testenv/selftests/tracing_test.go @@ -18,7 +18,7 @@ import ( "github.com/pomerium/pomerium/internal/testenv/scenarios" "github.com/pomerium/pomerium/internal/testenv/snippets" "github.com/pomerium/pomerium/internal/testenv/upstreams" - "github.com/pomerium/pomerium/internal/testutil" + . "github.com/pomerium/pomerium/internal/testutil/tracetest" //nolint:revive "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -29,7 +29,7 @@ import ( oteltrace "go.opentelemetry.io/otel/trace" ) -func otlpTraceReceiverOrFromEnv(t *testing.T) (modifier testenv.Modifier, newRemoteClient func() otlptrace.Client, getResults func() *testutil.TraceResults) { +func otlpTraceReceiverOrFromEnv(t *testing.T) (modifier testenv.Modifier, newRemoteClient func() otlptrace.Client, getResults func() *TraceResults) { t.Setenv("OTEL_TRACES_EXPORTER", "otlp") tracesEndpoint := os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") if tracesEndpoint == "" { @@ -40,8 +40,8 @@ func otlpTraceReceiverOrFromEnv(t *testing.T) (modifier testenv.Modifier, newRem func() otlptrace.Client { return srv.NewGRPCClient() }, - func() *testutil.TraceResults { - return testutil.NewTraceResults(srv.FlushResourceSpans()) + func() *TraceResults { + return NewTraceResults(srv.FlushResourceSpans()) } } } @@ -111,18 +111,18 @@ func TestOTLPTracing(t *testing.T) { ) results.MatchTraces(t, - testutil.MatchOptions{ + MatchOptions{ Exact: true, CheckDetachedSpans: true, }, - testutil.Match{Name: testEnvironmentLocalTest, TraceCount: 1, Services: []string{"Test Environment", "Control Plane", "Data Broker"}}, - testutil.Match{Name: testEnvironmentAuthenticate, TraceCount: 1, Services: allServices}, - testutil.Match{Name: authenticateOAuth2Client, TraceCount: testutil.Greater(0)}, - testutil.Match{Name: idpServerGetUserinfo, TraceCount: testutil.EqualToMatch(authenticateOAuth2Client)}, - testutil.Match{Name: idpServerPostToken, TraceCount: testutil.EqualToMatch(authenticateOAuth2Client)}, - testutil.Match{Name: controlPlaneEnvoyDiscovery, TraceCount: 1}, - testutil.Match{Name: controlPlaneExport, TraceCount: testutil.Greater(0)}, - testutil.Match{Name: controlPlaneEnvoyAccessLogs, TraceCount: testutil.Any{}}, + Match{Name: testEnvironmentLocalTest, TraceCount: 1, Services: []string{"Test Environment", "Control Plane", "Data Broker"}}, + Match{Name: testEnvironmentAuthenticate, TraceCount: 1, Services: allServices}, + Match{Name: authenticateOAuth2Client, TraceCount: Greater(0)}, + Match{Name: idpServerGetUserinfo, TraceCount: EqualToMatch(authenticateOAuth2Client)}, + Match{Name: idpServerPostToken, TraceCount: EqualToMatch(authenticateOAuth2Client)}, + Match{Name: controlPlaneEnvoyDiscovery, TraceCount: 1}, + Match{Name: controlPlaneExport, TraceCount: Greater(0)}, + Match{Name: controlPlaneEnvoyAccessLogs, TraceCount: Any{}}, ) } } @@ -164,17 +164,15 @@ func TestOTLPTracing_TraceCorrelation(t *testing.T) { if getResults != nil { results := getResults() traces := results.GetTraces() - downstreamTraces := traces.ByName[fmt.Sprintf("Envoy: ingress: GET foo.localhost.pomerium.io:%d/foo", env.Ports().ProxyHTTP.Value())].WithoutErrors() - upstreamTraces := traces.ByName[fmt.Sprintf("Envoy: ingress: GET 127.0.0.1:%d/foo", up.Port().Value())].WithoutErrors() - assert.Len(t, upstreamTraces, 1) - assert.Len(t, downstreamTraces, 1) + // one unauthenticated (ends in /.pomerium/callback redirect), one authenticated + assert.Len(t, traces.ByName[fmt.Sprintf("Envoy: ingress: GET foo.localhost.pomerium.io:%d/foo", env.Ports().ProxyHTTP.Value())].WithoutErrors(), 2) } } type SamplingTestSuite struct { suite.Suite env testenv.Environment - getResults func() *testutil.TraceResults + getResults func() *TraceResults route testenv.Route upstream upstreams.HTTPUpstream @@ -308,6 +306,7 @@ func (s *SamplingTestSuite) TestExternalTraceparentNeverSample() { if (len(traces.ByParticipant)) != 0 { // whether or not these show up is timing dependent, but not important possibleTraces := map[string]struct{}{ + "Test Environment: Start": {}, "IDP: Server: POST /oidc/token": {}, "IDP: Server: GET /oidc/userinfo": {}, "Authenticate: OAuth2 Client: GET /.well-known/jwks.json": {}, @@ -315,7 +314,7 @@ func (s *SamplingTestSuite) TestExternalTraceparentNeverSample() { actual := slices.Collect(maps.Keys(traces.ByName)) for _, name := range actual { if _, ok := possibleTraces[name]; !ok { - s.Failf("unexpected trace: %s", name) + s.Fail("unexpected trace: " + name) } } } @@ -384,8 +383,8 @@ func TestExternalSpans(t *testing.T) { if getResults != nil { results := getResults() - results.MatchTraces(t, testutil.MatchOptions{CheckDetachedSpans: true}, - testutil.Match{Name: "External: External Root", TraceCount: 1, Services: []string{ + results.MatchTraces(t, MatchOptions{CheckDetachedSpans: true}, + Match{Name: "External: External Root", TraceCount: 1, Services: []string{ "Authorize", "Authenticate", "Control Plane", diff --git a/internal/testenv/upstreams/grpc.go b/internal/testenv/upstreams/grpc.go index b4329c84e..90773d8c1 100644 --- a/internal/testenv/upstreams/grpc.go +++ b/internal/testenv/upstreams/grpc.go @@ -142,9 +142,9 @@ func (g *grpcUpstream) Run(ctx context.Context) error { } server := grpc.NewServer(append(g.serverOpts, grpc.Creds(g.creds), - grpc.StatsHandler(trace.NewServerStatsHandler(otelgrpc.NewServerHandler( + grpc.StatsHandler(otelgrpc.NewServerHandler( otelgrpc.WithTracerProvider(g.serverTracerProvider.Value()), - ))), + )), )...) for _, s := range g.services { server.RegisterService(s.desc, s.impl) diff --git a/internal/telemetry/trace/buffer.go b/internal/testutil/tracetest/buffer.go similarity index 67% rename from internal/telemetry/trace/buffer.go rename to internal/testutil/tracetest/buffer.go index f74ccb8ef..a39cdd9d8 100644 --- a/internal/telemetry/trace/buffer.go +++ b/internal/testutil/tracetest/buffer.go @@ -1,4 +1,4 @@ -package trace +package tracetest import ( "cmp" @@ -6,10 +6,8 @@ import ( "maps" "slices" "sync" - "unique" "github.com/pomerium/pomerium/internal/hashutil" - oteltrace "go.opentelemetry.io/otel/trace" commonv1 "go.opentelemetry.io/proto/otlp/common/v1" resourcev1 "go.opentelemetry.io/proto/otlp/resource/v1" tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" @@ -72,30 +70,6 @@ func (rb *ResourceBuffer) Flush() []*tracev1.ScopeSpans { return out } -func (rb *ResourceBuffer) FlushAs(rewriteTraceID unique.Handle[oteltrace.TraceID]) []*tracev1.ScopeSpans { - out := make([]*tracev1.ScopeSpans, 0, len(rb.spansByScope)) - for _, key := range slices.Sorted(maps.Keys(rb.spansByScope)) { - spans := rb.spansByScope[key] - { - id := rewriteTraceID.Value() - for _, span := range spans.spans { - copy(span.TraceId, id[:]) - } - } - slices.SortStableFunc(spans.spans, func(a, b *tracev1.Span) int { - return cmp.Compare(a.StartTimeUnixNano, b.StartTimeUnixNano) - }) - scopeSpans := &tracev1.ScopeSpans{ - Scope: spans.scope.Scope, - SchemaUrl: spans.scope.Schema, - Spans: spans.spans, - } - out = append(out, scopeSpans) - } - clear(rb.spansByScope) - return out -} - func (rb *ResourceBuffer) Merge(other *ResourceBuffer) { for scope, otherSpans := range other.spansByScope { if ourSpans, ok := rb.spansByScope[scope]; !ok { @@ -107,23 +81,6 @@ func (rb *ResourceBuffer) Merge(other *ResourceBuffer) { clear(other.spansByScope) } -func (rb *ResourceBuffer) MergeAs(other *ResourceBuffer, rewriteTraceID unique.Handle[oteltrace.TraceID]) { - for scope, otherSpans := range other.spansByScope { - { - id := rewriteTraceID.Value() - for _, span := range otherSpans.spans { - copy(span.TraceId, id[:]) - } - } - if ourSpans, ok := rb.spansByScope[scope]; !ok { - rb.spansByScope[scope] = otherSpans - } else { - ourSpans.Insert(otherSpans.spans...) - } - } - clear(other.spansByScope) -} - type Buffer struct { scopesByResourceID map[string]*ResourceBuffer } @@ -161,21 +118,6 @@ func (b *Buffer) Flush() []*tracev1.ResourceSpans { return out } -func (b *Buffer) FlushAs(rewriteTraceID unique.Handle[oteltrace.TraceID]) []*tracev1.ResourceSpans { - out := make([]*tracev1.ResourceSpans, 0, len(b.scopesByResourceID)) - for _, key := range slices.Sorted(maps.Keys(b.scopesByResourceID)) { - scopes := b.scopesByResourceID[key] - resourceSpans := &tracev1.ResourceSpans{ - Resource: scopes.resource.Resource, - ScopeSpans: scopes.FlushAs(rewriteTraceID), - SchemaUrl: scopes.resource.Schema, - } - out = append(out, resourceSpans) - } - clear(b.scopesByResourceID) - return out -} - func (b *Buffer) Merge(other *Buffer) { if b != nil { for k, otherV := range other.scopesByResourceID { @@ -189,21 +131,6 @@ func (b *Buffer) Merge(other *Buffer) { clear(other.scopesByResourceID) } -func (b *Buffer) MergeAs(other *Buffer, rewriteTraceID unique.Handle[oteltrace.TraceID]) { - if b != nil { - for k, otherV := range other.scopesByResourceID { - if v, ok := b.scopesByResourceID[k]; !ok { - newRb := NewResourceBuffer(otherV.resource) - newRb.MergeAs(otherV, rewriteTraceID) - b.scopesByResourceID[k] = newRb - } else { - v.MergeAs(otherV, rewriteTraceID) - } - } - } - clear(other.scopesByResourceID) -} - func (b *Buffer) IsEmpty() bool { return len(b.scopesByResourceID) == 0 } diff --git a/internal/telemetry/trace/buffer_test.go b/internal/testutil/tracetest/buffer_test.go similarity index 56% rename from internal/telemetry/trace/buffer_test.go rename to internal/testutil/tracetest/buffer_test.go index 557c9489a..5bb9146e9 100644 --- a/internal/telemetry/trace/buffer_test.go +++ b/internal/testutil/tracetest/buffer_test.go @@ -1,110 +1,13 @@ -package trace_test +package tracetest import ( - "encoding/binary" - "fmt" "testing" - "unique" - "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/testutil" "github.com/stretchr/testify/assert" - oteltrace "go.opentelemetry.io/otel/trace" - commonv1 "go.opentelemetry.io/proto/otlp/common/v1" - resourcev1 "go.opentelemetry.io/proto/otlp/resource/v1" tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" ) -type ( - Trace uint32 - Span uint32 - Scope uint32 - Schema uint32 - Resource uint32 -) - -func (n Trace) String() string { return fmt.Sprintf("Trace %d", n) } -func (n Span) String() string { return fmt.Sprintf("Span %d", n) } -func (n Scope) String() string { return fmt.Sprintf("Scope %d", n) } -func (n Schema) String() string { return fmt.Sprintf("Schema %d", n) } -func (n Resource) String() string { return fmt.Sprintf("Resource %d", n) } - -func (n Trace) ID() unique.Handle[oteltrace.TraceID] { - id, _ := trace.ToTraceID(n.B()) - return id -} - -func (n Trace) B() []byte { - var id oteltrace.TraceID - binary.BigEndian.PutUint32(id[12:], uint32(n)) - return id[:] -} - -func (n Span) ID() oteltrace.SpanID { - id, _ := trace.ToSpanID(n.B()) - return id -} - -func (n Span) B() []byte { - var id oteltrace.SpanID - binary.BigEndian.PutUint32(id[4:], uint32(n)) - return id[:] -} - -func (n Scope) Make(s ...Schema) *trace.ScopeInfo { - if len(s) == 0 { - s = append(s, Schema(0)) - } - return trace.NewScopeInfo(&commonv1.InstrumentationScope{ - Name: n.String(), - Version: "v1", - Attributes: []*commonv1.KeyValue{ - { - Key: "id", - Value: &commonv1.AnyValue{ - Value: &commonv1.AnyValue_IntValue{ - IntValue: int64(n), - }, - }, - }, - }, - }, s[0].String()) -} - -func (n Resource) Make(s ...Schema) *trace.ResourceInfo { - if len(s) == 0 { - s = append(s, Schema(0)) - } - return trace.NewResourceInfo(&resourcev1.Resource{ - Attributes: []*commonv1.KeyValue{ - { - Key: "name", - Value: &commonv1.AnyValue{ - Value: &commonv1.AnyValue_StringValue{ - StringValue: n.String(), - }, - }, - }, - { - Key: "id", - Value: &commonv1.AnyValue{ - Value: &commonv1.AnyValue_IntValue{ - IntValue: int64(n), - }, - }, - }, - }, - }, s[0].String()) -} - -func Traceparent(trace Trace, span Span, sampled bool) string { - sampledStr := "00" - if sampled { - sampledStr = "01" - } - return fmt.Sprintf("00-%s-%s-%s", trace.ID().Value(), span.ID(), sampledStr) -} - func TestBuffer(t *testing.T) { t.Parallel() @@ -128,8 +31,8 @@ func TestBuffer(t *testing.T) { {TraceId: Trace(2).B(), SpanId: Span(16).B(), StartTimeUnixNano: 16}, } - newTestBuffer := func() *trace.Buffer { - b := trace.NewBuffer() + newTestBuffer := func() *Buffer { + b := NewBuffer() b.Insert(Resource(1).Make(), Scope(1).Make(), s[0]) b.Insert(Resource(1).Make(), Scope(1).Make(), s[1]) b.Insert(Resource(1).Make(), Scope(1).Make(), s[2]) @@ -191,26 +94,12 @@ func TestBuffer(t *testing.T) { assert.True(t, b.IsEmpty()) testutil.AssertProtoEqual(t, newExpectedSpans(), actual) }) - t.Run("FlushAs", func(t *testing.T) { - b := newTestBuffer() - actual := b.FlushAs(Trace(100).ID()) - assert.True(t, b.IsEmpty()) - expected := newExpectedSpans() - for _, resourceSpans := range expected { - for _, scopeSpans := range resourceSpans.ScopeSpans { - for _, span := range scopeSpans.Spans { - span.TraceId = Trace(100).B() - } - } - } - testutil.AssertProtoEqual(t, expected, actual) - }) t.Run("Default scope", func(t *testing.T) { - b := trace.NewBuffer() - b.Insert(Resource(1).Make(Schema(2)), trace.NewScopeInfo(nil, ""), s[0]) - b.Insert(Resource(1).Make(Schema(2)), trace.NewScopeInfo(nil, ""), s[1]) - b.Insert(Resource(1).Make(Schema(2)), trace.NewScopeInfo(nil, ""), s[2]) + b := NewBuffer() + b.Insert(Resource(1).Make(Schema(2)), NewScopeInfo(nil, ""), s[0]) + b.Insert(Resource(1).Make(Schema(2)), NewScopeInfo(nil, ""), s[1]) + b.Insert(Resource(1).Make(Schema(2)), NewScopeInfo(nil, ""), s[2]) actual := b.Flush() testutil.AssertProtoEqual(t, []*tracev1.ResourceSpans{ { diff --git a/internal/telemetry/trace/mock_otlptrace/mock_client.go b/internal/testutil/tracetest/mock_otlptrace/mock_client.go similarity index 100% rename from internal/telemetry/trace/mock_otlptrace/mock_client.go rename to internal/testutil/tracetest/mock_otlptrace/mock_client.go diff --git a/internal/testutil/tracing.go b/internal/testutil/tracetest/tracing.go similarity index 87% rename from internal/testutil/tracing.go rename to internal/testutil/tracetest/tracing.go index 9b1365f18..06db43f62 100644 --- a/internal/testutil/tracing.go +++ b/internal/testutil/tracetest/tracing.go @@ -1,8 +1,9 @@ -package testutil +package tracetest import ( "cmp" "context" + "encoding/binary" "encoding/json" "fmt" "maps" @@ -28,6 +29,96 @@ import ( "google.golang.org/protobuf/testing/protocmp" ) +type ( + Trace uint32 + Span uint32 + Scope uint32 + Schema uint32 + Resource uint32 +) + +func (n Trace) String() string { return fmt.Sprintf("Trace %d", n) } +func (n Span) String() string { return fmt.Sprintf("Span %d", n) } +func (n Scope) String() string { return fmt.Sprintf("Scope %d", n) } +func (n Schema) String() string { return fmt.Sprintf("Schema %d", n) } +func (n Resource) String() string { return fmt.Sprintf("Resource %d", n) } + +func (n Trace) ID() unique.Handle[oteltrace.TraceID] { + id, _ := trace.ToTraceID(n.B()) + return id +} + +func (n Trace) B() []byte { + var id oteltrace.TraceID + binary.BigEndian.PutUint32(id[12:], uint32(n)) + return id[:] +} + +func (n Span) ID() oteltrace.SpanID { + id, _ := trace.ToSpanID(n.B()) + return id +} + +func (n Span) B() []byte { + var id oteltrace.SpanID + binary.BigEndian.PutUint32(id[4:], uint32(n)) + return id[:] +} + +func (n Scope) Make(s ...Schema) *ScopeInfo { + if len(s) == 0 { + s = append(s, Schema(0)) + } + return NewScopeInfo(&commonv1.InstrumentationScope{ + Name: n.String(), + Version: "v1", + Attributes: []*commonv1.KeyValue{ + { + Key: "id", + Value: &commonv1.AnyValue{ + Value: &commonv1.AnyValue_IntValue{ + IntValue: int64(n), + }, + }, + }, + }, + }, s[0].String()) +} + +func (n Resource) Make(s ...Schema) *ResourceInfo { + if len(s) == 0 { + s = append(s, Schema(0)) + } + return NewResourceInfo(&resourcev1.Resource{ + Attributes: []*commonv1.KeyValue{ + { + Key: "name", + Value: &commonv1.AnyValue{ + Value: &commonv1.AnyValue_StringValue{ + StringValue: n.String(), + }, + }, + }, + { + Key: "id", + Value: &commonv1.AnyValue{ + Value: &commonv1.AnyValue_IntValue{ + IntValue: int64(n), + }, + }, + }, + }, + }, s[0].String()) +} + +func Traceparent(trace Trace, span Span, sampled bool) string { + sampledStr := "00" + if sampled { + sampledStr = "01" + } + return fmt.Sprintf("00-%s-%s-%s", trace.ID().Value(), span.ID(), sampledStr) +} + type TraceResults struct { resourceSpans []*tracev1.ResourceSpans @@ -86,11 +177,7 @@ type TraceDetails struct { func (td *TraceDetails) Equal(other *TraceDetails) (bool, string) { diffSpans := func(a, b []*SpanDetails) (bool, string) { for i := range len(a) { - aRaw := proto.Clone(a[i].Raw).(*tracev1.Span) - trace.FormatSpanName(aRaw) - bRaw := proto.Clone(b[i].Raw).(*tracev1.Span) - trace.FormatSpanName(bRaw) - diff := gocmp.Diff(aRaw, bRaw, protocmp.Transform()) + diff := gocmp.Diff(a[i], b[i], protocmp.Transform()) if diff != "" { return false, diff } @@ -426,12 +513,12 @@ func (tr *TraceResults) AssertEqual(t testing.TB, expectedResults *TraceResults, } func FlattenResourceSpans(lists [][]*tracev1.ResourceSpans) []*tracev1.ResourceSpans { - res := trace.NewBuffer() + res := NewBuffer() for _, list := range lists { for _, resource := range list { - resInfo := trace.NewResourceInfo(resource.Resource, resource.SchemaUrl) + resInfo := NewResourceInfo(resource.Resource, resource.SchemaUrl) for _, scope := range resource.ScopeSpans { - scopeInfo := trace.NewScopeInfo(scope.Scope, scope.SchemaUrl) + scopeInfo := NewScopeInfo(scope.Scope, scope.SchemaUrl) for _, span := range scope.Spans { res.Insert(resInfo, scopeInfo, span) }