3pp-mirror/pomerium
1
0
Fork 0
mirror of https://github.com/pomerium/pomerium.git synced 2025-05-01 03:16:31 +02:00
Code Issues Projects Releases Packages Wiki Activity

envoyconfig: cleanup (#5350)

Browse source 
* envoyconfig: cleanup

* remove listener access log for mtls for insecure server which can't use mtls

* use new functions

* rename method

* refactor common code
This commit is contained in:
Caleb Doxsey 2024-11-18 09:50:23 -07:00 • committed by GitHub
parent 3e51b4f905
commit 20a9be891f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 1227 additions and 1182 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
config/envoyconfig/envoyconfig.go
View file

@ -4,9 +4,6 @@ package envoyconfig
import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"errors"
"fmt"
"net"
@ -21,7 +18,6 @@ import (
envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
envoy_extensions_access_loggers_grpc_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/grpc/v3"
envoy_extensions_transport_sockets_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
"github.com/martinlindhe/base36"
"golang.org/x/net/nettest"
"google.golang.org/protobuf/proto"
@ -151,39 +147,6 @@ func buildAddress(hostport string, defaultPort uint32) *envoy_config_core_v3.Add
}
}
func (b *Builder) envoyTLSCertificateFromGoTLSCertificate(
ctx context.Context,
cert *tls.Certificate,
) *envoy_extensions_transport_sockets_tls_v3.TlsCertificate {
envoyCert := &envoy_extensions_transport_sockets_tls_v3.TlsCertificate{}
var chain bytes.Buffer
for _, cbs := range cert.Certificate {
_ = pem.Encode(&chain, &pem.Block{
Type: "CERTIFICATE",
Bytes: cbs,
})
}
envoyCert.CertificateChain = b.filemgr.BytesDataSource("tls-crt.pem", chain.Bytes())
if cert.OCSPStaple != nil {
envoyCert.OcspStaple = b.filemgr.BytesDataSource("ocsp-staple", cert.OCSPStaple)
}
if bs, err := x509.MarshalPKCS8PrivateKey(cert.PrivateKey); err == nil {
envoyCert.PrivateKey = b.filemgr.BytesDataSource("tls-key.pem", pem.EncodeToMemory(
&pem.Block{
Type: "PRIVATE KEY",
Bytes: bs,
},
))
} else {
log.Ctx(ctx).Error().Err(err).Msg("failed to marshal private key for tls config")
}
for _, scts := range cert.SignedCertificateTimestamps {
envoyCert.SignedCertificateTimestamp = append(envoyCert.SignedCertificateTimestamp,
b.filemgr.BytesDataSource("signed-certificate-timestamp", scts))
}
return envoyCert
}
var rootCABundle struct {
sync.Once
value string

649
config/envoyconfig/listeners.go
View file

@ -1,36 +1,14 @@
package envoyconfig
import (
"bytes"
"cmp"
"context"
"crypto/tls"
"encoding/base64"
"fmt"
"net"
"net/url"
"runtime"
"strings"
"time"
envoy_config_accesslog_v3 "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3"
envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
envoy_config_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
envoy_config_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
envoy_extensions_access_loggers_grpc_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/grpc/v3"
envoy_http_connection_manager "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
envoy_extensions_transport_sockets_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3"
"github.com/hashicorp/go-set/v3"
"google.golang.org/protobuf/types/known/durationpb"
"google.golang.org/protobuf/types/known/wrapperspb"
"github.com/pomerium/pomerium/config"
"github.com/pomerium/pomerium/internal/hashutil"
"github.com/pomerium/pomerium/internal/log"
"github.com/pomerium/pomerium/internal/telemetry/metrics"
"github.com/pomerium/pomerium/internal/telemetry/trace"
"github.com/pomerium/pomerium/internal/urlutil"
)
const listenerBufferLimit uint32 = 32 * 1024
@ -62,7 +40,7 @@ func (b *Builder) BuildListeners(
listeners = append(listeners, li)
}
if cfg.Options.MetricsAddr != "" {
if shouldStartMetricsListener(cfg.Options) {
li, err := b.buildMetricsListener(cfg)
if err != nil {
return nil, err
@ -70,7 +48,7 @@ func (b *Builder) BuildListeners(
listeners = append(listeners, li)
}
if cfg.Options.EnvoyAdminAddress != "" {
if shouldStartEnvoyAdminListener(cfg.Options) {
li, err := b.buildEnvoyAdminListener(ctx, cfg)
if err != nil {
return nil, err
@ -87,615 +65,8 @@ func (b *Builder) BuildListeners(
return listeners, nil
}
func getAllCertificates(cfg *config.Config) ([]tls.Certificate, error) {
allCertificates, err := cfg.AllCertificates()
if err != nil {
return nil, fmt.Errorf("error collecting all certificates: %w", err)
}
wc, err := cfg.GenerateCatchAllCertificate()
if err != nil {
return nil, fmt.Errorf("error getting wildcard certificate: %w", err)
}
return append(allCertificates, *wc), nil
}
func (b *Builder) buildTLSSocket(ctx context.Context, cfg *config.Config, certs []tls.Certificate) (*envoy_config_core_v3.TransportSocket, error) {
tlsContext, err := b.buildDownstreamTLSContextMulti(ctx, cfg, certs)
if err != nil {
return nil, err
}
return &envoy_config_core_v3.TransportSocket{
Name: "tls",
ConfigType: &envoy_config_core_v3.TransportSocket_TypedConfig{
TypedConfig: marshalAny(tlsContext),
},
}, nil
}
func listenerAccessLog() []*envoy_config_accesslog_v3.AccessLog {
cc := &envoy_extensions_access_loggers_grpc_v3.CommonGrpcAccessLogConfig{
LogName: "ingress-http-listener",
GrpcService: &envoy_config_core_v3.GrpcService{
TargetSpecifier: &envoy_config_core_v3.GrpcService_EnvoyGrpc_{
EnvoyGrpc: &envoy_config_core_v3.GrpcService_EnvoyGrpc{
ClusterName: "pomerium-control-plane-grpc",
},
},
},
TransportApiVersion: envoy_config_core_v3.ApiVersion_V3,
}
tcp := marshalAny(
&envoy_extensions_access_loggers_grpc_v3.TcpGrpcAccessLogConfig{CommonConfig: cc})
return []*envoy_config_accesslog_v3.AccessLog{
{
Name: "envoy.access_loggers.tcp_grpc",
ConfigType: &envoy_config_accesslog_v3.AccessLog_TypedConfig{TypedConfig: tcp},
},
}
}
func (b *Builder) buildMainListener(
ctx context.Context,
cfg *config.Config,
fullyStatic bool,
) (*envoy_config_listener_v3.Listener, error) {
li := newEnvoyListener("http-ingress")
if cfg.Options.UseProxyProtocol {
li.ListenerFilters = append(li.ListenerFilters, ProxyProtocolFilter())
}
if cfg.Options.DownstreamMTLS.Enforcement == config.MTLSEnforcementRejectConnection {
li.AccessLog = listenerAccessLog()
}
if cfg.Options.InsecureServer {
li.Address = buildAddress(cfg.Options.Addr, 80)
filter, err := b.buildMainHTTPConnectionManagerFilter(ctx, cfg, fullyStatic)
if err != nil {
return nil, err
}
li.FilterChains = []*envoy_config_listener_v3.FilterChain{{
Filters: []*envoy_config_listener_v3.Filter{
filter,
},
}}
} else {
li.Address = buildAddress(cfg.Options.Addr, 443)
li.ListenerFilters = append(li.ListenerFilters, TLSInspectorFilter())
li.FilterChains = append(li.FilterChains, b.buildACMETLSALPNFilterChain())
allCertificates, err := getAllCertificates(cfg)
if err != nil {
return nil, err
}
filter, err := b.buildMainHTTPConnectionManagerFilter(ctx, cfg, fullyStatic)
if err != nil {
return nil, err
}
filterChain := &envoy_config_listener_v3.FilterChain{
Filters: []*envoy_config_listener_v3.Filter{filter},
}
li.FilterChains = append(li.FilterChains, filterChain)
sock, err := b.buildTLSSocket(ctx, cfg, allCertificates)
if err != nil {
return nil, fmt.Errorf("error building TLS socket: %w", err)
}
filterChain.TransportSocket = sock
}
return li, nil
}
func (b *Builder) buildMetricsListener(cfg *config.Config) (*envoy_config_listener_v3.Listener, error) {
filter, err := b.buildMetricsHTTPConnectionManagerFilter()
if err != nil {
return nil, err
}
filterChain := &envoy_config_listener_v3.FilterChain{
Filters: []*envoy_config_listener_v3.Filter{
filter,
},
}
cert, err := cfg.Options.GetMetricsCertificate()
if err != nil {
return nil, err
}
if cert != nil {
dtc := &envoy_extensions_transport_sockets_tls_v3.DownstreamTlsContext{
CommonTlsContext: &envoy_extensions_transport_sockets_tls_v3.CommonTlsContext{
TlsParams: tlsDownstreamParams,
TlsCertificates: []*envoy_extensions_transport_sockets_tls_v3.TlsCertificate{
b.envoyTLSCertificateFromGoTLSCertificate(context.TODO(), cert),
},
AlpnProtocols: []string{"h2", "http/1.1"},
},
}
if cfg.Options.MetricsClientCA != "" {
bs, err := base64.StdEncoding.DecodeString(cfg.Options.MetricsClientCA)
if err != nil {
return nil, fmt.Errorf("xds: invalid metrics_client_ca: %w", err)
}
dtc.RequireClientCertificate = wrapperspb.Bool(true)
dtc.CommonTlsContext.ValidationContextType = &envoy_extensions_transport_sockets_tls_v3.CommonTlsContext_ValidationContext{
ValidationContext: &envoy_extensions_transport_sockets_tls_v3.CertificateValidationContext{
TrustChainVerification: envoy_extensions_transport_sockets_tls_v3.CertificateValidationContext_VERIFY_TRUST_CHAIN,
TrustedCa: b.filemgr.BytesDataSource("metrics_client_ca.pem", bs),
},
}
} else if cfg.Options.MetricsClientCAFile != "" {
dtc.RequireClientCertificate = wrapperspb.Bool(true)
dtc.CommonTlsContext.ValidationContextType = &envoy_extensions_transport_sockets_tls_v3.CommonTlsContext_ValidationContext{
ValidationContext: &envoy_extensions_transport_sockets_tls_v3.CertificateValidationContext{
TrustChainVerification: envoy_extensions_transport_sockets_tls_v3.CertificateValidationContext_VERIFY_TRUST_CHAIN,
TrustedCa: b.filemgr.FileDataSource(cfg.Options.MetricsClientCAFile),
},
}
}
tc := marshalAny(dtc)
filterChain.TransportSocket = &envoy_config_core_v3.TransportSocket{
Name: "tls",
ConfigType: &envoy_config_core_v3.TransportSocket_TypedConfig{
TypedConfig: tc,
},
}
}
// we ignore the host part of the address, only binding to
host, port, err := net.SplitHostPort(cfg.Options.MetricsAddr)
if err != nil {
return nil, fmt.Errorf("metrics_addr %s: %w", cfg.Options.MetricsAddr, err)
}
if port == "" {
return nil, fmt.Errorf("metrics_addr %s: port is required", cfg.Options.MetricsAddr)
}
// unless an explicit IP address was provided, and bind to all interfaces if hostname was provided
if net.ParseIP(host) == nil {
host = ""
}
addr := buildAddress(net.JoinHostPort(host, port), 9902)
li := newEnvoyListener(fmt.Sprintf("metrics-ingress-%d", hashutil.MustHash(addr)))
li.Address = addr
li.FilterChains = []*envoy_config_listener_v3.FilterChain{filterChain}
return li, nil
}
func (b *Builder) buildMainHTTPConnectionManagerFilter(
ctx context.Context,
cfg *config.Config,
fullyStatic bool,
) (*envoy_config_listener_v3.Filter, error) {
var grpcClientTimeout *durationpb.Duration
if cfg.Options.GRPCClientTimeout != 0 {
grpcClientTimeout = durationpb.New(cfg.Options.GRPCClientTimeout)
} else {
grpcClientTimeout = durationpb.New(30 * time.Second)
}
filters := []*envoy_http_connection_manager.HttpFilter{
LuaFilter(luascripts.RemoveImpersonateHeaders),
LuaFilter(luascripts.SetClientCertificateMetadata),
ExtAuthzFilter(grpcClientTimeout),
LuaFilter(luascripts.ExtAuthzSetCookie),
LuaFilter(luascripts.CleanUpstream),
LuaFilter(luascripts.RewriteHeaders),
}
filters = append(filters, HTTPRouterFilter())
var maxStreamDuration *durationpb.Duration
if cfg.Options.WriteTimeout > 0 {
maxStreamDuration = durationpb.New(cfg.Options.WriteTimeout)
}
tracingProvider, err := buildTracingHTTP(cfg.Options)
if err != nil {
return nil, err
}
localReply, err := b.buildLocalReplyConfig(cfg.Options)
if err != nil {
return nil, err
}
mgr := &envoy_http_connection_manager.HttpConnectionManager{
AlwaysSetRequestIdInResponse: true,
CodecType: cfg.Options.GetCodecType().ToEnvoy(),
StatPrefix: "ingress",
HttpFilters: filters,
AccessLog: buildAccessLogs(cfg.Options),
CommonHttpProtocolOptions: &envoy_config_core_v3.HttpProtocolOptions{
IdleTimeout: durationpb.New(cfg.Options.IdleTimeout),
MaxStreamDuration: maxStreamDuration,
},
HttpProtocolOptions: http1ProtocolOptions,
RequestTimeout: durationpb.New(cfg.Options.ReadTimeout),
Tracing: &envoy_http_connection_manager.HttpConnectionManager_Tracing{
RandomSampling: &envoy_type_v3.Percent{Value: cfg.Options.TracingSampleRate * 100},
Provider: tracingProvider,
},
// See https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for
UseRemoteAddress: &wrapperspb.BoolValue{Value: true},
SkipXffAppend: cfg.Options.SkipXffAppend,
XffNumTrustedHops: cfg.Options.XffNumTrustedHops,
LocalReplyConfig: localReply,
NormalizePath: wrapperspb.Bool(true),
}
if fullyStatic {
routeConfiguration, err := b.buildMainRouteConfiguration(ctx, cfg)
if err != nil {
return nil, err
}
mgr.RouteSpecifier = &envoy_http_connection_manager.HttpConnectionManager_RouteConfig{
RouteConfig: routeConfiguration,
}
} else {
mgr.RouteSpecifier = &envoy_http_connection_manager.HttpConnectionManager_Rds{
Rds: &envoy_http_connection_manager.Rds{
ConfigSource: &envoy_config_core_v3.ConfigSource{
ResourceApiVersion: envoy_config_core_v3.ApiVersion_V3,
ConfigSourceSpecifier: &envoy_config_core_v3.ConfigSource_Ads{},
},
RouteConfigName: "main",
},
}
}
return HTTPConnectionManagerFilter(mgr), nil
}
func (b *Builder) buildMetricsHTTPConnectionManagerFilter() (*envoy_config_listener_v3.Filter, error) {
rc, err := b.buildRouteConfiguration("metrics", []*envoy_config_route_v3.VirtualHost{{
Name: "metrics",
Domains: []string{"*"},
Routes: []*envoy_config_route_v3.Route{
{
Name: "envoy-metrics",
Match: &envoy_config_route_v3.RouteMatch{
PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: metrics.EnvoyMetricsPath},
},
Action: &envoy_config_route_v3.Route_Route{
Route: &envoy_config_route_v3.RouteAction{
ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
Cluster: envoyAdminClusterName,
},
PrefixRewrite: "/stats/prometheus",
},
},
},
{
Name: "metrics",
Match: &envoy_config_route_v3.RouteMatch{
PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: "/"},
},
Action: &envoy_config_route_v3.Route_Route{
Route: &envoy_config_route_v3.RouteAction{
ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
Cluster: "pomerium-control-plane-metrics",
},
},
},
},
},
}})
if err != nil {
return nil, err
}
return HTTPConnectionManagerFilter(&envoy_http_connection_manager.HttpConnectionManager{
CodecType: envoy_http_connection_manager.HttpConnectionManager_AUTO,
StatPrefix: "metrics",
RouteSpecifier: &envoy_http_connection_manager.HttpConnectionManager_RouteConfig{
RouteConfig: rc,
},
HttpFilters: []*envoy_http_connection_manager.HttpFilter{
HTTPRouterFilter(),
},
}), nil
}
func (b *Builder) buildGRPCListener(ctx context.Context, cfg *config.Config) (*envoy_config_listener_v3.Listener, error) {
filter, err := b.buildGRPCHTTPConnectionManagerFilter()
if err != nil {
return nil, err
}
filterChain := envoy_config_listener_v3.FilterChain{
Filters: []*envoy_config_listener_v3.Filter{filter},
}
li := newEnvoyListener("grpc-ingress")
li.FilterChains = []*envoy_config_listener_v3.FilterChain{&filterChain}
if cfg.Options.GetGRPCInsecure() {
li.Address = buildAddress(cfg.Options.GetGRPCAddr(), 80)
return li, nil
}
li.Address = buildAddress(cfg.Options.GetGRPCAddr(), 443)
li.ListenerFilters = []*envoy_config_listener_v3.ListenerFilter{
TLSInspectorFilter(),
}
allCertificates, err := getAllCertificates(cfg)
if err != nil {
return nil, err
}
envoyCerts, err := b.envoyCertificates(ctx, allCertificates)
if err != nil {
return nil, err
}
tlsContext := &envoy_extensions_transport_sockets_tls_v3.DownstreamTlsContext{
CommonTlsContext: &envoy_extensions_transport_sockets_tls_v3.CommonTlsContext{
TlsParams: tlsDownstreamParams,
TlsCertificates: envoyCerts,
AlpnProtocols: []string{"h2"}, // gRPC requires HTTP/2
},
}
filterChain.TransportSocket = &envoy_config_core_v3.TransportSocket{
Name: "tls",
ConfigType: &envoy_config_core_v3.TransportSocket_TypedConfig{
TypedConfig: marshalAny(tlsContext),
},
}
return li, nil
}
func (b *Builder) buildGRPCHTTPConnectionManagerFilter() (*envoy_config_listener_v3.Filter, error) {
allow := []string{
"envoy.service.auth.v3.Authorization",
"databroker.DataBrokerService",
"registry.Registry",
"grpc.health.v1.Health",
}
routes := make([]*envoy_config_route_v3.Route, 0, len(allow))
for _, svc := range allow {
routes = append(routes, &envoy_config_route_v3.Route{
Name: "grpc",
Match: &envoy_config_route_v3.RouteMatch{
PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: fmt.Sprintf("/%s/", svc)},
Grpc: &envoy_config_route_v3.RouteMatch_GrpcRouteMatchOptions{},
},
Action: &envoy_config_route_v3.Route_Route{
Route: &envoy_config_route_v3.RouteAction{
ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
Cluster: "pomerium-control-plane-grpc",
},
// disable the timeout to support grpc streaming
Timeout: &durationpb.Duration{
Seconds: 0,
},
IdleTimeout: &durationpb.Duration{
Seconds: 0,
},
},
},
})
}
rc, err := b.buildRouteConfiguration("grpc", []*envoy_config_route_v3.VirtualHost{{
Name: "grpc",
Domains: []string{"*"},
Routes: routes,
}})
if err != nil {
return nil, err
}
return HTTPConnectionManagerFilter(&envoy_http_connection_manager.HttpConnectionManager{
CodecType: envoy_http_connection_manager.HttpConnectionManager_AUTO,
StatPrefix: "grpc_ingress",
// limit request first byte to last byte time
RequestTimeout: &durationpb.Duration{
Seconds: 15,
},
RouteSpecifier: &envoy_http_connection_manager.HttpConnectionManager_RouteConfig{
RouteConfig: rc,
},
HttpFilters: []*envoy_http_connection_manager.HttpFilter{
HTTPRouterFilter(),
},
}), nil
}
func (b *Builder) buildRouteConfiguration(name string, virtualHosts []*envoy_config_route_v3.VirtualHost) (*envoy_config_route_v3.RouteConfiguration, error) {
return &envoy_config_route_v3.RouteConfiguration{
Name: name,
VirtualHosts: virtualHosts,
// disable cluster validation since the order of LDS/CDS updates isn't guaranteed
ValidateClusters: &wrapperspb.BoolValue{Value: false},
}, nil
}
func (b *Builder) envoyCertificates(ctx context.Context, certs []tls.Certificate) (
[]*envoy_extensions_transport_sockets_tls_v3.TlsCertificate, error,
) {
envoyCerts := make([]*envoy_extensions_transport_sockets_tls_v3.TlsCertificate, 0, len(certs))
for i := range certs {
cert := &certs[i]
if err := validateCertificate(cert); err != nil {
return nil, fmt.Errorf("invalid certificate for domain %s: %w",
cert.Leaf.Subject.CommonName, err)
}
envoyCert := b.envoyTLSCertificateFromGoTLSCertificate(ctx, cert)
envoyCerts = append(envoyCerts, envoyCert)
}
return envoyCerts, nil
}
func (b *Builder) buildDownstreamTLSContextMulti(
ctx context.Context,
cfg *config.Config,
certs []tls.Certificate,
) (
*envoy_extensions_transport_sockets_tls_v3.DownstreamTlsContext,
error,
) {
envoyCerts, err := b.envoyCertificates(ctx, certs)
if err != nil {
return nil, err
}
dtc := &envoy_extensions_transport_sockets_tls_v3.DownstreamTlsContext{
CommonTlsContext: &envoy_extensions_transport_sockets_tls_v3.CommonTlsContext{
TlsParams: tlsDownstreamParams,
TlsCertificates: envoyCerts,
AlpnProtocols: getALPNProtos(cfg.Options),
},
}
b.buildDownstreamValidationContext(ctx, dtc, cfg)
return dtc, nil
}
func getALPNProtos(opts *config.Options) []string {
switch opts.GetCodecType() {
case config.CodecTypeHTTP1:
return []string{"http/1.1"}
case config.CodecTypeHTTP2:
return []string{"h2"}
default:
return []string{"h2", "http/1.1"}
}
}
func (b *Builder) buildDownstreamValidationContext(
ctx context.Context,
dtc *envoy_extensions_transport_sockets_tls_v3.DownstreamTlsContext,
cfg *config.Config,
) {
clientCA := clientCABundle(ctx, cfg)
if len(clientCA) == 0 {
return
}
vc := &envoy_extensions_transport_sockets_tls_v3.CertificateValidationContext{
TrustedCa: b.filemgr.BytesDataSource("client-ca.pem", clientCA),
MatchTypedSubjectAltNames: make([]*envoy_extensions_transport_sockets_tls_v3.SubjectAltNameMatcher,
0, len(cfg.Options.DownstreamMTLS.MatchSubjectAltNames)),
OnlyVerifyLeafCertCrl: true,
}
for i := range cfg.Options.DownstreamMTLS.MatchSubjectAltNames {
vc.MatchTypedSubjectAltNames = append(vc.MatchTypedSubjectAltNames,
cfg.Options.DownstreamMTLS.MatchSubjectAltNames[i].ToEnvoyProto())
}
if d := cfg.Options.DownstreamMTLS.GetMaxVerifyDepth(); d > 0 {
vc.MaxVerifyDepth = wrapperspb.UInt32(d)
}
if cfg.Options.DownstreamMTLS.GetEnforcement() == config.MTLSEnforcementRejectConnection {
dtc.RequireClientCertificate = wrapperspb.Bool(true)
} else {
vc.TrustChainVerification = envoy_extensions_transport_sockets_tls_v3.CertificateValidationContext_ACCEPT_UNTRUSTED
}
if crl := cfg.Options.DownstreamMTLS.CRL; crl != "" {
bs, err := base64.StdEncoding.DecodeString(crl)
if err != nil {
log.Ctx(ctx).Error().Err(err).Msg("invalid client CRL")
} else {
vc.Crl = b.filemgr.BytesDataSource("client-crl.pem", bs)
}
} else if crlf := cfg.Options.DownstreamMTLS.CRLFile; crlf != "" {
vc.Crl = b.filemgr.FileDataSource(crlf)
}
dtc.CommonTlsContext.ValidationContextType = &envoy_extensions_transport_sockets_tls_v3.CommonTlsContext_ValidationContext{
ValidationContext: vc,
}
}
// clientCABundle returns a bundle of the globally configured client CA and any
// per-route client CAs.
func clientCABundle(ctx context.Context, cfg *config.Config) []byte {
var bundle bytes.Buffer
ca, _ := cfg.Options.DownstreamMTLS.GetCA()
addCAToBundle(&bundle, ca)
for p := range cfg.Options.GetAllPolicies() {
// We don't need to check TLSDownstreamClientCAFile here because
// Policy.Validate() will populate TLSDownstreamClientCA when
// TLSDownstreamClientCAFile is set.
if p.TLSDownstreamClientCA == "" {
continue
}
ca, err := base64.StdEncoding.DecodeString(p.TLSDownstreamClientCA)
if err != nil {
log.Ctx(ctx).Error().Stringer("policy", p).Err(err).Msg("invalid client CA")
continue
}
addCAToBundle(&bundle, ca)
}
return bundle.Bytes()
}
func addCAToBundle(bundle *bytes.Buffer, ca []byte) {
if len(ca) == 0 {
return
}
bundle.Write(ca)
// Make sure each CA is separated by a newline.
if ca[len(ca)-1] != '\n' {
bundle.WriteByte('\n')
}
}
func getAllRouteableHosts(options *config.Options, addr string) ([]string, error) {
allHosts := set.NewTreeSet(cmp.Compare[string])
if addr == options.Addr {
hosts, err := options.GetAllRouteableHTTPHosts()
if err != nil {
return nil, err
}
allHosts.InsertSlice(hosts)
}
if addr == options.GetGRPCAddr() {
hosts, err := options.GetAllRouteableGRPCHosts()
if err != nil {
return nil, err
}
allHosts.InsertSlice(hosts)
}
var filtered []string
for host := range allHosts.Items() {
if !strings.Contains(host, "*") {
filtered = append(filtered, host)
}
}
return filtered, nil
}
func urlsMatchHost(urls []*url.URL, host string) bool {
for _, u := range urls {
if urlMatchesHost(u, host) {
return true
}
}
return false
}
func urlMatchesHost(u *url.URL, host string) bool {
for _, h := range urlutil.GetDomainsForURL(u, true) {
if h == host {
return true
}
}
return false
}
// newEnvoyListener creates envoy listener with certain default values
func newEnvoyListener(name string) *envoy_config_listener_v3.Listener {
// newListener creates envoy listener with certain default values
func newListener(name string) *envoy_config_listener_v3.Listener {
return &envoy_config_listener_v3.Listener{
Name: name,
PerConnectionBufferLimitBytes: wrapperspb.UInt32(listenerBufferLimit),
@ -706,15 +77,3 @@ func newEnvoyListener(name string) *envoy_config_listener_v3.Listener {
EnableReusePort: wrapperspb.Bool(runtime.GOOS == "linux"),
}
}
func shouldStartMainListener(options *config.Options) bool {
return config.IsAuthenticate(options.Services) || config.IsProxy(options.Services)
}
func shouldStartGRPCListener(options *config.Options) bool {
if options.GetGRPCAddr() == "" {
return false
}
return config.IsAuthorize(options.Services) || config.IsDataBroker(options.Services)
}

20
config/envoyconfig/listeners_envoy_admin.go
View file

@ -12,10 +12,7 @@ import (
)
func (b *Builder) buildEnvoyAdminListener(_ context.Context, cfg *config.Config) (*envoy_config_listener_v3.Listener, error) {
filter, err := b.buildEnvoyAdminHTTPConnectionManagerFilter()
if err != nil {
return nil, err
}
filter := b.buildEnvoyAdminHTTPConnectionManagerFilter()
filterChain := &envoy_config_listener_v3.FilterChain{
Filters: []*envoy_config_listener_v3.Filter{
@ -28,14 +25,14 @@ func (b *Builder) buildEnvoyAdminListener(_ context.Context, cfg *config.Config)
return nil, fmt.Errorf("envoy_admin_addr %s: %w", cfg.Options.EnvoyAdminAddress, err)
}
li := newEnvoyListener("envoy-admin")
li := newListener("envoy-admin")
li.Address = addr
li.FilterChains = []*envoy_config_listener_v3.FilterChain{filterChain}
return li, nil
}
func (b *Builder) buildEnvoyAdminHTTPConnectionManagerFilter() (*envoy_config_listener_v3.Filter, error) {
rc, err := b.buildRouteConfiguration("envoy-admin", []*envoy_config_route_v3.VirtualHost{{
func (b *Builder) buildEnvoyAdminHTTPConnectionManagerFilter() *envoy_config_listener_v3.Filter {
rc := newRouteConfiguration("envoy-admin", []*envoy_config_route_v3.VirtualHost{{
Name: "envoy-admin",
Domains: []string{"*"},
Routes: []*envoy_config_route_v3.Route{
@ -54,9 +51,6 @@ func (b *Builder) buildEnvoyAdminHTTPConnectionManagerFilter() (*envoy_config_li
},
},
}})
if err != nil {
return nil, err
}
return HTTPConnectionManagerFilter(&envoy_http_connection_manager.HttpConnectionManager{
CodecType: envoy_http_connection_manager.HttpConnectionManager_AUTO,
@ -67,5 +61,9 @@ func (b *Builder) buildEnvoyAdminHTTPConnectionManagerFilter() (*envoy_config_li
HttpFilters: []*envoy_http_connection_manager.HttpFilter{
HTTPRouterFilter(),
},
}), nil
})
}
func shouldStartEnvoyAdminListener(options *config.Options) bool {
return options.EnvoyAdminAddress != ""
}

114
config/envoyconfig/listeners_grpc.go Normal file
View file

@ -0,0 +1,114 @@
package envoyconfig
import (
"context"
"fmt"
envoy_config_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
envoy_config_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
envoy_http_connection_manager "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
envoy_extensions_transport_sockets_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
"google.golang.org/protobuf/types/known/durationpb"
"github.com/pomerium/pomerium/config"
)
func (b *Builder) buildGRPCListener(ctx context.Context, cfg *config.Config) (*envoy_config_listener_v3.Listener, error) {
filter := b.buildGRPCHTTPConnectionManagerFilter()
filterChain := envoy_config_listener_v3.FilterChain{
Filters: []*envoy_config_listener_v3.Filter{filter},
}
li := newListener("grpc-ingress")
li.FilterChains = []*envoy_config_listener_v3.FilterChain{&filterChain}
if cfg.Options.GetGRPCInsecure() {
li.Address = buildAddress(cfg.Options.GetGRPCAddr(), 80)
return li, nil
}
li.Address = buildAddress(cfg.Options.GetGRPCAddr(), 443)
li.ListenerFilters = []*envoy_config_listener_v3.ListenerFilter{
TLSInspectorFilter(),
}
allCertificates, err := getAllCertificates(cfg)
if err != nil {
return nil, err
}
envoyCerts, err := b.envoyTLSCertificatesFromGoTLSCertificates(ctx, allCertificates)
if err != nil {
return nil, err
}
tlsContext := &envoy_extensions_transport_sockets_tls_v3.DownstreamTlsContext{
CommonTlsContext: &envoy_extensions_transport_sockets_tls_v3.CommonTlsContext{
TlsParams: tlsDownstreamParams,
TlsCertificates: envoyCerts,
AlpnProtocols: []string{"h2"}, // gRPC requires HTTP/2
},
}
filterChain.TransportSocket = newDownstreamTLSTransportSocket(tlsContext)
return li, nil
}
func (b *Builder) buildGRPCHTTPConnectionManagerFilter() *envoy_config_listener_v3.Filter {
allow := []string{
"envoy.service.auth.v3.Authorization",
"databroker.DataBrokerService",
"registry.Registry",
"grpc.health.v1.Health",
}
routes := make([]*envoy_config_route_v3.Route, 0, len(allow))
for _, svc := range allow {
routes = append(routes, &envoy_config_route_v3.Route{
Name: "grpc",
Match: &envoy_config_route_v3.RouteMatch{
PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: fmt.Sprintf("/%s/", svc)},
Grpc: &envoy_config_route_v3.RouteMatch_GrpcRouteMatchOptions{},
},
Action: &envoy_config_route_v3.Route_Route{
Route: &envoy_config_route_v3.RouteAction{
ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
Cluster: "pomerium-control-plane-grpc",
},
// disable the timeout to support grpc streaming
Timeout: &durationpb.Duration{
Seconds: 0,
},
IdleTimeout: &durationpb.Duration{
Seconds: 0,
},
},
},
})
}
rc := newRouteConfiguration("grpc", []*envoy_config_route_v3.VirtualHost{{
Name: "grpc",
Domains: []string{"*"},
Routes: routes,
}})
return HTTPConnectionManagerFilter(&envoy_http_connection_manager.HttpConnectionManager{
CodecType: envoy_http_connection_manager.HttpConnectionManager_AUTO,
StatPrefix: "grpc_ingress",
// limit request first byte to last byte time
RequestTimeout: &durationpb.Duration{
Seconds: 15,
},
RouteSpecifier: &envoy_http_connection_manager.HttpConnectionManager_RouteConfig{
RouteConfig: rc,
},
HttpFilters: []*envoy_http_connection_manager.HttpFilter{
HTTPRouterFilter(),
},
})
}
func shouldStartGRPCListener(options *config.Options) bool {
if options.GetGRPCAddr() == "" {
return false
}
return config.IsAuthorize(options.Services) || config.IsDataBroker(options.Services)
}

216
config/envoyconfig/listeners_main.go Normal file
View file

@ -0,0 +1,216 @@
package envoyconfig
import (
"context"
"time"
envoy_config_accesslog_v3 "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3"
envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
envoy_config_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
envoy_extensions_access_loggers_grpc_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/grpc/v3"
envoy_extensions_filters_network_http_connection_manager "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3"
"google.golang.org/protobuf/types/known/durationpb"
"google.golang.org/protobuf/types/known/wrapperspb"
"github.com/pomerium/pomerium/config"
)
func (b *Builder) buildMainListener(
ctx context.Context,
cfg *config.Config,
fullyStatic bool,
) (*envoy_config_listener_v3.Listener, error) {
if cfg.Options.InsecureServer {
return b.buildMainInsecureListener(ctx, cfg, fullyStatic)
}
return b.buildMainTLSListener(ctx, cfg, fullyStatic)
}
func (b *Builder) buildMainInsecureListener(
ctx context.Context,
cfg *config.Config,
fullyStatic bool,
) (*envoy_config_listener_v3.Listener, error) {
li := newListener("http-ingress")
li.Address = buildAddress(cfg.Options.Addr, 80)
// listener filters
if cfg.Options.UseProxyProtocol {
li.ListenerFilters = append(li.ListenerFilters, ProxyProtocolFilter())
}
filterChain, err := b.buildMainHTTPConnectionManagerFilterChain(ctx, cfg, fullyStatic, nil)
if err != nil {
return nil, err
}
li.FilterChains = append(li.FilterChains, filterChain)
return li, nil
}
func (b *Builder) buildMainTLSListener(
ctx context.Context,
cfg *config.Config,
fullyStatic bool,
) (*envoy_config_listener_v3.Listener, error) {
li := newListener("https-ingress")
li.Address = buildAddress(cfg.Options.Addr, 443)
// listener filters
if cfg.Options.UseProxyProtocol {
li.ListenerFilters = append(li.ListenerFilters, ProxyProtocolFilter())
}
li.ListenerFilters = append(li.ListenerFilters, TLSInspectorFilter())
// access log
if cfg.Options.DownstreamMTLS.Enforcement == config.MTLSEnforcementRejectConnection {
li.AccessLog = append(li.AccessLog, newListenerAccessLog())
}
// filter chains
li.FilterChains = append(li.FilterChains, b.buildACMETLSALPNFilterChain())
allCertificates, err := getAllCertificates(cfg)
if err != nil {
return nil, err
}
tlsContext, err := b.buildDownstreamTLSContextMulti(ctx, cfg, allCertificates)
if err != nil {
return nil, err
}
filterChain, err := b.buildMainHTTPConnectionManagerFilterChain(ctx, cfg, fullyStatic, newDownstreamTLSTransportSocket(tlsContext))
if err != nil {
return nil, err
}
li.FilterChains = append(li.FilterChains, filterChain)
return li, nil
}
func (b *Builder) buildMainHTTPConnectionManagerFilterChain(
ctx context.Context,
cfg *config.Config,
fullyStatic bool,
transportSocket *envoy_config_core_v3.TransportSocket,
) (*envoy_config_listener_v3.FilterChain, error) {
filter, err := b.buildMainHTTPConnectionManagerFilter(ctx, cfg, fullyStatic)
if err != nil {
return nil, err
}
return &envoy_config_listener_v3.FilterChain{
Filters: []*envoy_config_listener_v3.Filter{filter},
TransportSocket: transportSocket,
}, nil
}
func (b *Builder) buildMainHTTPConnectionManagerFilter(
ctx context.Context,
cfg *config.Config,
fullyStatic bool,
) (*envoy_config_listener_v3.Filter, error) {
var grpcClientTimeout *durationpb.Duration
if cfg.Options.GRPCClientTimeout != 0 {
grpcClientTimeout = durationpb.New(cfg.Options.GRPCClientTimeout)
} else {
grpcClientTimeout = durationpb.New(30 * time.Second)
}
filters := []*envoy_extensions_filters_network_http_connection_manager.HttpFilter{
LuaFilter(luascripts.RemoveImpersonateHeaders),
LuaFilter(luascripts.SetClientCertificateMetadata),
ExtAuthzFilter(grpcClientTimeout),
LuaFilter(luascripts.ExtAuthzSetCookie),
LuaFilter(luascripts.CleanUpstream),
LuaFilter(luascripts.RewriteHeaders),
}
filters = append(filters, HTTPRouterFilter())
var maxStreamDuration *durationpb.Duration
if cfg.Options.WriteTimeout > 0 {
maxStreamDuration = durationpb.New(cfg.Options.WriteTimeout)
}
tracingProvider, err := buildTracingHTTP(cfg.Options)
if err != nil {
return nil, err
}
localReply, err := b.buildLocalReplyConfig(cfg.Options)
if err != nil {
return nil, err
}
mgr := &envoy_extensions_filters_network_http_connection_manager.HttpConnectionManager{
AlwaysSetRequestIdInResponse: true,
CodecType: cfg.Options.GetCodecType().ToEnvoy(),
StatPrefix: "ingress",
HttpFilters: filters,
AccessLog: buildAccessLogs(cfg.Options),
CommonHttpProtocolOptions: &envoy_config_core_v3.HttpProtocolOptions{
IdleTimeout: durationpb.New(cfg.Options.IdleTimeout),
MaxStreamDuration: maxStreamDuration,
},
HttpProtocolOptions: http1ProtocolOptions,
RequestTimeout: durationpb.New(cfg.Options.ReadTimeout),
Tracing: &envoy_extensions_filters_network_http_connection_manager.HttpConnectionManager_Tracing{
RandomSampling: &envoy_type_v3.Percent{Value: cfg.Options.TracingSampleRate * 100},
Provider: tracingProvider,
},
// See https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for
UseRemoteAddress: &wrapperspb.BoolValue{Value: true},
SkipXffAppend: cfg.Options.SkipXffAppend,
XffNumTrustedHops: cfg.Options.XffNumTrustedHops,
LocalReplyConfig: localReply,
NormalizePath: wrapperspb.Bool(true),
}
if fullyStatic {
routeConfiguration, err := b.buildMainRouteConfiguration(ctx, cfg)
if err != nil {
return nil, err
}
mgr.RouteSpecifier = &envoy_extensions_filters_network_http_connection_manager.HttpConnectionManager_RouteConfig{
RouteConfig: routeConfiguration,
}
} else {
mgr.RouteSpecifier = &envoy_extensions_filters_network_http_connection_manager.HttpConnectionManager_Rds{
Rds: &envoy_extensions_filters_network_http_connection_manager.Rds{
ConfigSource: &envoy_config_core_v3.ConfigSource{
ResourceApiVersion: envoy_config_core_v3.ApiVersion_V3,
ConfigSourceSpecifier: &envoy_config_core_v3.ConfigSource_Ads{},
},
RouteConfigName: "main",
},
}
}
return HTTPConnectionManagerFilter(mgr), nil
}
func newListenerAccessLog() *envoy_config_accesslog_v3.AccessLog {
return &envoy_config_accesslog_v3.AccessLog{
Name: "envoy.access_loggers.tcp_grpc",
ConfigType: &envoy_config_accesslog_v3.AccessLog_TypedConfig{
TypedConfig: marshalAny(&envoy_extensions_access_loggers_grpc_v3.TcpGrpcAccessLogConfig{
CommonConfig: &envoy_extensions_access_loggers_grpc_v3.CommonGrpcAccessLogConfig{
LogName: "ingress-http-listener",
GrpcService: &envoy_config_core_v3.GrpcService{
TargetSpecifier: &envoy_config_core_v3.GrpcService_EnvoyGrpc_{
EnvoyGrpc: &envoy_config_core_v3.GrpcService_EnvoyGrpc{
ClusterName: "pomerium-control-plane-grpc",
},
},
},
TransportApiVersion: envoy_config_core_v3.ApiVersion_V3,
},
}),
},
}
}
func shouldStartMainListener(options *config.Options) bool {
return config.IsAuthenticate(options.Services) || config.IsProxy(options.Services)
}

39
config/envoyconfig/listeners_main_test.go Normal file
View file

@ -0,0 +1,39 @@
package envoyconfig
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/pomerium/pomerium/config"
"github.com/pomerium/pomerium/internal/testutil"
)
func Test_requireProxyProtocol(t *testing.T) {
b := New("local-grpc", "local-http", "local-metrics", nil, nil)
t.Run("required", func(t *testing.T) {
li, err := b.buildMainListener(context.Background(), &config.Config{Options: &config.Options{
UseProxyProtocol: true,
InsecureServer: true,
}}, false)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `[
{
"name": "envoy.filters.listener.proxy_protocol",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.listener.proxy_protocol.v3.ProxyProtocol"
}
}
]`, li.GetListenerFilters())
})
t.Run("not required", func(t *testing.T) {
li, err := b.buildMainListener(context.Background(), &config.Config{Options: &config.Options{
UseProxyProtocol: false,
InsecureServer: true,
}}, false)
require.NoError(t, err)
assert.Len(t, li.GetListenerFilters(), 0)
})
}

139
config/envoyconfig/listeners_metrics.go Normal file
View file

@ -0,0 +1,139 @@
package envoyconfig
import (
"context"
"encoding/base64"
"fmt"
"net"
envoy_config_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
envoy_config_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
envoy_http_connection_manager "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
envoy_extensions_transport_sockets_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
"google.golang.org/protobuf/types/known/wrapperspb"
"github.com/pomerium/pomerium/config"
"github.com/pomerium/pomerium/internal/hashutil"
"github.com/pomerium/pomerium/internal/telemetry/metrics"
)
func (b *Builder) buildMetricsListener(cfg *config.Config) (*envoy_config_listener_v3.Listener, error) {
filter := b.buildMetricsHTTPConnectionManagerFilter()
filterChain := &envoy_config_listener_v3.FilterChain{
Filters: []*envoy_config_listener_v3.Filter{
filter,
},
}
cert, err := cfg.Options.GetMetricsCertificate()
if err != nil {
return nil, err
}
if cert != nil {
dtc := &envoy_extensions_transport_sockets_tls_v3.DownstreamTlsContext{
CommonTlsContext: &envoy_extensions_transport_sockets_tls_v3.CommonTlsContext{
TlsParams: tlsDownstreamParams,
TlsCertificates: []*envoy_extensions_transport_sockets_tls_v3.TlsCertificate{
b.envoyTLSCertificateFromGoTLSCertificate(context.TODO(), cert),
},
AlpnProtocols: []string{"h2", "http/1.1"},
},
}
if cfg.Options.MetricsClientCA != "" {
bs, err := base64.StdEncoding.DecodeString(cfg.Options.MetricsClientCA)
if err != nil {
return nil, fmt.Errorf("xds: invalid metrics_client_ca: %w", err)
}
dtc.RequireClientCertificate = wrapperspb.Bool(true)
dtc.CommonTlsContext.ValidationContextType = &envoy_extensions_transport_sockets_tls_v3.CommonTlsContext_ValidationContext{
ValidationContext: &envoy_extensions_transport_sockets_tls_v3.CertificateValidationContext{
TrustChainVerification: envoy_extensions_transport_sockets_tls_v3.CertificateValidationContext_VERIFY_TRUST_CHAIN,
TrustedCa: b.filemgr.BytesDataSource("metrics_client_ca.pem", bs),
},
}
} else if cfg.Options.MetricsClientCAFile != "" {
dtc.RequireClientCertificate = wrapperspb.Bool(true)
dtc.CommonTlsContext.ValidationContextType = &envoy_extensions_transport_sockets_tls_v3.CommonTlsContext_ValidationContext{
ValidationContext: &envoy_extensions_transport_sockets_tls_v3.CertificateValidationContext{
TrustChainVerification: envoy_extensions_transport_sockets_tls_v3.CertificateValidationContext_VERIFY_TRUST_CHAIN,
TrustedCa: b.filemgr.FileDataSource(cfg.Options.MetricsClientCAFile),
},
}
}
filterChain.TransportSocket = newDownstreamTLSTransportSocket(dtc)
}
// we ignore the host part of the address, only binding to
host, port, err := net.SplitHostPort(cfg.Options.MetricsAddr)
if err != nil {
return nil, fmt.Errorf("metrics_addr %s: %w", cfg.Options.MetricsAddr, err)
}
if port == "" {
return nil, fmt.Errorf("metrics_addr %s: port is required", cfg.Options.MetricsAddr)
}
// unless an explicit IP address was provided, and bind to all interfaces if hostname was provided
if net.ParseIP(host) == nil {
host = ""
}
addr := buildAddress(net.JoinHostPort(host, port), 9902)
li := newListener(fmt.Sprintf("metrics-ingress-%d", hashutil.MustHash(addr)))
li.Address = addr
li.FilterChains = []*envoy_config_listener_v3.FilterChain{filterChain}
return li, nil
}
func (b *Builder) buildMetricsHTTPConnectionManagerFilter() *envoy_config_listener_v3.Filter {
rc := newRouteConfiguration("metrics", []*envoy_config_route_v3.VirtualHost{{
Name: "metrics",
Domains: []string{"*"},
Routes: []*envoy_config_route_v3.Route{
{
Name: "envoy-metrics",
Match: &envoy_config_route_v3.RouteMatch{
PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: metrics.EnvoyMetricsPath},
},
Action: &envoy_config_route_v3.Route_Route{
Route: &envoy_config_route_v3.RouteAction{
ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
Cluster: envoyAdminClusterName,
},
PrefixRewrite: "/stats/prometheus",
},
},
},
{
Name: "metrics",
Match: &envoy_config_route_v3.RouteMatch{
PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: "/"},
},
Action: &envoy_config_route_v3.Route_Route{
Route: &envoy_config_route_v3.RouteAction{
ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
Cluster: "pomerium-control-plane-metrics",
},
},
},
},
},
}})
return HTTPConnectionManagerFilter(&envoy_http_connection_manager.HttpConnectionManager{
CodecType: envoy_http_connection_manager.HttpConnectionManager_AUTO,
StatPrefix: "metrics",
RouteSpecifier: &envoy_http_connection_manager.HttpConnectionManager_RouteConfig{
RouteConfig: rc,
},
HttpFilters: []*envoy_http_connection_manager.HttpFilter{
HTTPRouterFilter(),
},
})
}
func shouldStartMetricsListener(options *config.Options) bool {
return options.MetricsAddr != ""
}

471
config/envoyconfig/listeners_test.go
View file

@ -4,21 +4,18 @@ import (
"bytes"
"context"
"embed"
"encoding/base64"
"os"
"path/filepath"
"runtime"
"testing"
"text/template"
envoy_config_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/pomerium/pomerium/config"
"github.com/pomerium/pomerium/config/envoyconfig/filemgr"
"github.com/pomerium/pomerium/internal/testutil"
"github.com/pomerium/pomerium/pkg/cryptutil"
)
const (
@ -109,471 +106,3 @@ func Test_buildMainHTTPConnectionManagerFilter(t *testing.T) {
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, testData(t, "main_http_connection_manager_filter.json", nil), filter)
}
func Test_buildDownstreamTLSContext(t *testing.T) {
b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
cacheDir, _ := os.UserCacheDir()
clientCAFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "client-ca-313754424855313435355a5348.pem")
t.Run("no-validation", func(t *testing.T) {
downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), &config.Config{Options: &config.Options{}}, nil)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `{
"commonTlsContext": {
"tlsParams": {
"cipherSuites": [
"ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-CHACHA20-POLY1305",
"ECDHE-RSA-CHACHA20-POLY1305"
],
"tlsMinimumProtocolVersion": "TLSv1_2",
"tlsMaximumProtocolVersion": "TLSv1_3"
},
"alpnProtocols": ["h2", "http/1.1"]
}
}`, downstreamTLSContext)
})
t.Run("client-ca", func(t *testing.T) {
downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), &config.Config{Options: &config.Options{
DownstreamMTLS: config.DownstreamMTLSSettings{
CA: "VEVTVAo=", // "TEST\n" (with a trailing newline)
},
}}, nil)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `{
"commonTlsContext": {
"tlsParams": {
"cipherSuites": [
"ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-CHACHA20-POLY1305",
"ECDHE-RSA-CHACHA20-POLY1305"
],
"tlsMinimumProtocolVersion": "TLSv1_2",
"tlsMaximumProtocolVersion": "TLSv1_3"
},
"alpnProtocols": ["h2", "http/1.1"],
"validationContext": {
"maxVerifyDepth": 1,
"onlyVerifyLeafCertCrl": true,
"trustChainVerification": "ACCEPT_UNTRUSTED",
"trustedCa": {
"filename": "`+clientCAFileName+`"
}
}
}
}`, downstreamTLSContext)
})
t.Run("client-ca-strict", func(t *testing.T) {
downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), &config.Config{Options: &config.Options{
DownstreamMTLS: config.DownstreamMTLSSettings{
CA: "VEVTVAo=", // "TEST\n" (with a trailing newline)
Enforcement: config.MTLSEnforcementRejectConnection,
},
}}, nil)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `{
"commonTlsContext": {
"tlsParams": {
"cipherSuites": [
"ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-CHACHA20-POLY1305",
"ECDHE-RSA-CHACHA20-POLY1305"
],
"tlsMinimumProtocolVersion": "TLSv1_2",
"tlsMaximumProtocolVersion": "TLSv1_3"
},
"alpnProtocols": ["h2", "http/1.1"],
"validationContext": {
"maxVerifyDepth": 1,
"onlyVerifyLeafCertCrl": true,
"trustedCa": {
"filename": "`+clientCAFileName+`"
}
}
},
"requireClientCertificate": true
}`, downstreamTLSContext)
})
t.Run("policy-client-ca", func(t *testing.T) {
downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), &config.Config{Options: &config.Options{
Policies: []config.Policy{
{
From: "https://a.example.com:1234",
TLSDownstreamClientCA: "VEVTVA==", // "TEST" (no trailing newline)
},
},
}}, nil)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `{
"commonTlsContext": {
"tlsParams": {
"cipherSuites": [
"ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-CHACHA20-POLY1305",
"ECDHE-RSA-CHACHA20-POLY1305"
],
"tlsMinimumProtocolVersion": "TLSv1_2",
"tlsMaximumProtocolVersion": "TLSv1_3"
},
"alpnProtocols": ["h2", "http/1.1"],
"validationContext": {
"maxVerifyDepth": 1,
"onlyVerifyLeafCertCrl": true,
"trustChainVerification": "ACCEPT_UNTRUSTED",
"trustedCa": {
"filename": "`+clientCAFileName+`"
}
}
}
}`, downstreamTLSContext)
})
t.Run("client-ca-max-verify-depth", func(t *testing.T) {
var maxVerifyDepth uint32
config := &config.Config{Options: &config.Options{
DownstreamMTLS: config.DownstreamMTLSSettings{
MaxVerifyDepth: &maxVerifyDepth,
CA: "VEVTVAo=", // "TEST\n"
},
}}
maxVerifyDepth = 10
downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), config, nil)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `{
"maxVerifyDepth": 10,
"onlyVerifyLeafCertCrl": true,
"trustChainVerification": "ACCEPT_UNTRUSTED",
"trustedCa": {
"filename": "`+clientCAFileName+`"
}
}`, downstreamTLSContext.GetCommonTlsContext().GetValidationContext())
maxVerifyDepth = 0
downstreamTLSContext, err = b.buildDownstreamTLSContextMulti(context.Background(), config, nil)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `{
"onlyVerifyLeafCertCrl": true,
"trustChainVerification": "ACCEPT_UNTRUSTED",
"trustedCa": {
"filename": "`+clientCAFileName+`"
}
}`, downstreamTLSContext.GetCommonTlsContext().GetValidationContext())
})
t.Run("client-ca-san-matchers", func(t *testing.T) {
config := &config.Config{Options: &config.Options{
DownstreamMTLS: config.DownstreamMTLSSettings{
CA: "VEVTVAo=", // "TEST\n"
MatchSubjectAltNames: []config.SANMatcher{
{Type: config.SANTypeDNS, Pattern: `.*\.corp\.example\.com`},
{Type: config.SANTypeEmail, Pattern: `.*@example\.com`},
{Type: config.SANTypeIPAddress, Pattern: `10\.10\.42\..*`},
{Type: config.SANTypeURI, Pattern: `spiffe://example\.com/.*`},
{Type: config.SANTypeUserPrincipalName, Pattern: `^device-id$`},
},
},
}}
downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), config, nil)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `{
"maxVerifyDepth": 1,
"matchTypedSubjectAltNames": [
{
"matcher": {
"safeRegex": {
"googleRe2": {},
"regex": ".*\\.corp\\.example\\.com"
}
},
"sanType": "DNS"
},
{
"matcher": {
"safeRegex": {
"googleRe2": {},
"regex": ".*@example\\.com"
}
},
"sanType": "EMAIL"
},
{
"matcher": {
"safeRegex": {
"googleRe2": {},
"regex": "10\\.10\\.42\\..*"
}
},
"sanType": "IP_ADDRESS"
},
{
"matcher": {
"safeRegex": {
"googleRe2": {},
"regex": "spiffe://example\\.com/.*"
}
},
"sanType": "URI"
},
{
"matcher": {
"safeRegex": {
"googleRe2": {},
"regex": "^device-id$"
}
},
"sanType": "OTHER_NAME",
"oid": "1.3.6.1.4.1.311.20.2.3"
}
],
"onlyVerifyLeafCertCrl": true,
"trustChainVerification": "ACCEPT_UNTRUSTED",
"trustedCa": {
"filename": "`+clientCAFileName+`"
}
}`, downstreamTLSContext.GetCommonTlsContext().GetValidationContext())
})
t.Run("http1", func(t *testing.T) {
downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), &config.Config{Options: &config.Options{
Cert: aExampleComCert,
Key: aExampleComKey,
CodecType: config.CodecTypeHTTP1,
}}, nil)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `{
"commonTlsContext": {
"tlsParams": {
"cipherSuites": [
"ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-CHACHA20-POLY1305",
"ECDHE-RSA-CHACHA20-POLY1305"
],
"tlsMinimumProtocolVersion": "TLSv1_2",
"tlsMaximumProtocolVersion": "TLSv1_3"
},
"alpnProtocols": ["http/1.1"]
}
}`, downstreamTLSContext)
})
t.Run("http2", func(t *testing.T) {
downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), &config.Config{Options: &config.Options{
Cert: aExampleComCert,
Key: aExampleComKey,
CodecType: config.CodecTypeHTTP2,
}}, nil)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `{
"commonTlsContext": {
"tlsParams": {
"cipherSuites": [
"ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-CHACHA20-POLY1305",
"ECDHE-RSA-CHACHA20-POLY1305"
],
"tlsMinimumProtocolVersion": "TLSv1_2",
"tlsMaximumProtocolVersion": "TLSv1_3"
},
"alpnProtocols": ["h2"]
}
}`, downstreamTLSContext)
})
}
func Test_clientCABundle(t *testing.T) {
// Make sure multiple bundled CAs are separated by newlines.
clientCA1 := []byte("client CA 1")
clientCA2 := []byte("client CA 2")
clientCA3 := []byte("client CA 3")
b64 := base64.StdEncoding.EncodeToString
cfg := &config.Config{Options: &config.Options{
DownstreamMTLS: config.DownstreamMTLSSettings{
CA: b64(clientCA3),
},
Policies: []config.Policy{
{
From: "https://foo.example.com",
TLSDownstreamClientCA: b64(clientCA2),
},
{
From: "https://bar.example.com",
TLSDownstreamClientCA: b64(clientCA1),
},
},
}}
expected := []byte("client CA 3\nclient CA 2\nclient CA 1\n")
actual := clientCABundle(context.Background(), cfg)
assert.Equal(t, expected, actual)
}
func Test_getAllDomains(t *testing.T) {
cert, err := cryptutil.GenerateCertificate(nil, "*.unknown.example.com")
require.NoError(t, err)
certPEM, keyPEM, err := cryptutil.EncodeCertificate(cert)
require.NoError(t, err)
options := &config.Options{
Addr: "127.0.0.1:9000",
GRPCAddr: "127.0.0.1:9001",
Services: "all",
AuthenticateURLString: "https://authenticate.example.com",
AuthenticateInternalURLString: "https://authenticate.int.example.com",
AuthorizeURLString: "https://authorize.example.com:9001",
DataBrokerURLString: "https://cache.example.com:9001",
Policies: []config.Policy{
{From: "http://a.example.com"},
{From: "https://b.example.com"},
{From: "https://c.example.com"},
{From: "https://d.unknown.example.com"},
},
Cert: base64.StdEncoding.EncodeToString(certPEM),
Key: base64.StdEncoding.EncodeToString(keyPEM),
}
t.Run("routable", func(t *testing.T) {
t.Run("http", func(t *testing.T) {
actual, err := getAllRouteableHosts(options, "127.0.0.1:9000")
require.NoError(t, err)
expect := []string{
"a.example.com",
"a.example.com:80",
"authenticate.example.com",
"authenticate.example.com:443",
"authenticate.int.example.com",
"authenticate.int.example.com:443",
"b.example.com",
"b.example.com:443",
"c.example.com",
"c.example.com:443",
"d.unknown.example.com",
"d.unknown.example.com:443",
}
assert.Equal(t, expect, actual)
})
t.Run("grpc", func(t *testing.T) {
actual, err := getAllRouteableHosts(options, "127.0.0.1:9001")
require.NoError(t, err)
expect := []string{
"authorize.example.com:9001",
"cache.example.com:9001",
}
assert.Equal(t, expect, actual)
})
t.Run("both", func(t *testing.T) {
newOptions := *options
newOptions.GRPCAddr = newOptions.Addr
actual, err := getAllRouteableHosts(&newOptions, "127.0.0.1:9000")
require.NoError(t, err)
expect := []string{
"a.example.com",
"a.example.com:80",
"authenticate.example.com",
"authenticate.example.com:443",
"authenticate.int.example.com",
"authenticate.int.example.com:443",
"authorize.example.com:9001",
"b.example.com",
"b.example.com:443",
"c.example.com",
"c.example.com:443",
"cache.example.com:9001",
"d.unknown.example.com",
"d.unknown.example.com:443",
}
assert.Equal(t, expect, actual)
})
})
t.Run("exclude default authenticate", func(t *testing.T) {
options := config.NewDefaultOptions()
options.Policies = []config.Policy{
{From: "https://a.example.com"},
}
actual, err := getAllRouteableHosts(options, ":443")
require.NoError(t, err)
assert.Equal(t, []string{"a.example.com"}, actual)
})
}
func Test_urlMatchesHost(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
name string
sourceURL string
host string
matches bool
}{
{"no port", "http://example.com", "example.com", true},
{"host http port", "http://example.com", "example.com:80", true},
{"host https port", "https://example.com", "example.com:443", true},
{"with port", "https://example.com:443", "example.com:443", true},
{"url port", "https://example.com:443", "example.com", true},
{"non standard port", "http://example.com:81", "example.com", false},
{"non standard host port", "http://example.com:81", "example.com:80", false},
} {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
assert.Equal(t, tc.matches, urlMatchesHost(mustParseURL(t, tc.sourceURL), tc.host),
"urlMatchesHost(%s,%s)", tc.sourceURL, tc.host)
})
}
}
func Test_buildRouteConfiguration(t *testing.T) {
b := New("local-grpc", "local-http", "local-metrics", nil, nil)
virtualHosts := make([]*envoy_config_route_v3.VirtualHost, 10)
routeConfig, err := b.buildRouteConfiguration("test-route-configuration", virtualHosts)
require.NoError(t, err)
assert.Equal(t, "test-route-configuration", routeConfig.GetName())
assert.Equal(t, virtualHosts, routeConfig.GetVirtualHosts())
assert.False(t, routeConfig.GetValidateClusters().GetValue())
}
func Test_requireProxyProtocol(t *testing.T) {
b := New("local-grpc", "local-http", "local-metrics", nil, nil)
t.Run("required", func(t *testing.T) {
li, err := b.buildMainListener(context.Background(), &config.Config{Options: &config.Options{
UseProxyProtocol: true,
InsecureServer: true,
}}, false)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `[
{
"name": "envoy.filters.listener.proxy_protocol",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.listener.proxy_protocol.v3.ProxyProtocol"
}
}
]`, li.GetListenerFilters())
})
t.Run("not required", func(t *testing.T) {
li, err := b.buildMainListener(context.Background(), &config.Config{Options: &config.Options{
UseProxyProtocol: false,
InsecureServer: true,
}}, false)
require.NoError(t, err)
assert.Len(t, li.GetListenerFilters(), 0)
})
}

20
config/envoyconfig/outbound.go
View file

@ -20,12 +20,9 @@ func (b *Builder) buildOutboundListener(cfg *config.Config) (*envoy_config_liste
return nil, fmt.Errorf("invalid outbound port %v: %w", cfg.OutboundPort, err)
}
filter, err := b.buildOutboundHTTPConnectionManager()
if err != nil {
return nil, fmt.Errorf("error building outbound http connection manager filter: %w", err)
}
filter := b.buildOutboundHTTPConnectionManager()
li := newEnvoyListener("outbound-ingress")
li := newListener("outbound-ingress")
li.Address = &envoy_config_core_v3.Address{
Address: &envoy_config_core_v3.Address_SocketAddress{
SocketAddress: &envoy_config_core_v3.SocketAddress{
@ -43,11 +40,8 @@ func (b *Builder) buildOutboundListener(cfg *config.Config) (*envoy_config_liste
return li, nil
}
func (b *Builder) buildOutboundHTTPConnectionManager() (*envoy_config_listener_v3.Filter, error) {
rc, err := b.buildOutboundRouteConfiguration()
if err != nil {
return nil, err
}
func (b *Builder) buildOutboundHTTPConnectionManager() *envoy_config_listener_v3.Filter {
rc := b.buildOutboundRouteConfiguration()
tc := marshalAny(&envoy_http_connection_manager.HttpConnectionManager{
CodecType: envoy_http_connection_manager.HttpConnectionManager_AUTO,
@ -69,11 +63,11 @@ func (b *Builder) buildOutboundHTTPConnectionManager() (*envoy_config_listener_v
ConfigType: &envoy_config_listener_v3.Filter_TypedConfig{
TypedConfig: tc,
},
}, nil
}
}
func (b *Builder) buildOutboundRouteConfiguration() (*envoy_config_route_v3.RouteConfiguration, error) {
return b.buildRouteConfiguration("grpc", []*envoy_config_route_v3.VirtualHost{{
func (b *Builder) buildOutboundRouteConfiguration() *envoy_config_route_v3.RouteConfiguration {
return newRouteConfiguration("grpc", []*envoy_config_route_v3.VirtualHost{{
Name: "grpc",
Domains: []string{"*"},
Routes: b.buildOutboundRoutes(),

67
config/envoyconfig/route_configurations.go
View file

@ -1,12 +1,18 @@
package envoyconfig
import (
"cmp"
"context"
"net/url"
"strings"
envoy_config_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
"github.com/hashicorp/go-set/v3"
"google.golang.org/protobuf/types/known/wrapperspb"
"github.com/pomerium/pomerium/config"
"github.com/pomerium/pomerium/internal/telemetry/trace"
"github.com/pomerium/pomerium/internal/urlutil"
)
// BuildRouteConfigurations builds the route configurations for the RDS service.
@ -96,10 +102,61 @@ func (b *Builder) buildMainRouteConfiguration(
virtualHosts = append(virtualHosts, vh)
rc, err := b.buildRouteConfiguration("main", virtualHosts)
if err != nil {
return nil, err
}
rc := newRouteConfiguration("main", virtualHosts)
return rc, nil
}
func getAllRouteableHosts(options *config.Options, addr string) ([]string, error) {
allHosts := set.NewTreeSet(cmp.Compare[string])
if addr == options.Addr {
hosts, err := options.GetAllRouteableHTTPHosts()
if err != nil {
return nil, err
}
allHosts.InsertSlice(hosts)
}
if addr == options.GetGRPCAddr() {
hosts, err := options.GetAllRouteableGRPCHosts()
if err != nil {
return nil, err
}
allHosts.InsertSlice(hosts)
}
var filtered []string
for host := range allHosts.Items() {
if !strings.Contains(host, "*") {
filtered = append(filtered, host)
}
}
return filtered, nil
}
func newRouteConfiguration(name string, virtualHosts []*envoy_config_route_v3.VirtualHost) *envoy_config_route_v3.RouteConfiguration {
return &envoy_config_route_v3.RouteConfiguration{
Name: name,
VirtualHosts: virtualHosts,
// disable cluster validation since the order of LDS/CDS updates isn't guaranteed
ValidateClusters: &wrapperspb.BoolValue{Value: false},
}
}
func urlsMatchHost(urls []*url.URL, host string) bool {
for _, u := range urls {
if urlMatchesHost(u, host) {
return true
}
}
return false
}
func urlMatchesHost(u *url.URL, host string) bool {
for _, h := range urlutil.GetDomainsForURL(u, true) {
if h == host {
return true
}
}
return false
}

117
config/envoyconfig/route_configurations_test.go
View file

@ -2,10 +2,12 @@ package envoyconfig
import (
"context"
"encoding/base64"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/encoding/protojson"
"github.com/pomerium/pomerium/config"
@ -159,3 +161,118 @@ func TestBuilder_buildMainRouteConfiguration(t *testing.T) {
}`, routeConfiguration)
}
func Test_getAllDomains(t *testing.T) {
cert, err := cryptutil.GenerateCertificate(nil, "*.unknown.example.com")
require.NoError(t, err)
certPEM, keyPEM, err := cryptutil.EncodeCertificate(cert)
require.NoError(t, err)
options := &config.Options{
Addr: "127.0.0.1:9000",
GRPCAddr: "127.0.0.1:9001",
Services: "all",
AuthenticateURLString: "https://authenticate.example.com",
AuthenticateInternalURLString: "https://authenticate.int.example.com",
AuthorizeURLString: "https://authorize.example.com:9001",
DataBrokerURLString: "https://cache.example.com:9001",
Policies: []config.Policy{
{From: "http://a.example.com"},
{From: "https://b.example.com"},
{From: "https://c.example.com"},
{From: "https://d.unknown.example.com"},
},
Cert: base64.StdEncoding.EncodeToString(certPEM),
Key: base64.StdEncoding.EncodeToString(keyPEM),
}
t.Run("routable", func(t *testing.T) {
t.Run("http", func(t *testing.T) {
actual, err := getAllRouteableHosts(options, "127.0.0.1:9000")
require.NoError(t, err)
expect := []string{
"a.example.com",
"a.example.com:80",
"authenticate.example.com",
"authenticate.example.com:443",
"authenticate.int.example.com",
"authenticate.int.example.com:443",
"b.example.com",
"b.example.com:443",
"c.example.com",
"c.example.com:443",
"d.unknown.example.com",
"d.unknown.example.com:443",
}
assert.Equal(t, expect, actual)
})
t.Run("grpc", func(t *testing.T) {
actual, err := getAllRouteableHosts(options, "127.0.0.1:9001")
require.NoError(t, err)
expect := []string{
"authorize.example.com:9001",
"cache.example.com:9001",
}
assert.Equal(t, expect, actual)
})
t.Run("both", func(t *testing.T) {
newOptions := *options
newOptions.GRPCAddr = newOptions.Addr
actual, err := getAllRouteableHosts(&newOptions, "127.0.0.1:9000")
require.NoError(t, err)
expect := []string{
"a.example.com",
"a.example.com:80",
"authenticate.example.com",
"authenticate.example.com:443",
"authenticate.int.example.com",
"authenticate.int.example.com:443",
"authorize.example.com:9001",
"b.example.com",
"b.example.com:443",
"c.example.com",
"c.example.com:443",
"cache.example.com:9001",
"d.unknown.example.com",
"d.unknown.example.com:443",
}
assert.Equal(t, expect, actual)
})
})
t.Run("exclude default authenticate", func(t *testing.T) {
options := config.NewDefaultOptions()
options.Policies = []config.Policy{
{From: "https://a.example.com"},
}
actual, err := getAllRouteableHosts(options, ":443")
require.NoError(t, err)
assert.Equal(t, []string{"a.example.com"}, actual)
})
}
func Test_urlMatchesHost(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
name string
sourceURL string
host string
matches bool
}{
{"no port", "http://example.com", "example.com", true},
{"host http port", "http://example.com", "example.com:80", true},
{"host https port", "https://example.com", "example.com:443", true},
{"with port", "https://example.com:443", "example.com:443", true},
{"url port", "https://example.com:443", "example.com", true},
{"non standard port", "http://example.com:81", "example.com", false},
{"non standard host port", "http://example.com:81", "example.com:80", false},
} {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
assert.Equal(t, tc.matches, urlMatchesHost(mustParseURL(t, tc.sourceURL), tc.host),
"urlMatchesHost(%s,%s)", tc.sourceURL, tc.host)
})
}
}

198
config/envoyconfig/tls.go
View file

@ -1,17 +1,26 @@
package envoyconfig
import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"encoding/asn1"
"encoding/base64"
"encoding/pem"
"fmt"
"net/netip"
"net/url"
"regexp"
"strings"
envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
envoy_extensions_transport_sockets_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
envoy_type_matcher_v3 "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
"google.golang.org/protobuf/types/known/wrapperspb"
"github.com/pomerium/pomerium/config"
"github.com/pomerium/pomerium/internal/log"
)
var (
@ -121,6 +130,103 @@ func (b *Builder) buildSubjectNameIndication(
return sni
}
func (b *Builder) envoyTLSCertificateFromGoTLSCertificate(
ctx context.Context,
cert *tls.Certificate,
) *envoy_extensions_transport_sockets_tls_v3.TlsCertificate {
envoyCert := &envoy_extensions_transport_sockets_tls_v3.TlsCertificate{}
var chain bytes.Buffer
for _, cbs := range cert.Certificate {
_ = pem.Encode(&chain, &pem.Block{
Type: "CERTIFICATE",
Bytes: cbs,
})
}
envoyCert.CertificateChain = b.filemgr.BytesDataSource("tls-crt.pem", chain.Bytes())
if cert.OCSPStaple != nil {
envoyCert.OcspStaple = b.filemgr.BytesDataSource("ocsp-staple", cert.OCSPStaple)
}
if bs, err := x509.MarshalPKCS8PrivateKey(cert.PrivateKey); err == nil {
envoyCert.PrivateKey = b.filemgr.BytesDataSource("tls-key.pem", pem.EncodeToMemory(
&pem.Block{
Type: "PRIVATE KEY",
Bytes: bs,
},
))
} else {
log.Ctx(ctx).Error().Err(err).Msg("failed to marshal private key for tls config")
}
for _, scts := range cert.SignedCertificateTimestamps {
envoyCert.SignedCertificateTimestamp = append(envoyCert.SignedCertificateTimestamp,
b.filemgr.BytesDataSource("signed-certificate-timestamp", scts))
}
return envoyCert
}
func (b *Builder) envoyTLSCertificatesFromGoTLSCertificates(ctx context.Context, certs []tls.Certificate) (
[]*envoy_extensions_transport_sockets_tls_v3.TlsCertificate, error,
) {
envoyCerts := make([]*envoy_extensions_transport_sockets_tls_v3.TlsCertificate, 0, len(certs))
for i := range certs {
cert := &certs[i]
if err := validateCertificate(cert); err != nil {
return nil, fmt.Errorf("invalid certificate for domain %s: %w",
cert.Leaf.Subject.CommonName, err)
}
envoyCert := b.envoyTLSCertificateFromGoTLSCertificate(ctx, cert)
envoyCerts = append(envoyCerts, envoyCert)
}
return envoyCerts, nil
}
// clientCABundle returns a bundle of the globally configured client CA and any
// per-route client CAs.
func clientCABundle(ctx context.Context, cfg *config.Config) []byte {
var bundle bytes.Buffer
ca, _ := cfg.Options.DownstreamMTLS.GetCA()
addCAToBundle(&bundle, ca)
for p := range cfg.Options.GetAllPolicies() {
// We don't need to check TLSDownstreamClientCAFile here because
// Policy.Validate() will populate TLSDownstreamClientCA when
// TLSDownstreamClientCAFile is set.
if p.TLSDownstreamClientCA == "" {
continue
}
ca, err := base64.StdEncoding.DecodeString(p.TLSDownstreamClientCA)
if err != nil {
log.Ctx(ctx).Error().Stringer("policy", p).Err(err).Msg("invalid client CA")
continue
}
addCAToBundle(&bundle, ca)
}
return bundle.Bytes()
}
func addCAToBundle(bundle *bytes.Buffer, ca []byte) {
if len(ca) == 0 {
return
}
bundle.Write(ca)
// Make sure each CA is separated by a newline.
if ca[len(ca)-1] != '\n' {
bundle.WriteByte('\n')
}
}
func getAllCertificates(cfg *config.Config) ([]tls.Certificate, error) {
allCertificates, err := cfg.AllCertificates()
if err != nil {
return nil, fmt.Errorf("error collecting all certificates: %w", err)
}
wc, err := cfg.GenerateCatchAllCertificate()
if err != nil {
return nil, fmt.Errorf("error getting wildcard certificate: %w", err)
}
return append(allCertificates, *wc), nil
}
// validateCertificate validates that a certificate can be used with Envoy's TLS stack.
func validateCertificate(cert *tls.Certificate) error {
if len(cert.Certificate) == 0 {
@ -149,3 +255,95 @@ func hasMustStaple(cert *x509.Certificate) bool {
}
return false
}
func newDownstreamTLSTransportSocket(
downstreamTLSContext *envoy_extensions_transport_sockets_tls_v3.DownstreamTlsContext,
) *envoy_config_core_v3.TransportSocket {
return &envoy_config_core_v3.TransportSocket{
Name: "tls",
ConfigType: &envoy_config_core_v3.TransportSocket_TypedConfig{
TypedConfig: marshalAny(downstreamTLSContext),
},
}
}
func (b *Builder) buildDownstreamTLSContextMulti(
ctx context.Context,
cfg *config.Config,
certs []tls.Certificate,
) (
*envoy_extensions_transport_sockets_tls_v3.DownstreamTlsContext,
error,
) {
envoyCerts, err := b.envoyTLSCertificatesFromGoTLSCertificates(ctx, certs)
if err != nil {
return nil, err
}
dtc := &envoy_extensions_transport_sockets_tls_v3.DownstreamTlsContext{
CommonTlsContext: &envoy_extensions_transport_sockets_tls_v3.CommonTlsContext{
TlsParams: tlsDownstreamParams,
TlsCertificates: envoyCerts,
AlpnProtocols: getALPNProtos(cfg.Options),
},
}
b.buildDownstreamValidationContext(ctx, dtc, cfg)
return dtc, nil
}
func getALPNProtos(opts *config.Options) []string {
switch opts.GetCodecType() {
case config.CodecTypeHTTP1:
return []string{"http/1.1"}
case config.CodecTypeHTTP2:
return []string{"h2"}
default:
return []string{"h2", "http/1.1"}
}
}
func (b *Builder) buildDownstreamValidationContext(
ctx context.Context,
dtc *envoy_extensions_transport_sockets_tls_v3.DownstreamTlsContext,
cfg *config.Config,
) {
clientCA := clientCABundle(ctx, cfg)
if len(clientCA) == 0 {
return
}
vc := &envoy_extensions_transport_sockets_tls_v3.CertificateValidationContext{
TrustedCa: b.filemgr.BytesDataSource("client-ca.pem", clientCA),
MatchTypedSubjectAltNames: make([]*envoy_extensions_transport_sockets_tls_v3.SubjectAltNameMatcher,
0, len(cfg.Options.DownstreamMTLS.MatchSubjectAltNames)),
OnlyVerifyLeafCertCrl: true,
}
for i := range cfg.Options.DownstreamMTLS.MatchSubjectAltNames {
vc.MatchTypedSubjectAltNames = append(vc.MatchTypedSubjectAltNames,
cfg.Options.DownstreamMTLS.MatchSubjectAltNames[i].ToEnvoyProto())
}
if d := cfg.Options.DownstreamMTLS.GetMaxVerifyDepth(); d > 0 {
vc.MaxVerifyDepth = wrapperspb.UInt32(d)
}
if cfg.Options.DownstreamMTLS.GetEnforcement() == config.MTLSEnforcementRejectConnection {
dtc.RequireClientCertificate = wrapperspb.Bool(true)
} else {
vc.TrustChainVerification = envoy_extensions_transport_sockets_tls_v3.CertificateValidationContext_ACCEPT_UNTRUSTED
}
if crl := cfg.Options.DownstreamMTLS.CRL; crl != "" {
bs, err := base64.StdEncoding.DecodeString(crl)
if err != nil {
log.Ctx(ctx).Error().Err(err).Msg("invalid client CRL")
} else {
vc.Crl = b.filemgr.BytesDataSource("client-crl.pem", bs)
}
} else if crlf := cfg.Options.DownstreamMTLS.CRLFile; crlf != "" {
vc.Crl = b.filemgr.FileDataSource(crlf)
}
dtc.CommonTlsContext.ValidationContextType = &envoy_extensions_transport_sockets_tls_v3.CommonTlsContext_ValidationContext{
ValidationContext: vc,
}
}

322
config/envoyconfig/tls_test.go
View file

@ -1,14 +1,20 @@
package envoyconfig
import (
"context"
"crypto/x509"
"crypto/x509/pkix"
"encoding/base64"
"net/url"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/pomerium/pomerium/config"
"github.com/pomerium/pomerium/config/envoyconfig/filemgr"
"github.com/pomerium/pomerium/internal/testutil"
"github.com/pomerium/pomerium/pkg/cryptutil"
)
@ -74,3 +80,319 @@ func TestValidateCertificate(t *testing.T) {
assert.Error(t, validateCertificate(cert), "should return an error for a must-staple TLS certificate that has no stapled OCSP response")
}
func Test_buildDownstreamTLSContext(t *testing.T) {
b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
cacheDir, _ := os.UserCacheDir()
clientCAFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "client-ca-313754424855313435355a5348.pem")
t.Run("no-validation", func(t *testing.T) {
downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), &config.Config{Options: &config.Options{}}, nil)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `{
"commonTlsContext": {
"tlsParams": {
"cipherSuites": [
"ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-CHACHA20-POLY1305",
"ECDHE-RSA-CHACHA20-POLY1305"
],
"tlsMinimumProtocolVersion": "TLSv1_2",
"tlsMaximumProtocolVersion": "TLSv1_3"
},
"alpnProtocols": ["h2", "http/1.1"]
}
}`, downstreamTLSContext)
})
t.Run("client-ca", func(t *testing.T) {
downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), &config.Config{Options: &config.Options{
DownstreamMTLS: config.DownstreamMTLSSettings{
CA: "VEVTVAo=", // "TEST\n" (with a trailing newline)
},
}}, nil)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `{
"commonTlsContext": {
"tlsParams": {
"cipherSuites": [
"ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-CHACHA20-POLY1305",
"ECDHE-RSA-CHACHA20-POLY1305"
],
"tlsMinimumProtocolVersion": "TLSv1_2",
"tlsMaximumProtocolVersion": "TLSv1_3"
},
"alpnProtocols": ["h2", "http/1.1"],
"validationContext": {
"maxVerifyDepth": 1,
"onlyVerifyLeafCertCrl": true,
"trustChainVerification": "ACCEPT_UNTRUSTED",
"trustedCa": {
"filename": "`+clientCAFileName+`"
}
}
}
}`, downstreamTLSContext)
})
t.Run("client-ca-strict", func(t *testing.T) {
downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), &config.Config{Options: &config.Options{
DownstreamMTLS: config.DownstreamMTLSSettings{
CA: "VEVTVAo=", // "TEST\n" (with a trailing newline)
Enforcement: config.MTLSEnforcementRejectConnection,
},
}}, nil)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `{
"commonTlsContext": {
"tlsParams": {
"cipherSuites": [
"ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-CHACHA20-POLY1305",
"ECDHE-RSA-CHACHA20-POLY1305"
],
"tlsMinimumProtocolVersion": "TLSv1_2",
"tlsMaximumProtocolVersion": "TLSv1_3"
},
"alpnProtocols": ["h2", "http/1.1"],
"validationContext": {
"maxVerifyDepth": 1,
"onlyVerifyLeafCertCrl": true,
"trustedCa": {
"filename": "`+clientCAFileName+`"
}
}
},
"requireClientCertificate": true
}`, downstreamTLSContext)
})
t.Run("policy-client-ca", func(t *testing.T) {
downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), &config.Config{Options: &config.Options{
Policies: []config.Policy{
{
From: "https://a.example.com:1234",
TLSDownstreamClientCA: "VEVTVA==", // "TEST" (no trailing newline)
},
},
}}, nil)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `{
"commonTlsContext": {
"tlsParams": {
"cipherSuites": [
"ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-CHACHA20-POLY1305",
"ECDHE-RSA-CHACHA20-POLY1305"
],
"tlsMinimumProtocolVersion": "TLSv1_2",
"tlsMaximumProtocolVersion": "TLSv1_3"
},
"alpnProtocols": ["h2", "http/1.1"],
"validationContext": {
"maxVerifyDepth": 1,
"onlyVerifyLeafCertCrl": true,
"trustChainVerification": "ACCEPT_UNTRUSTED",
"trustedCa": {
"filename": "`+clientCAFileName+`"
}
}
}
}`, downstreamTLSContext)
})
t.Run("client-ca-max-verify-depth", func(t *testing.T) {
var maxVerifyDepth uint32
config := &config.Config{Options: &config.Options{
DownstreamMTLS: config.DownstreamMTLSSettings{
MaxVerifyDepth: &maxVerifyDepth,
CA: "VEVTVAo=", // "TEST\n"
},
}}
maxVerifyDepth = 10
downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), config, nil)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `{
"maxVerifyDepth": 10,
"onlyVerifyLeafCertCrl": true,
"trustChainVerification": "ACCEPT_UNTRUSTED",
"trustedCa": {
"filename": "`+clientCAFileName+`"
}
}`, downstreamTLSContext.GetCommonTlsContext().GetValidationContext())
maxVerifyDepth = 0
downstreamTLSContext, err = b.buildDownstreamTLSContextMulti(context.Background(), config, nil)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `{
"onlyVerifyLeafCertCrl": true,
"trustChainVerification": "ACCEPT_UNTRUSTED",
"trustedCa": {
"filename": "`+clientCAFileName+`"
}
}`, downstreamTLSContext.GetCommonTlsContext().GetValidationContext())
})
t.Run("client-ca-san-matchers", func(t *testing.T) {
config := &config.Config{Options: &config.Options{
DownstreamMTLS: config.DownstreamMTLSSettings{
CA: "VEVTVAo=", // "TEST\n"
MatchSubjectAltNames: []config.SANMatcher{
{Type: config.SANTypeDNS, Pattern: `.*\.corp\.example\.com`},
{Type: config.SANTypeEmail, Pattern: `.*@example\.com`},
{Type: config.SANTypeIPAddress, Pattern: `10\.10\.42\..*`},
{Type: config.SANTypeURI, Pattern: `spiffe://example\.com/.*`},
{Type: config.SANTypeUserPrincipalName, Pattern: `^device-id$`},
},
},
}}
downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), config, nil)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `{
"maxVerifyDepth": 1,
"matchTypedSubjectAltNames": [
{
"matcher": {
"safeRegex": {
"googleRe2": {},
"regex": ".*\\.corp\\.example\\.com"
}
},
"sanType": "DNS"
},
{
"matcher": {
"safeRegex": {
"googleRe2": {},
"regex": ".*@example\\.com"
}
},
"sanType": "EMAIL"
},
{
"matcher": {
"safeRegex": {
"googleRe2": {},
"regex": "10\\.10\\.42\\..*"
}
},
"sanType": "IP_ADDRESS"
},
{
"matcher": {
"safeRegex": {
"googleRe2": {},
"regex": "spiffe://example\\.com/.*"
}
},
"sanType": "URI"
},
{
"matcher": {
"safeRegex": {
"googleRe2": {},
"regex": "^device-id$"
}
},
"sanType": "OTHER_NAME",
"oid": "1.3.6.1.4.1.311.20.2.3"
}
],
"onlyVerifyLeafCertCrl": true,
"trustChainVerification": "ACCEPT_UNTRUSTED",
"trustedCa": {
"filename": "`+clientCAFileName+`"
}
}`, downstreamTLSContext.GetCommonTlsContext().GetValidationContext())
})
t.Run("http1", func(t *testing.T) {
downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), &config.Config{Options: &config.Options{
Cert: aExampleComCert,
Key: aExampleComKey,
CodecType: config.CodecTypeHTTP1,
}}, nil)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `{
"commonTlsContext": {
"tlsParams": {
"cipherSuites": [
"ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-CHACHA20-POLY1305",
"ECDHE-RSA-CHACHA20-POLY1305"
],
"tlsMinimumProtocolVersion": "TLSv1_2",
"tlsMaximumProtocolVersion": "TLSv1_3"
},
"alpnProtocols": ["http/1.1"]
}
}`, downstreamTLSContext)
})
t.Run("http2", func(t *testing.T) {
downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), &config.Config{Options: &config.Options{
Cert: aExampleComCert,
Key: aExampleComKey,
CodecType: config.CodecTypeHTTP2,
}}, nil)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `{
"commonTlsContext": {
"tlsParams": {
"cipherSuites": [
"ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-CHACHA20-POLY1305",
"ECDHE-RSA-CHACHA20-POLY1305"
],
"tlsMinimumProtocolVersion": "TLSv1_2",
"tlsMaximumProtocolVersion": "TLSv1_3"
},
"alpnProtocols": ["h2"]
}
}`, downstreamTLSContext)
})
}
func Test_clientCABundle(t *testing.T) {
// Make sure multiple bundled CAs are separated by newlines.
clientCA1 := []byte("client CA 1")
clientCA2 := []byte("client CA 2")
clientCA3 := []byte("client CA 3")
b64 := base64.StdEncoding.EncodeToString
cfg := &config.Config{Options: &config.Options{
DownstreamMTLS: config.DownstreamMTLSSettings{
CA: b64(clientCA3),
},
Policies: []config.Policy{
{
From: "https://foo.example.com",
TLSDownstreamClientCA: b64(clientCA2),
},
{
From: "https://bar.example.com",
TLSDownstreamClientCA: b64(clientCA1),
},
},
}}
expected := []byte("client CA 3\nclient CA 2\nclient CA 1\n")
actual := clientCABundle(context.Background(), cfg)
assert.Equal(t, expected, actual)
}