From 24995b18061979e91f196281e58f9682a201ece6 Mon Sep 17 00:00:00 2001 From: Kenneth Jenkins <51246568+kenjenkins@users.noreply.github.com> Date: Mon, 10 Feb 2025 15:44:00 -0800 Subject: [PATCH] set up for dynamic forward proxy --- config/envoyconfig/clusters.go | 52 ++++++++++++++++++++++ config/envoyconfig/filters.go | 17 +++++++ config/envoyconfig/listeners.go | 1 + config/envoyconfig/listeners_main.go | 1 + config/envoyconfig/route_configurations.go | 3 ++ config/envoyconfig/routes.go | 20 +++++++++ 6 files changed, 94 insertions(+) diff --git a/config/envoyconfig/clusters.go b/config/envoyconfig/clusters.go index bfb083cc5..3a8a254e7 100644 --- a/config/envoyconfig/clusters.go +++ b/config/envoyconfig/clusters.go @@ -12,6 +12,9 @@ import ( envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + envoy_extensions_clusters_dynamic_forward_proxy_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/dynamic_forward_proxy/v3" + envoy_extensions_common_dynamic_forward_proxy_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/common/dynamic_forward_proxy/v3" + envoy_extensions_network_dns_resolver_cares_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/network/dns_resolver/cares/v3" envoy_extensions_transport_sockets_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/durationpb" @@ -22,6 +25,7 @@ import ( "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/urlutil" + "github.com/pomerium/pomerium/pkg/protoutil" ) // BuildClusters builds envoy clusters from the given config. @@ -114,6 +118,9 @@ func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*env clusters = append(clusters, cluster) } } + + // XXX + clusters = append(clusters, b.forwardProxyCluster()) } if err = validateClusters(clusters); err != nil { @@ -539,3 +546,48 @@ func getClusterDiscoveryType(lbEndpoints []*envoy_config_endpoint_v3.LbEndpoint) } return &envoy_config_cluster_v3.Cluster_Type{Type: envoy_config_cluster_v3.Cluster_STRICT_DNS} } + +func (b *Builder) forwardProxyCluster() *envoy_config_cluster_v3.Cluster { + clusterConfig := protoutil.NewAny(&envoy_extensions_clusters_dynamic_forward_proxy_v3.ClusterConfig{ + ClusterImplementationSpecifier: &envoy_extensions_clusters_dynamic_forward_proxy_v3.ClusterConfig_DnsCacheConfig{ + DnsCacheConfig: b.forwardProxyDNSCacheConfig(), + }, + }) + return &envoy_config_cluster_v3.Cluster{ + Name: "dynamic-forward-proxy-cluster", + LbPolicy: envoy_config_cluster_v3.Cluster_CLUSTER_PROVIDED, + ClusterDiscoveryType: &envoy_config_cluster_v3.Cluster_ClusterType{ + ClusterType: &envoy_config_cluster_v3.Cluster_CustomClusterType{ + Name: "envoy.clusters.dynamic_forward_proxy", + TypedConfig: clusterConfig, + }, + }, + } +} + +func (b *Builder) forwardProxyDNSCacheConfig() *envoy_extensions_common_dynamic_forward_proxy_v3.DnsCacheConfig { + resolverConfig := protoutil.NewAny(&envoy_extensions_network_dns_resolver_cares_v3.CaresDnsResolverConfig{ + Resolvers: []*envoy_config_core_v3.Address{{ + Address: &envoy_config_core_v3.Address_SocketAddress{ + SocketAddress: &envoy_config_core_v3.SocketAddress{ + Address: "8.8.8.8", // XXX: this should be configurable + PortSpecifier: &envoy_config_core_v3.SocketAddress_PortValue{ + PortValue: 53, + }, + }, + }, + }}, + DnsResolverOptions: &envoy_config_core_v3.DnsResolverOptions{ + UseTcpForDnsLookups: true, + NoDefaultSearchDomain: true, + }, + }) + return &envoy_extensions_common_dynamic_forward_proxy_v3.DnsCacheConfig{ + Name: "dynamic_forward_proxy_cache_config", + DnsLookupFamily: envoy_config_cluster_v3.Cluster_AUTO, + TypedDnsResolverConfig: &envoy_config_core_v3.TypedExtensionConfig{ + Name: "envoy.network.dns_resolver.cares", + TypedConfig: resolverConfig, + }, + } +} diff --git a/config/envoyconfig/filters.go b/config/envoyconfig/filters.go index e4c593b5e..4181f9151 100644 --- a/config/envoyconfig/filters.go +++ b/config/envoyconfig/filters.go @@ -3,6 +3,8 @@ package envoyconfig import ( envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_config_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + envoy_extensions_common_dynamic_forward_proxy_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/common/dynamic_forward_proxy/v3" + envoy_extensions_filters_http_dynamic_forward_proxy_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/dynamic_forward_proxy/v3" envoy_extensions_filters_http_ext_authz_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_authz/v3" envoy_extensions_filters_http_header_mutation_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/header_mutation/v3" envoy_extensions_filters_http_lua_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/lua/v3" @@ -76,6 +78,21 @@ func HTTPRouterFilter() *envoy_extensions_filters_network_http_connection_manage } } +func DynamicForwardProxyFilter( + dnsCacheConfig *envoy_extensions_common_dynamic_forward_proxy_v3.DnsCacheConfig, +) *envoy_extensions_filters_network_http_connection_manager.HttpFilter { + return &envoy_extensions_filters_network_http_connection_manager.HttpFilter{ + Name: "envoy.filters.http.dynamic_forward_proxy", + ConfigType: &envoy_extensions_filters_network_http_connection_manager.HttpFilter_TypedConfig{ + TypedConfig: protoutil.NewAny(&envoy_extensions_filters_http_dynamic_forward_proxy_v3.FilterConfig{ + ImplementationSpecifier: &envoy_extensions_filters_http_dynamic_forward_proxy_v3.FilterConfig_DnsCacheConfig{ + DnsCacheConfig: dnsCacheConfig, + }, + }), + }, + } +} + // LuaFilter creates a lua HTTP filter. func LuaFilter(defaultSourceCode string) *envoy_extensions_filters_network_http_connection_manager.HttpFilter { return &envoy_extensions_filters_network_http_connection_manager.HttpFilter{ diff --git a/config/envoyconfig/listeners.go b/config/envoyconfig/listeners.go index 2759fe9a2..fb03abac8 100644 --- a/config/envoyconfig/listeners.go +++ b/config/envoyconfig/listeners.go @@ -35,6 +35,7 @@ func (b *Builder) BuildListeners( li = proto.Clone(li).(*envoy_config_listener_v3.Listener) li.Name = "http-ingress-internal-listener" li.Address = nil + li.EnableReusePort = nil // not supported for an internal listener li.ListenerSpecifier = &envoy_config_listener_v3.Listener_InternalListener{ InternalListener: &envoy_config_listener_v3.Listener_InternalListenerConfig{}, } diff --git a/config/envoyconfig/listeners_main.go b/config/envoyconfig/listeners_main.go index da5544188..83e8a7823 100644 --- a/config/envoyconfig/listeners_main.go +++ b/config/envoyconfig/listeners_main.go @@ -175,6 +175,7 @@ func (b *Builder) buildMainHTTPConnectionManagerFilter( if !useQUIC && cfg.Options.CodecType == config.CodecTypeHTTP3 { filters = append(filters, newQUICAltSvcHeaderFilter(cfg)) } + filters = append(filters, DynamicForwardProxyFilter(b.forwardProxyDNSCacheConfig())) filters = append(filters, HTTPRouterFilter()) var maxStreamDuration *durationpb.Duration diff --git a/config/envoyconfig/route_configurations.go b/config/envoyconfig/route_configurations.go index 9881e7322..1249663e4 100644 --- a/config/envoyconfig/route_configurations.go +++ b/config/envoyconfig/route_configurations.go @@ -98,6 +98,9 @@ func (b *Builder) buildMainRouteConfiguration( return nil, err } vh.Routes = append(vh.Routes, rs...) + + // XXX + vh.Routes = append(vh.Routes, b.buildDynamicForwardProxyRoute(cfg)) } virtualHosts = append(virtualHosts, vh) diff --git a/config/envoyconfig/routes.go b/config/envoyconfig/routes.go index 4bbb10263..058c7d474 100644 --- a/config/envoyconfig/routes.go +++ b/config/envoyconfig/routes.go @@ -355,6 +355,26 @@ func (b *Builder) buildRouteForPolicyAndMatch( return route, nil } +func (b *Builder) buildDynamicForwardProxyRoute(cfg *config.Config) *envoy_config_route_v3.Route { + return &envoy_config_route_v3.Route{ + Name: "dynamic", + Match: &envoy_config_route_v3.RouteMatch{ + PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{ + Prefix: "/", + }, + }, + Action: &envoy_config_route_v3.Route_Route{ + Route: &envoy_config_route_v3.RouteAction{ + ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{ + Cluster: "dynamic-forward-proxy-cluster", + }, + }, + }, + // XXX: does this need any RequestHeadersToRemove? + // XXX: does this need a Decorator? + } +} + func (b *Builder) buildPolicyRouteDirectResponseAction(r *config.DirectResponse) *envoy_config_route_v3.DirectResponseAction { return &envoy_config_route_v3.DirectResponseAction{ Status: uint32(r.Status),