config: migrate deprecated cluster DNS settings (#5690)

Address the deprecation warnings for `respect_dns_ttl` by migrating to
the newer CustomClusterType config proto.

config/envoyconfig/clusters.go

@@ -12,6 +12,8 @@ import (
 	envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	envoy_extensions_clusters_common_dns_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/common/dns/v3"
+	envoy_extensions_clusters_dns_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/dns/v3"
 	envoy_extensions_transport_sockets_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/durationpb"
@@ -137,7 +139,6 @@ func (b *Builder) buildInternalCluster(
 	keepalive Keepalive,
 ) (*envoy_config_cluster_v3.Cluster, error) {
 	cluster := newDefaultEnvoyClusterConfig()
-	cluster.DnsLookupFamily = config.GetEnvoyDNSLookupFamily(cfg.Options.DNSLookupFamily)
 	// Match the Go standard library default TCP keepalive settings.
 	cluster.UpstreamConnectionOptions = &envoy_config_cluster_v3.UpstreamConnectionOptions{
 		TcpKeepalive: defaultTCPKeepalive,
@@ -150,7 +151,10 @@ func (b *Builder) buildInternalCluster(
 		}
 		endpoints = append(endpoints, NewEndpoint(dst, ts, 1))
 	}
-	if err := b.buildCluster(cluster, name, endpoints, upstreamProtocol, keepalive); err != nil {
+	dnsLookupFamily := config.GetEnvoyDNSLookupFamily(cfg.Options.DNSLookupFamily)
+	if err := b.buildCluster(
+		cluster, name, endpoints, upstreamProtocol, dnsLookupFamily, keepalive,
+	); err != nil {
 		return nil, err
 	}
 	cluster.CircuitBreakers = buildInternalCircuitBreakers(cfg)
@@ -199,12 +203,14 @@ func (b *Builder) buildPolicyCluster(ctx context.Context, cfg *config.Config, po
 		return nil, err
 	}
 
-	cluster.DnsLookupFamily = config.GetEnvoyDNSLookupFamily(options.DNSLookupFamily)
+	dnsLookupFamily := config.GetEnvoyDNSLookupFamily(options.DNSLookupFamily)
 	if policy.EnableGoogleCloudServerlessAuthentication {
-		cluster.DnsLookupFamily = envoy_config_cluster_v3.Cluster_V4_ONLY
+		dnsLookupFamily = envoy_extensions_clusters_common_dns_v3.DnsLookupFamily_V4_ONLY
 	}
 
-	if err := b.buildCluster(cluster, name, endpoints, upstreamProtocol, Keepalive(false)); err != nil {
+	if err := b.buildCluster(
+		cluster, name, endpoints, upstreamProtocol, dnsLookupFamily, Keepalive(false),
+	); err != nil {
 		return nil, err
 	}
 	cluster.CircuitBreakers = buildRouteCircuitBreakers(cfg, policy)
@@ -362,6 +368,7 @@ func (b *Builder) buildCluster(
 	name string,
 	endpoints []Endpoint,
 	upstreamProtocol upstreamProtocolConfig,
+	dnsLookupFamily envoy_extensions_clusters_common_dns_v3.DnsLookupFamily,
 	keepalive Keepalive,
 ) error {
 	if len(endpoints) == 0 {
@@ -371,7 +378,6 @@ func (b *Builder) buildCluster(
 	if cluster.ConnectTimeout == nil {
 		cluster.ConnectTimeout = defaultConnectionTimeout
 	}
-	cluster.RespectDnsTtl = true
 	lbEndpoints, err := b.buildLbEndpoints(endpoints)
 	if err != nil {
 		return err
@@ -394,7 +400,8 @@ func (b *Builder) buildCluster(
 	}
 
 	cluster.TypedExtensionProtocolOptions = buildTypedExtensionProtocolOptions(endpoints, upstreamProtocol, keepalive)
-	cluster.ClusterDiscoveryType = getClusterDiscoveryType(lbEndpoints)
+
+	cluster.ClusterDiscoveryType = getClusterDiscoveryType(lbEndpoints, dnsLookupFamily)
 
 	return cluster.Validate()
 }
@@ -528,16 +535,35 @@ func validateClusterNamesUnique(clusters []*envoy_config_cluster_v3.Cluster) err
 	return nil
 }
 
-func getClusterDiscoveryType(lbEndpoints []*envoy_config_endpoint_v3.LbEndpoint) *envoy_config_cluster_v3.Cluster_Type {
+func allIPAddresses(lbEndpoints []*envoy_config_endpoint_v3.LbEndpoint) bool {
-	// for IPs we use a static discovery type, otherwise we use DNS
-	allIP := true
 	for _, lbe := range lbEndpoints {
 		if net.ParseIP(urlutil.StripPort(lbe.GetEndpoint().GetAddress().GetSocketAddress().GetAddress())) == nil {
-			allIP = false
+			return false
 		}
 	}
-	if allIP {
+	return true
-		return &envoy_config_cluster_v3.Cluster_Type{Type: envoy_config_cluster_v3.Cluster_STATIC}
+}
-	}
+
-	return &envoy_config_cluster_v3.Cluster_Type{Type: envoy_config_cluster_v3.Cluster_STRICT_DNS}
+func getClusterDiscoveryType(
+	lbEndpoints []*envoy_config_endpoint_v3.LbEndpoint,
+	dnsLookupFamily envoy_extensions_clusters_common_dns_v3.DnsLookupFamily,
+) *envoy_config_cluster_v3.Cluster_ClusterType {
+	// for IPs we use a static discovery type, otherwise we use DNS
+	if allIPAddresses(lbEndpoints) {
+		return &envoy_config_cluster_v3.Cluster_ClusterType{
+			ClusterType: &envoy_config_cluster_v3.Cluster_CustomClusterType{
+				Name: "envoy.cluster.static",
+			},
+		}
+	}
+
+	return &envoy_config_cluster_v3.Cluster_ClusterType{
+		ClusterType: &envoy_config_cluster_v3.Cluster_CustomClusterType{
+			Name: "envoy.clusters.dns",
+			TypedConfig: marshalAny(&envoy_extensions_clusters_dns_v3.DnsCluster{
+				RespectDnsTtl:   true,
+				DnsLookupFamily: dnsLookupFamily,
+			}),
+		},
+	}
 }

config/envoyconfig/clusters_test.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	envoy_extensions_clusters_common_dns_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/common/dns/v3"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/volatiletech/null/v9"
@@ -527,16 +528,21 @@ func Test_buildCluster(t *testing.T) {
 		})
 		require.NoError(t, err)
 		cluster := newDefaultEnvoyClusterConfig()
-		cluster.DnsLookupFamily = envoy_config_cluster_v3.Cluster_V4_ONLY
+		dnsLookupFamily := envoy_extensions_clusters_common_dns_v3.DnsLookupFamily_V4_ONLY
-		err = b.buildCluster(cluster, "example", endpoints, upstreamProtocolHTTP2, Keepalive(false))
+		err = b.buildCluster(cluster, "example", endpoints, upstreamProtocolHTTP2, dnsLookupFamily, Keepalive(false))
 		require.NoErrorf(t, err, "cluster %+v", cluster)
 		testutil.AssertProtoJSONEqual(t, `
 			{
 				"name": "example",
-				"type": "STRICT_DNS",
+				"clusterType": {
+ 					"name": "envoy.clusters.dns",
+					"typedConfig": {
+						"@type": "type.googleapis.com/envoy.extensions.clusters.dns.v3.DnsCluster",
+						"dnsLookupFamily": "V4_ONLY",
+						"respectDnsTtl": true
+					}
+				},
 				"connectTimeout": "10s",
-				"respectDnsTtl": true,
-				"dnsLookupFamily": "V4_ONLY",
 				"perConnectionBufferLimitBytes": 32768,
 				"typedExtensionProtocolOptions": {
 					"envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
@@ -589,14 +595,21 @@ func Test_buildCluster(t *testing.T) {
 		})
 		require.NoError(t, err)
 		cluster := newDefaultEnvoyClusterConfig()
-		err = b.buildCluster(cluster, "example", endpoints, upstreamProtocolHTTP2, Keepalive(true))
+		dnsLookupFamily := envoy_extensions_clusters_common_dns_v3.DnsLookupFamily_V4_PREFERRED
+		err = b.buildCluster(cluster, "example", endpoints, upstreamProtocolHTTP2, dnsLookupFamily, Keepalive(true))
 		require.NoErrorf(t, err, "cluster %+v", cluster)
 		testutil.AssertProtoJSONEqual(t, `
 			{
 				"name": "example",
-				"type": "STRICT_DNS",
+				"clusterType": {
+ 					"name": "envoy.clusters.dns",
+					"typedConfig": {
+						"@type": "type.googleapis.com/envoy.extensions.clusters.dns.v3.DnsCluster",
+						"dnsLookupFamily": "V4_PREFERRED",
+						"respectDnsTtl": true
+					}
+				},
 				"connectTimeout": "10s",
-				"respectDnsTtl": true,
 				"perConnectionBufferLimitBytes": 32768,
 				"transportSocketMatches": [{
 					"name": "`+endpoints[0].TransportSocketName()+`",
@@ -719,7 +732,6 @@ func Test_buildCluster(t *testing.T) {
 						}
 					}
 				},
-				"dnsLookupFamily": "V4_PREFERRED",
 				"loadAssignment": {
 					"clusterName": "example",
 					"endpoints": [{
@@ -769,14 +781,15 @@ func Test_buildCluster(t *testing.T) {
 		})
 		require.NoError(t, err)
 		cluster := newDefaultEnvoyClusterConfig()
-		err = b.buildCluster(cluster, "example", endpoints, upstreamProtocolHTTP2, Keepalive(false))
+		err = b.buildCluster(cluster, "example", endpoints, upstreamProtocolHTTP2, 0, Keepalive(false))
 		require.NoErrorf(t, err, "cluster %+v", cluster)
 		testutil.AssertProtoJSONEqual(t, `
 			{
 				"name": "example",
-				"type": "STATIC",
+				"clusterType": {
+ 					"name": "envoy.cluster.static"
+				},
 				"connectTimeout": "10s",
-				"respectDnsTtl": true,
 				"perConnectionBufferLimitBytes": 32768,
 				"typedExtensionProtocolOptions": {
 					"envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
@@ -791,7 +804,6 @@ func Test_buildCluster(t *testing.T) {
 						}
 					}
 				},
-				"dnsLookupFamily": "V4_PREFERRED",
 				"loadAssignment": {
 					"clusterName": "example",
 					"endpoints": [{
@@ -827,14 +839,15 @@ func Test_buildCluster(t *testing.T) {
 		})
 		require.NoError(t, err)
 		cluster := newDefaultEnvoyClusterConfig()
-		err = b.buildCluster(cluster, "example", endpoints, upstreamProtocolHTTP2, Keepalive(false))
+		err = b.buildCluster(cluster, "example", endpoints, upstreamProtocolHTTP2, 0, Keepalive(false))
 		require.NoErrorf(t, err, "cluster %+v", cluster)
 		testutil.AssertProtoJSONEqual(t, `
 			{
 				"name": "example",
-				"type": "STATIC",
+				"clusterType": {
+ 					"name": "envoy.cluster.static"
+				},
 				"connectTimeout": "10s",
-				"respectDnsTtl": true,
 				"perConnectionBufferLimitBytes": 32768,
 				"typedExtensionProtocolOptions": {
 					"envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
@@ -849,7 +862,6 @@ func Test_buildCluster(t *testing.T) {
 						}
 					}
 				},
-				"dnsLookupFamily": "V4_PREFERRED",
 				"loadAssignment": {
 					"clusterName": "example",
 					"endpoints": [{
@@ -887,14 +899,15 @@ func Test_buildCluster(t *testing.T) {
 		})
 		require.NoError(t, err)
 		cluster := newDefaultEnvoyClusterConfig()
-		err = b.buildCluster(cluster, "example", endpoints, upstreamProtocolHTTP2, Keepalive(false))
+		err = b.buildCluster(cluster, "example", endpoints, upstreamProtocolHTTP2, 0, Keepalive(false))
 		require.NoErrorf(t, err, "cluster %+v", cluster)
 		testutil.AssertProtoJSONEqual(t, `
 			{
 				"name": "example",
-				"type": "STATIC",
+				"clusterType": {
+ 					"name": "envoy.cluster.static"
+				},
 				"connectTimeout": "10s",
-				"respectDnsTtl": true,
 				"perConnectionBufferLimitBytes": 32768,
 				"typedExtensionProtocolOptions": {
 					"envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
@@ -909,7 +922,6 @@ func Test_buildCluster(t *testing.T) {
 						}
 					}
 				},
-				"dnsLookupFamily": "V4_PREFERRED",
 				"loadAssignment": {
 					"clusterName": "example",
 					"endpoints": [{
@@ -935,19 +947,25 @@ func Test_buildCluster(t *testing.T) {
 		})
 		require.NoError(t, err)
 		cluster := newDefaultEnvoyClusterConfig()
-		cluster.DnsLookupFamily = envoy_config_cluster_v3.Cluster_V4_ONLY
 		cluster.OutlierDetection = &envoy_config_cluster_v3.OutlierDetection{
 			EnforcingConsecutive_5Xx:       wrapperspb.UInt32(17),
 			SplitExternalLocalOriginErrors: true,
 		}
-		err = b.buildCluster(cluster, "example", endpoints, upstreamProtocolHTTP2, Keepalive(false))
+		dnsLookupFamily := envoy_extensions_clusters_common_dns_v3.DnsLookupFamily_V4_ONLY
+		err = b.buildCluster(cluster, "example", endpoints, upstreamProtocolHTTP2, dnsLookupFamily, Keepalive(false))
 		require.NoErrorf(t, err, "cluster %+v", cluster)
 		testutil.AssertProtoJSONEqual(t, `
 			{
 				"name": "example",
-				"type": "STRICT_DNS",
+				"clusterType": {
+ 					"name": "envoy.clusters.dns",
+					"typedConfig": {
+						"@type": "type.googleapis.com/envoy.extensions.clusters.dns.v3.DnsCluster",
+						"dnsLookupFamily": "V4_ONLY",
+						"respectDnsTtl": true
+					}
+				},
 				"connectTimeout": "10s",
-				"respectDnsTtl": true,
 				"perConnectionBufferLimitBytes": 32768,
 				"typedExtensionProtocolOptions": {
 					"envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
@@ -962,7 +980,6 @@ func Test_buildCluster(t *testing.T) {
 						}
 					}
 				},
-				"dnsLookupFamily": "V4_ONLY",
 				"outlierDetection": {
 					"enforcingConsecutive5xx": 17,
 					"splitExternalLocalOriginErrors": true

config/envoyconfig/envoyconfig.go

@@ -66,8 +66,6 @@ func (e Endpoint) TransportSocketName() string {
 func newDefaultEnvoyClusterConfig() *envoy_config_cluster_v3.Cluster {
 	return &envoy_config_cluster_v3.Cluster{
 		ConnectTimeout:                defaultConnectionTimeout,
-		RespectDnsTtl:                 true,
-		DnsLookupFamily:               envoy_config_cluster_v3.Cluster_V4_PREFERRED,
 		PerConnectionBufferLimitBytes: wrapperspb.UInt32(connectionBufferLimit),
 	}
 }

config/envoyconfig/testdata/clusters.json

@@ -33,7 +33,14 @@
         }
       ]
     },
-    "dnsLookupFamily": "V4_PREFERRED",
+    "clusterType": {
+      "name": "envoy.clusters.dns",
+      "typedConfig": {
+        "@type": "type.googleapis.com/envoy.extensions.clusters.dns.v3.DnsCluster",
+        "dnsLookupFamily": "V4_PREFERRED",
+        "respectDnsTtl": true
+      }
+    },
     "loadAssignment": {
       "clusterName": "pomerium-control-plane-grpc",
       "endpoints": [
@@ -57,8 +64,6 @@
     },
     "name": "pomerium-control-plane-grpc",
     "perConnectionBufferLimitBytes": 32768,
-    "respectDnsTtl": true,
-    "type": "STRICT_DNS",
     "typedExtensionProtocolOptions": {
       "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
         "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions",
@@ -81,6 +86,14 @@
   },
   {
     "connectTimeout": "10s",
+    "clusterType": {
+      "name": "envoy.clusters.dns",
+      "typedConfig": {
+        "@type": "type.googleapis.com/envoy.extensions.clusters.dns.v3.DnsCluster",
+        "dnsLookupFamily": "V4_PREFERRED",
+        "respectDnsTtl": true
+      }
+    },
     "circuitBreakers": {
       "thresholds": [
         {
@@ -91,7 +104,6 @@
         }
       ]
     },
-    "dnsLookupFamily": "V4_PREFERRED",
     "loadAssignment": {
       "clusterName": "pomerium-control-plane-http",
       "endpoints": [
@@ -115,8 +127,6 @@
     },
     "name": "pomerium-control-plane-http",
     "perConnectionBufferLimitBytes": 32768,
-    "respectDnsTtl": true,
-    "type": "STRICT_DNS",
     "typedExtensionProtocolOptions": {
       "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
         "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions",
@@ -143,6 +153,14 @@
   },
   {
     "connectTimeout": "10s",
+    "clusterType": {
+      "name": "envoy.clusters.dns",
+      "typedConfig": {
+        "@type": "type.googleapis.com/envoy.extensions.clusters.dns.v3.DnsCluster",
+        "dnsLookupFamily": "V4_PREFERRED",
+        "respectDnsTtl": true
+      }
+    },
     "circuitBreakers": {
       "thresholds": [
         {
@@ -153,7 +171,6 @@
         }
       ]
     },
-    "dnsLookupFamily": "V4_PREFERRED",
     "loadAssignment": {
       "clusterName": "pomerium-control-plane-metrics",
       "endpoints": [
@@ -177,8 +194,6 @@
     },
     "name": "pomerium-control-plane-metrics",
     "perConnectionBufferLimitBytes": 32768,
-    "respectDnsTtl": true,
-    "type": "STRICT_DNS",
     "typedExtensionProtocolOptions": {
       "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
         "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions",
@@ -205,6 +220,14 @@
   },
   {
     "connectTimeout": "10s",
+    "clusterType": {
+      "name": "envoy.clusters.dns",
+      "typedConfig": {
+        "@type": "type.googleapis.com/envoy.extensions.clusters.dns.v3.DnsCluster",
+        "dnsLookupFamily": "V4_PREFERRED",
+        "respectDnsTtl": true
+      }
+    },
     "circuitBreakers": {
       "thresholds": [
         {
@@ -215,7 +238,6 @@
         }
       ]
     },
-    "dnsLookupFamily": "V4_PREFERRED",
     "loadAssignment": {
       "clusterName": "pomerium-authorize",
       "endpoints": [
@@ -239,8 +261,6 @@
     },
     "name": "pomerium-authorize",
     "perConnectionBufferLimitBytes": 32768,
-    "respectDnsTtl": true,
-    "type": "STRICT_DNS",
     "typedExtensionProtocolOptions": {
       "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
         "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions",
@@ -263,6 +283,14 @@
   },
   {
     "connectTimeout": "10s",
+    "clusterType": {
+      "name": "envoy.clusters.dns",
+      "typedConfig": {
+        "@type": "type.googleapis.com/envoy.extensions.clusters.dns.v3.DnsCluster",
+        "dnsLookupFamily": "V4_PREFERRED",
+        "respectDnsTtl": true
+      }
+    },
     "circuitBreakers": {
       "thresholds": [
         {
@@ -273,7 +301,6 @@
         }
       ]
     },
-    "dnsLookupFamily": "V4_PREFERRED",
     "loadAssignment": {
       "clusterName": "pomerium-databroker",
       "endpoints": [
@@ -297,8 +324,6 @@
     },
     "name": "pomerium-databroker",
     "perConnectionBufferLimitBytes": 32768,
-    "respectDnsTtl": true,
-    "type": "STRICT_DNS",
     "typedExtensionProtocolOptions": {
       "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
         "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions",

config/validate.go

@@ -6,7 +6,7 @@ import (
 	"strconv"
 	"strings"
 
-	envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	envoy_extensions_clusters_common_dns_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/common/dns/v3"
 )
 
 // DNSLookupFamily values.
@@ -53,22 +53,22 @@ func ValidateCookieSameSite(value string) error {
 }
 
 // GetEnvoyDNSLookupFamily gets the envoy DNS lookup family.
-func GetEnvoyDNSLookupFamily(value string) envoy_config_cluster_v3.Cluster_DnsLookupFamily {
+func GetEnvoyDNSLookupFamily(value string) envoy_extensions_clusters_common_dns_v3.DnsLookupFamily {
 	switch value {
 	case DNSLookupFamilyAuto:
-		return envoy_config_cluster_v3.Cluster_AUTO
+		return envoy_extensions_clusters_common_dns_v3.DnsLookupFamily_AUTO
 	case DNSLookupFamilyV4Only:
-		return envoy_config_cluster_v3.Cluster_V4_ONLY
+		return envoy_extensions_clusters_common_dns_v3.DnsLookupFamily_V4_ONLY
 	case DNSLookupFamilyV6Only:
-		return envoy_config_cluster_v3.Cluster_V6_ONLY
+		return envoy_extensions_clusters_common_dns_v3.DnsLookupFamily_V6_ONLY
 	case DNSLookupFamilyV4Preferred:
-		return envoy_config_cluster_v3.Cluster_V4_PREFERRED
+		return envoy_extensions_clusters_common_dns_v3.DnsLookupFamily_V4_PREFERRED
 	case DNSLookupFamilyAll:
-		return envoy_config_cluster_v3.Cluster_ALL
+		return envoy_extensions_clusters_common_dns_v3.DnsLookupFamily_ALL
 	}
 
 	// default
-	return envoy_config_cluster_v3.Cluster_V4_PREFERRED
+	return envoy_extensions_clusters_common_dns_v3.DnsLookupFamily_V4_PREFERRED
 }
 
 // ValidateMetricsAddress validates address for the metrics