expose envoy cluster options in policy (#1804)

config/constants.go

@@ -1,13 +1,21 @@
 package config
 
-import "errors"
+import (
+	"errors"
+
+	"google.golang.org/protobuf/encoding/protojson"
+)
 
 const (
-	policyKey      = "policy"
+	policyKey    = "policy"
-	toKey          = "to"
+	toKey        = "to"
-	healthCheckKey = "health_check"
+	envoyOptsKey = "_envoy_opts"
 )
 
 var (
 	errKeysMustBeStrings = errors.New("cannot convert nested map: all keys must be strings")
 )
+
+var (
+	protoPartial = protojson.UnmarshalOptions{AllowPartial: true, DiscardUnknown: true}
+)

config/custom.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"reflect"
 
-	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	"github.com/mitchellh/mapstructure"
 	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/proto"
@@ -117,14 +117,13 @@ func DecodeOptionsHookFunc() mapstructure.DecodeHookFunc {
 			if !ok {
 				continue
 			}
-			raw, ok := pm[healthCheckKey]
+
-			if ok {
+			envoyOpts, err := parseEnvoyClusterOpts(pm)
-				hc := new(envoy_config_core_v3.HealthCheck)
+			if err != nil {
-				if err := parseJSONPB(raw, hc); err != nil {
+				return nil, err
-					return nil, fmt.Errorf("%s: %w", healthCheckKey, err)
-				}
-				pm[healthCheckKey] = hc
 			}
+			pm[envoyOptsKey] = envoyOpts
+
 			rawTo, ok := pm[toKey]
 			if !ok {
 				continue
@@ -145,9 +144,20 @@ func DecodeOptionsHookFunc() mapstructure.DecodeHookFunc {
 	}
 }
 
+// parseEnvoyClusterOpts parses src as envoy cluster spec https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto
+// on top of some pre-filled default values
+func parseEnvoyClusterOpts(src interface{}) (*envoy_config_cluster_v3.Cluster, error) {
+	c := new(envoy_config_cluster_v3.Cluster)
+	if err := parseJSONPB(src, c, protoPartial); err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
 // parseJSONPB takes an intermediate representation and parses it using protobuf parser
 // that correctly handles oneof and other data types
-func parseJSONPB(raw interface{}, dst proto.Message) error {
+func parseJSONPB(raw interface{}, dst proto.Message, opts protojson.UnmarshalOptions) error {
 	ms, err := serializable(raw)
 	if err != nil {
 		return err
@@ -158,7 +168,7 @@ func parseJSONPB(raw interface{}, dst proto.Message) error {
 		return err
 	}
 
-	return protojson.Unmarshal(data, dst)
+	return opts.Unmarshal(data, dst)
 }
 
 // serializable converts mapstructure nested map into map[string]interface{} that is serializable to JSON

config/options.go

@@ -332,7 +332,7 @@ func NewDefaultOptions() *Options {
 func newOptionsFromConfig(configFile string) (*Options, error) {
 	o, err := optionsFromViper(configFile)
 	if err != nil {
-		return nil, fmt.Errorf("config: options from config file %w", err)
+		return nil, fmt.Errorf("config: options from config file %q: %w", configFile, err)
 	}
 	serviceName := telemetry.ServiceName(o.Services)
 	metrics.AddPolicyCountCallback(serviceName, func() int64 {

config/policy.go

@@ -13,7 +13,6 @@ import (
 	"time"
 
 	envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
-	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	"github.com/golang/protobuf/ptypes"
 
 	"github.com/pomerium/pomerium/internal/hashutil"
@@ -139,13 +138,9 @@ type Policy struct {
 	// to upstream requests.
 	EnableGoogleCloudServerlessAuthentication bool `mapstructure:"enable_google_cloud_serverless_authentication" yaml:"enable_google_cloud_serverless_authentication,omitempty"` //nolint
 
-	// OutlierDetection configures outlier detection for the upstream cluster.
-	OutlierDetection *PolicyOutlierDetection `mapstructure:"outlier_detection" yaml:"outlier_detection,omitempty" json:"outlier_detection,omitempty"`
-
-	// HealthCheck defines active health checks. See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/health_check.proto
-	HealthCheck *envoy_config_core_v3.HealthCheck `mapstructure:"health_check" yaml:"health_check,omitempty" json:"health_check,omitempty"`
-
 	SubPolicies []SubPolicy `mapstructure:"sub_policies" yaml:"sub_policies,omitempty" json:"sub_policies,omitempty"`
+
+	EnvoyOpts *envoy_config_cluster_v3.Cluster `mapstructure:"_envoy_opts" yaml:"-" json:"-"`
 }
 
 // A SubPolicy is a protobuf Policy within a protobuf Route.
@@ -221,30 +216,7 @@ func NewPolicyFromProto(pb *configpb.Route) (*Policy, error) {
 			StripQuery:     pb.Redirect.StripQuery,
 		}
 	}
-	if pb.OutlierDetection != nil {
+
-		p.OutlierDetection = &PolicyOutlierDetection{
-			Consecutive_5Xx:                        pb.OutlierDetection.Consecutive_5Xx,
-			Interval:                               pb.OutlierDetection.Interval,
-			BaseEjectionTime:                       pb.OutlierDetection.BaseEjectionTime,
-			MaxEjectionPercent:                     pb.OutlierDetection.MaxEjectionPercent,
-			EnforcingConsecutive_5Xx:               pb.OutlierDetection.EnforcingConsecutive_5Xx,
-			EnforcingSuccessRate:                   pb.OutlierDetection.EnforcingSuccessRate,
-			SuccessRateMinimumHosts:                pb.OutlierDetection.SuccessRateMinimumHosts,
-			SuccessRateRequestVolume:               pb.OutlierDetection.SuccessRateRequestVolume,
-			SuccessRateStdevFactor:                 pb.OutlierDetection.SuccessRateStdevFactor,
-			ConsecutiveGatewayFailure:              pb.OutlierDetection.ConsecutiveGatewayFailure,
-			EnforcingConsecutiveGatewayFailure:     pb.OutlierDetection.EnforcingConsecutiveGatewayFailure,
-			SplitExternalLocalOriginErrors:         pb.OutlierDetection.SplitExternalLocalOriginErrors,
-			ConsecutiveLocalOriginFailure:          pb.OutlierDetection.ConsecutiveLocalOriginFailure,
-			EnforcingConsecutiveLocalOriginFailure: pb.OutlierDetection.EnforcingConsecutiveLocalOriginFailure,
-			EnforcingLocalOriginSuccessRate:        pb.OutlierDetection.EnforcingLocalOriginSuccessRate,
-			FailurePercentageThreshold:             pb.OutlierDetection.FailurePercentageThreshold,
-			EnforcingFailurePercentage:             pb.OutlierDetection.EnforcingFailurePercentage,
-			EnforcingFailurePercentageLocalOrigin:  pb.OutlierDetection.EnforcingFailurePercentageLocalOrigin,
-			FailurePercentageMinimumHosts:          pb.OutlierDetection.FailurePercentageMinimumHosts,
-			FailurePercentageRequestVolume:         pb.OutlierDetection.FailurePercentageRequestVolume,
-		}
-	}
 	for _, sp := range pb.GetPolicies() {
 		p.SubPolicies = append(p.SubPolicies, SubPolicy{
 			ID:               sp.GetId(),
@@ -320,30 +292,7 @@ func (p *Policy) ToProto() *configpb.Route {
 			StripQuery:     p.Redirect.StripQuery,
 		}
 	}
-	if p.OutlierDetection != nil {
+
-		pb.OutlierDetection = &configpb.OutlierDetection{
-			Consecutive_5Xx:                        p.OutlierDetection.Consecutive_5Xx,
-			Interval:                               p.OutlierDetection.Interval,
-			BaseEjectionTime:                       p.OutlierDetection.BaseEjectionTime,
-			MaxEjectionPercent:                     p.OutlierDetection.MaxEjectionPercent,
-			EnforcingConsecutive_5Xx:               p.OutlierDetection.EnforcingConsecutive_5Xx,
-			EnforcingSuccessRate:                   p.OutlierDetection.EnforcingSuccessRate,
-			SuccessRateMinimumHosts:                p.OutlierDetection.SuccessRateMinimumHosts,
-			SuccessRateRequestVolume:               p.OutlierDetection.SuccessRateRequestVolume,
-			SuccessRateStdevFactor:                 p.OutlierDetection.SuccessRateStdevFactor,
-			ConsecutiveGatewayFailure:              p.OutlierDetection.ConsecutiveGatewayFailure,
-			EnforcingConsecutiveGatewayFailure:     p.OutlierDetection.EnforcingConsecutiveGatewayFailure,
-			SplitExternalLocalOriginErrors:         p.OutlierDetection.SplitExternalLocalOriginErrors,
-			ConsecutiveLocalOriginFailure:          p.OutlierDetection.ConsecutiveLocalOriginFailure,
-			EnforcingConsecutiveLocalOriginFailure: p.OutlierDetection.EnforcingConsecutiveLocalOriginFailure,
-			EnforcingLocalOriginSuccessRate:        p.OutlierDetection.EnforcingLocalOriginSuccessRate,
-			FailurePercentageThreshold:             p.OutlierDetection.FailurePercentageThreshold,
-			EnforcingFailurePercentage:             p.OutlierDetection.EnforcingFailurePercentage,
-			EnforcingFailurePercentageLocalOrigin:  p.OutlierDetection.EnforcingFailurePercentageLocalOrigin,
-			FailurePercentageMinimumHosts:          p.OutlierDetection.FailurePercentageMinimumHosts,
-			FailurePercentageRequestVolume:         p.OutlierDetection.FailurePercentageRequestVolume,
-		}
-	}
 	return pb
 }
 
@@ -433,12 +382,6 @@ func (p *Policy) Validate() error {
 		return fmt.Errorf("config: only prefix_rewrite or regex_rewrite_pattern can be specified, but not both")
 	}
 
-	if p.HealthCheck != nil {
-		if err := p.HealthCheck.Validate(); err != nil {
-			return err
-		}
-	}
-
 	return nil
 }
 

docs/docs/topics/load_balancing.md

@@ -0,0 +1,60 @@
+---
+title: Upstream Load Balancing
+description: >-
+  This article covers Pomerium built-in load balancing capabilities in presence of multiple upstreams.
+---
+
+# Upstream Load Balancing
+
+This article covers Pomerium built-in load balancing capabilities in presence of multiple upstreams.
+
+## Multiple Upstreams
+
+You may specify multiple servers for your upstream application, and Pomerium would load balance user requests between them.
+
+```yaml
+policy:
+  - from: https://myapp.localhost.pomerium.io
+    to:
+      - http://myapp-srv-1:8080
+      - http://myapp-srv-2:8080
+```
+
+::: tip
+    In presence of multiple upstreams, make sure to specify either an active or passive health check, or both, to avoid requests served to unhealthy backend.
+:::
+
+### Active Health Checks
+
+Active health checks issue periodic requests to each upstream to determine its health.
+See [Health Checking](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/health_checking) for a comprehensive overview.
+
+```yaml
+policy:
+  - from: https://myapp.localhost.pomerium.io
+    to:
+      - http://myapp-srv-1:8080
+      - http://myapp-srv-2:8080
+    health_checks:
+      - timeout: 10s
+        interval: 60s
+        healthy_threshold: 1
+        unhealthy_threshold: 2
+        http_health_check:
+          path: "/"
+```
+### Passive Health Checks
+
+Passive health check tries to deduce upstream server health based on recent observed responses.
+See [Outlier Detection](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/outlier) for a comprehensive overview.
+
+## Load Balancing Method
+
+`lb_policy` should be set to one of the values:
+
+- [`ROUND_ROBIN`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#weighted-round-robin) (default)
+- [`LEAST_REQUEST`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#weighted-least-request) and may be further configured using [``](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#envoy-v3-api-msg-config-cluster-v3-cluster-leastrequestlbconfig)
+- [`RING_HASH`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#ring-hash) and may be further configured using [`ring_hash_lb_config`](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#config-cluster-v3-cluster-ringhashlbconfig) option
+- [`RANDOM`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#random)
+- [`MAGLEV`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#maglev) and may be further configured using [`maglev_lb_config`](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#envoy-v3-api-msg-config-cluster-v3-cluster-maglevlbconfig) option
+

docs/reference/readme.md

@@ -1370,12 +1370,28 @@ When enabled, this option will pass identity headers to upstream applications. T
 If set, enables proxying of SPDY protocol upgrades.
 
 
-### Health Check
+### Load Balancing
-- Config File Key: `health_check`
+- Config File Key: `lb_policy`
-- Type: `object`
+- Type: `enum`
 - Optional
 
-When defined, will issue periodic health check requests to upstream servers.
+In presence of multiple upstreams, defines load balancing strategy between them.
+
+- [`ROUND_ROBIN`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#weighted-round-robin) (default)
+- [`LEAST_REQUEST`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#weighted-least-request) and may be further configured using [``](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#envoy-v3-api-msg-config-cluster-v3-cluster-leastrequestlbconfig)
+- [`RING_HASH`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#ring-hash) and may be further configured using [`ring_hash_lb_config`](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#config-cluster-v3-cluster-ringhashlbconfig) option
+- [`RANDOM`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#random)
+- [`MAGLEV`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#maglev) and may be further configured using [`maglev_lb_config`](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#envoy-v3-api-msg-config-cluster-v3-cluster-maglevlbconfig) option
+
+
+### Health Checks
+- Config File Key: `health_checks`
+- Type: `array of objects`
+- Optional
+
+When defined, will issue periodic health check requests to upstream servers. When health checks are defined, unhealthy upstream servers would not serve traffic.
+See also `outlier_detection` for automatic upstream server health detection.
+In presence of multiple upstream servers, it is recommended to set up either `health_checks` or `outlier_detection` or both.
 See [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/health_checking) for a list of supported parameters.
 
 

docs/reference/settings.yaml

@@ -1502,14 +1502,30 @@ settings:
           - Default: `false`
         doc: |
           If set, enables proxying of SPDY protocol upgrades.
-      - name: "Health Check"
+      - name: "Load Balancing"
-        keys: ["health_check"]
+        keys: ["lb_policy"]
         attributes: |
-          - Config File Key: `health_check`
+          - Config File Key: `lb_policy`
-          - Type: `object`
+          - Type: `enum`
           - Optional
         doc: |
-          When defined, will issue periodic health check requests to upstream servers.
+          In presence of multiple upstreams, defines load balancing strategy between them.
+
+          - [`ROUND_ROBIN`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#weighted-round-robin) (default)
+          - [`LEAST_REQUEST`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#weighted-least-request) and may be further configured using [``](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#envoy-v3-api-msg-config-cluster-v3-cluster-leastrequestlbconfig)
+          - [`RING_HASH`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#ring-hash) and may be further configured using [`ring_hash_lb_config`](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#config-cluster-v3-cluster-ringhashlbconfig) option
+          - [`RANDOM`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#random)
+          - [`MAGLEV`](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#maglev) and may be further configured using [`maglev_lb_config`](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#envoy-v3-api-msg-config-cluster-v3-cluster-maglevlbconfig) option
+      - name: "Health Checks"
+        keys: ["health_checks"]
+        attributes: |
+          - Config File Key: `health_checks`
+          - Type: `array of objects`
+          - Optional
+        doc: |
+          When defined, will issue periodic health check requests to upstream servers. When health checks are defined, unhealthy upstream servers would not serve traffic.
+          See also `outlier_detection` for automatic upstream server health detection.
+          In presence of multiple upstream servers, it is recommended to set up either `health_checks` or `outlier_detection` or both.
           See [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/health_checking) for a list of supported parameters.
       - name: "Websocket Connections"
         keys: ["allow_websockets"]

internal/cmd/pomerium/pomerium.go

@@ -64,8 +64,15 @@ func Run(ctx context.Context, configFile string) error {
 	if err != nil {
 		return fmt.Errorf("error creating control plane: %w", err)
 	}
-	src.OnConfigChange(controlPlane.OnConfigChange)
+	src.OnConfigChange(func(cfg *config.Config) {
-	controlPlane.OnConfigChange(src.GetConfig())
+		if err := controlPlane.OnConfigChange(cfg); err != nil {
+			log.Error().Err(err).Msg("config change")
+		}
+	})
+
+	if err = controlPlane.OnConfigChange(src.GetConfig()); err != nil {
+		return fmt.Errorf("applying config: %w", err)
+	}
 
 	_, grpcPort, _ := net.SplitHostPort(controlPlane.GRPCListener.Addr().String())
 	_, httpPort, _ := net.SplitHostPort(controlPlane.HTTPListener.Addr().String())

internal/controlplane/constants.go

@@ -0,0 +1,23 @@
+package controlplane
+
+import (
+	"errors"
+	"time"
+
+	envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	"github.com/golang/protobuf/ptypes"
+)
+
+var (
+	errNoEndpoints           = errors.New("cluster must have endpoints")
+	defaultConnectionTimeout = ptypes.DurationProto(time.Second * 10)
+)
+
+// newDefaultEnvoyClusterConfig creates envoy cluster with certain default values
+func newDefaultEnvoyClusterConfig() *envoy_config_cluster_v3.Cluster {
+	return &envoy_config_cluster_v3.Cluster{
+		ConnectTimeout:  defaultConnectionTimeout,
+		RespectDnsTtl:   true,
+		DnsLookupFamily: envoy_config_cluster_v3.Cluster_AUTO,
+	}
+}

internal/controlplane/server.go

@@ -88,7 +88,12 @@ func NewServer(name string) (*Server, error) {
 	srv.HTTPRouter = mux.NewRouter()
 	srv.addHTTPMiddleware()
 
-	srv.xdsmgr = xdsmgr.NewManager(srv.buildDiscoveryResources())
+	res, err := srv.buildDiscoveryResources()
+	if err != nil {
+		return nil, err
+	}
+
+	srv.xdsmgr = xdsmgr.NewManager(res)
 	envoy_service_discovery_v3.RegisterAggregatedDiscoveryServiceServer(srv.GRPCServer, srv.xdsmgr)
 
 	srv.filemgr = filemgr.NewManager()
@@ -158,11 +163,16 @@ func (srv *Server) Run(ctx context.Context) error {
 }
 
 // OnConfigChange updates the pomerium config options.
-func (srv *Server) OnConfigChange(cfg *config.Config) {
+func (srv *Server) OnConfigChange(cfg *config.Config) error {
 	prev := srv.currentConfig.Load()
 	srv.currentConfig.Store(versionedConfig{
 		Config:  cfg,
 		version: prev.version + 1,
 	})
-	srv.xdsmgr.Update(srv.buildDiscoveryResources())
+	res, err := srv.buildDiscoveryResources()
+	if err != nil {
+		return err
+	}
+	srv.xdsmgr.Update(res)
+	return nil
 }

internal/controlplane/xds.go

@@ -31,10 +31,15 @@ const (
 	listenerTypeURL = "type.googleapis.com/envoy.config.listener.v3.Listener"
 )
 
-func (srv *Server) buildDiscoveryResources() map[string][]*envoy_service_discovery_v3.Resource {
+func (srv *Server) buildDiscoveryResources() (map[string][]*envoy_service_discovery_v3.Resource, error) {
 	resources := map[string][]*envoy_service_discovery_v3.Resource{}
 	cfg := srv.currentConfig.Load()
-	for _, cluster := range srv.buildClusters(cfg.Options) {
+
+	clusters, err := srv.buildClusters(cfg.Options)
+	if err != nil {
+		return nil, err
+	}
+	for _, cluster := range clusters {
 		any, _ := anypb.New(cluster)
 		resources[clusterTypeURL] = append(resources[clusterTypeURL], &envoy_service_discovery_v3.Resource{
 			Name:     cluster.Name,
@@ -50,7 +55,7 @@ func (srv *Server) buildDiscoveryResources() map[string][]*envoy_service_discove
 			Resource: any,
 		})
 	}
-	return resources
+	return resources, nil
 }
 
 func buildAccessLogs(options *config.Options) []*envoy_config_accesslog_v3.AccessLog {

internal/controlplane/xds_cluster_test.go

@@ -8,6 +8,7 @@ import (
 
 	envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"google.golang.org/protobuf/types/known/wrapperspb"
 
 	"github.com/pomerium/pomerium/config"
@@ -212,9 +213,10 @@ func Test_buildCluster(t *testing.T) {
 		endpoints := srv.buildPolicyEndpoints(&config.Policy{
 			Destinations: mustParseURLs("http://example.com"),
 		})
-		cluster := buildCluster("example", endpoints, true,
+		cluster := newDefaultEnvoyClusterConfig()
-			config.GetEnvoyDNSLookupFamily(config.DNSLookupFamilyV4Only),
+		cluster.DnsLookupFamily = envoy_config_cluster_v3.Cluster_V4_ONLY
-			nil, nil)
+		err := buildCluster(cluster, "example", endpoints, true)
+		require.NoErrorf(t, err, "cluster %+v", cluster)
 		testutil.AssertProtoJSONEqual(t, `
 			{
 				"name": "example",
@@ -251,9 +253,9 @@ func Test_buildCluster(t *testing.T) {
 				"https://example.com",
 			),
 		})
-		cluster := buildCluster("example", endpoints, true,
+		cluster := newDefaultEnvoyClusterConfig()
-			config.GetEnvoyDNSLookupFamily(config.DNSLookupFamilyAuto),
+		err := buildCluster(cluster, "example", endpoints, true)
-			nil, nil)
+		require.NoErrorf(t, err, "cluster %+v", cluster)
 		testutil.AssertProtoJSONEqual(t, `
 			{
 				"name": "example",
@@ -342,9 +344,9 @@ func Test_buildCluster(t *testing.T) {
 		endpoints := srv.buildPolicyEndpoints(&config.Policy{
 			Destinations: mustParseURLs("http://127.0.0.1"),
 		})
-		cluster := buildCluster("example", endpoints, true,
+		cluster := newDefaultEnvoyClusterConfig()
-			config.GetEnvoyDNSLookupFamily(config.DNSLookupFamilyAuto),
+		err := buildCluster(cluster, "example", endpoints, true)
-			nil, nil)
+		require.NoErrorf(t, err, "cluster %+v", cluster)
 		testutil.AssertProtoJSONEqual(t, `
 			{
 				"name": "example",
@@ -377,9 +379,9 @@ func Test_buildCluster(t *testing.T) {
 		endpoints := srv.buildPolicyEndpoints(&config.Policy{
 			Destinations: mustParseURLs("http://localhost"),
 		})
-		cluster := buildCluster("example", endpoints, true,
+		cluster := newDefaultEnvoyClusterConfig()
-			config.GetEnvoyDNSLookupFamily(config.DNSLookupFamilyAuto),
+		err := buildCluster(cluster, "example", endpoints, true)
-			nil, nil)
+		require.NoErrorf(t, err, "cluster %+v", cluster)
 		testutil.AssertProtoJSONEqual(t, `
 			{
 				"name": "example",
@@ -412,12 +414,14 @@ func Test_buildCluster(t *testing.T) {
 		endpoints := srv.buildPolicyEndpoints(&config.Policy{
 			Destinations: mustParseURLs("http://example.com"),
 		})
-		cluster := buildCluster("example", endpoints, true,
+		cluster := newDefaultEnvoyClusterConfig()
-			config.GetEnvoyDNSLookupFamily(config.DNSLookupFamilyV4Only),
+		cluster.DnsLookupFamily = envoy_config_cluster_v3.Cluster_V4_ONLY
-			&envoy_config_cluster_v3.OutlierDetection{
+		cluster.OutlierDetection = &envoy_config_cluster_v3.OutlierDetection{
-				EnforcingConsecutive_5Xx:       wrapperspb.UInt32(17),
+			EnforcingConsecutive_5Xx:       wrapperspb.UInt32(17),
-				SplitExternalLocalOriginErrors: true,
+			SplitExternalLocalOriginErrors: true,
-			}, nil)
+		}
+		err := buildCluster(cluster, "example", endpoints, true)
+		require.NoErrorf(t, err, "cluster %+v", cluster)
 		testutil.AssertProtoJSONEqual(t, `
 			{
 				"name": "example",

internal/controlplane/xds_clusters.go

@@ -2,17 +2,16 @@ package controlplane
 
 import (
 	"encoding/base64"
+	"fmt"
 	"net"
 	"net/url"
 	"strings"
-	"time"
 
 	envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
 	envoy_extensions_transport_sockets_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	envoy_type_matcher_v3 "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
-	"github.com/golang/protobuf/ptypes"
 	"github.com/martinlindhe/base36"
 	"google.golang.org/protobuf/types/known/structpb"
 
@@ -42,7 +41,7 @@ func (e Endpoint) TransportSocketName() string {
 	return "ts-" + base36.EncodeBytes(h)
 }
 
-func (srv *Server) buildClusters(options *config.Options) []*envoy_config_cluster_v3.Cluster {
+func (srv *Server) buildClusters(options *config.Options) ([]*envoy_config_cluster_v3.Cluster, error) {
 	grpcURL := &url.URL{
 		Scheme: "http",
 		Host:   srv.GRPCListener.Addr().String(),
@@ -56,40 +55,73 @@ func (srv *Server) buildClusters(options *config.Options) []*envoy_config_cluste
 		Host:   options.GetAuthorizeURL().Host,
 	}
 
-	clusters := []*envoy_config_cluster_v3.Cluster{
+	controlGRPC, err := srv.buildInternalCluster(options, "pomerium-control-plane-grpc", grpcURL, true)
-		srv.buildInternalCluster(options, "pomerium-control-plane-grpc", grpcURL, true),
+	if err != nil {
-		srv.buildInternalCluster(options, "pomerium-control-plane-http", httpURL, false),
+		return nil, err
+	}
+	controlHTTP, err := srv.buildInternalCluster(options, "pomerium-control-plane-http", httpURL, false)
+	if err != nil {
+		return nil, err
+	}
+	authZ, err := srv.buildInternalCluster(options, authzURL.Host, authzURL, true)
+	if err != nil {
+		return nil, err
 	}
 
-	clusters = append(clusters, srv.buildInternalCluster(options, authzURL.Host, authzURL, true))
+	clusters := []*envoy_config_cluster_v3.Cluster{
+		controlGRPC,
+		controlHTTP,
+		authZ,
+	}
 
 	if config.IsProxy(options.Services) {
 		for i := range options.Policies {
 			policy := options.Policies[i]
+			if policy.EnvoyOpts == nil {
+				policy.EnvoyOpts = newDefaultEnvoyClusterConfig()
+			}
 			if len(policy.Destinations) > 0 {
-				clusters = append(clusters, srv.buildPolicyCluster(options, &policy))
+				cluster, err := srv.buildPolicyCluster(options, &policy)
+				if err != nil {
+					return nil, fmt.Errorf("policy #%d: %w", i, err)
+				}
+				clusters = append(clusters, cluster)
 			}
 		}
 	}
 
-	return clusters
+	return clusters, nil
 }
 
-func (srv *Server) buildInternalCluster(options *config.Options, name string, dst *url.URL, forceHTTP2 bool) *envoy_config_cluster_v3.Cluster {
+func (srv *Server) buildInternalCluster(options *config.Options, name string, dst *url.URL, forceHTTP2 bool) (*envoy_config_cluster_v3.Cluster, error) {
+	cluster := newDefaultEnvoyClusterConfig()
+	cluster.DnsLookupFamily = config.GetEnvoyDNSLookupFamily(options.DNSLookupFamily)
 	endpoints := []Endpoint{NewEndpoint(dst, srv.buildInternalTransportSocket(options, dst))}
-	dnsLookupFamily := config.GetEnvoyDNSLookupFamily(options.DNSLookupFamily)
+	if err := buildCluster(cluster, name, endpoints, forceHTTP2); err != nil {
-	return buildCluster(name, endpoints, forceHTTP2, dnsLookupFamily, nil, nil)
+		return nil, err
+	}
+	return cluster, nil
 }
 
-func (srv *Server) buildPolicyCluster(options *config.Options, policy *config.Policy) *envoy_config_cluster_v3.Cluster {
+func (srv *Server) buildPolicyCluster(options *config.Options, policy *config.Policy) (*envoy_config_cluster_v3.Cluster, error) {
+	cluster := policy.EnvoyOpts
+
 	name := getPolicyName(policy)
 	endpoints := srv.buildPolicyEndpoints(policy)
-	dnsLookupFamily := config.GetEnvoyDNSLookupFamily(options.DNSLookupFamily)
+
-	if policy.EnableGoogleCloudServerlessAuthentication {
+	if cluster.DnsLookupFamily == envoy_config_cluster_v3.Cluster_AUTO {
-		dnsLookupFamily = envoy_config_cluster_v3.Cluster_V4_ONLY
+		cluster.DnsLookupFamily = config.GetEnvoyDNSLookupFamily(options.DNSLookupFamily)
 	}
-	outlierDetection := (*envoy_config_cluster_v3.OutlierDetection)(policy.OutlierDetection)
+
-	return buildCluster(name, endpoints, false, dnsLookupFamily, outlierDetection, policy.HealthCheck)
+	if policy.EnableGoogleCloudServerlessAuthentication {
+		cluster.DnsLookupFamily = envoy_config_cluster_v3.Cluster_V4_ONLY
+	}
+
+	if err := buildCluster(cluster, name, endpoints, false); err != nil {
+		return nil, err
+	}
+
+	return cluster, nil
 }
 
 func (srv *Server) buildPolicyEndpoints(policy *config.Policy) []Endpoint {
@@ -230,36 +262,28 @@ func (srv *Server) buildPolicyValidationContext(policy *config.Policy, dst *url.
 }
 
 func buildCluster(
+	cluster *envoy_config_cluster_v3.Cluster,
 	name string,
 	endpoints []Endpoint,
 	forceHTTP2 bool,
-	dnsLookupFamily envoy_config_cluster_v3.Cluster_DnsLookupFamily,
+) error {
-	outlierDetection *envoy_config_cluster_v3.OutlierDetection,
-	healthCheck *envoy_config_core_v3.HealthCheck,
-) *envoy_config_cluster_v3.Cluster {
 	if len(endpoints) == 0 {
-		return nil
+		return errNoEndpoints
 	}
 
+	if cluster.ConnectTimeout == nil {
+		cluster.ConnectTimeout = defaultConnectionTimeout
+	}
+	cluster.RespectDnsTtl = true
 	lbEndpoints := buildLbEndpoints(endpoints)
-	cluster := &envoy_config_cluster_v3.Cluster{
+	cluster.Name = name
-		Name:           name,
+	cluster.LoadAssignment = &envoy_config_endpoint_v3.ClusterLoadAssignment{
-		ConnectTimeout: ptypes.DurationProto(time.Second * 10),
+		ClusterName: name,
-		LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{
+		Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{{
-			ClusterName: name,
+			LbEndpoints: lbEndpoints,
-			Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{{
+		}},
-				LbEndpoints: lbEndpoints,
-			}},
-		},
-		RespectDnsTtl:          true,
-		TransportSocketMatches: buildTransportSocketMatches(endpoints),
-		DnsLookupFamily:        dnsLookupFamily,
-		OutlierDetection:       outlierDetection,
-	}
-
-	if healthCheck != nil {
-		cluster.HealthChecks = append(cluster.HealthChecks, healthCheck)
 	}
+	cluster.TransportSocketMatches = buildTransportSocketMatches(endpoints)
 
 	if forceHTTP2 {
 		cluster.Http2ProtocolOptions = &envoy_config_core_v3.Http2ProtocolOptions{
@@ -280,7 +304,7 @@ func buildCluster(
 		cluster.ClusterDiscoveryType = &envoy_config_cluster_v3.Cluster_Type{Type: envoy_config_cluster_v3.Cluster_STRICT_DNS}
 	}
 
-	return cluster
+	return cluster.Validate()
 }
 
 func buildLbEndpoints(endpoints []Endpoint) []*envoy_config_endpoint_v3.LbEndpoint {