config: disable envoy admin by default, expose stats via envoy route (#3677)

config/envoyconfig/bootstrap.go

@@ -3,10 +3,12 @@ package envoyconfig
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 
 	envoy_config_accesslog_v3 "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3"
 	envoy_config_bootstrap_v3 "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
 	envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
 	envoy_config_metrics_v3 "github.com/envoyproxy/go-control-plane/envoy/config/metrics/v3"
 	envoy_extensions_access_loggers_file_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/file/v3"
@@ -17,15 +19,25 @@ import (
 	"github.com/pomerium/pomerium/internal/telemetry"
 )
 
+var (
+	envoyAdminAddressPath = filepath.Join(os.TempDir(), "pomerium-envoy-admin.sock")
+	envoyAdminAddressMode = 0o600
+	envoyAdminClusterName = "pomerium-envoy-admin"
+)
+
 // BuildBootstrapAdmin builds the admin config for the envoy bootstrap.
 func (b *Builder) BuildBootstrapAdmin(cfg *config.Config) (admin *envoy_config_bootstrap_v3.Admin, err error) {
 	admin = &envoy_config_bootstrap_v3.Admin{
 		ProfilePath: cfg.Options.EnvoyAdminProfilePath,
 	}
 
-	admin.Address, err = parseAddress(cfg.Options.EnvoyAdminAddress)
-	if err != nil {
-		return nil, fmt.Errorf("envoyconfig: invalid envoy admin address: %w", err)
+	admin.Address = &envoy_config_core_v3.Address{
+		Address: &envoy_config_core_v3.Address_Pipe{
+			Pipe: &envoy_config_core_v3.Pipe{
+				Path: envoyAdminAddressPath,
+				Mode: uint32(envoyAdminAddressMode),
+			},
+		},
 	}
 
 	if cfg.Options.EnvoyAdminAccessLogPath != os.DevNull && cfg.Options.EnvoyAdminAccessLogPath != "" {

config/envoyconfig/bootstrap_test.go

@@ -22,22 +22,14 @@ func TestBuilder_BuildBootstrapAdmin(t *testing.T) {
 		testutil.AssertProtoJSONEqual(t, `
 			{
 				"address": {
-					"socketAddress": {
-						"address": "127.0.0.1",
-						"portValue": 9901
+					"pipe": {
+						"mode": 384,
+						"path": "`+envoyAdminAddressPath+`"
 					}
 				}
 			}
 		`, adminCfg)
 	})
-	t.Run("bad address", func(t *testing.T) {
-		_, err := b.BuildBootstrapAdmin(&config.Config{
-			Options: &config.Options{
-				EnvoyAdminAddress: "xyz1234:zyx4321",
-			},
-		})
-		assert.Error(t, err)
-	})
 }
 
 func TestBuilder_BuildBootstrapLayeredRuntime(t *testing.T) {

config/envoyconfig/clusters.go

@@ -79,6 +79,11 @@ func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*env
 		authorizeCluster.OutlierDetection = grpcAuthorizeOutlierDetection()
 	}
 
+	envoyAdminCluster, err := b.buildEnvoyAdminCluster(ctx, cfg)
+	if err != nil {
+		return nil, err
+	}
+
 	clusters := []*envoy_config_cluster_v3.Cluster{
 		b.buildACMETLSALPNCluster(cfg),
 		controlGRPC,
@@ -86,6 +91,7 @@ func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*env
 		controlMetrics,
 		authorizeCluster,
 		databrokerCluster,
+		envoyAdminCluster,
 	}
 
 	tracingCluster, err := buildTracingCluster(cfg.Options)

config/envoyconfig/clusters_envoy_admin.go

@@ -0,0 +1,36 @@
+package envoyconfig
+
+import (
+	"context"
+
+	envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+
+	"github.com/pomerium/pomerium/config"
+)
+
+func (b *Builder) buildEnvoyAdminCluster(ctx context.Context, cfg *config.Config) (*envoy_config_cluster_v3.Cluster, error) {
+	return &envoy_config_cluster_v3.Cluster{
+		Name:           envoyAdminClusterName,
+		ConnectTimeout: defaultConnectionTimeout,
+		LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{
+			ClusterName: envoyAdminClusterName,
+			Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{{
+				LbEndpoints: []*envoy_config_endpoint_v3.LbEndpoint{{
+					HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{
+						Endpoint: &envoy_config_endpoint_v3.Endpoint{
+							Address: &envoy_config_core_v3.Address{
+								Address: &envoy_config_core_v3.Address_Pipe{
+									Pipe: &envoy_config_core_v3.Pipe{
+										Path: envoyAdminAddressPath,
+									},
+								},
+							},
+						},
+					},
+				}},
+			}},
+		},
+	}, nil
+}

config/envoyconfig/listeners.go

@@ -24,6 +24,7 @@ import (
 	"github.com/pomerium/pomerium/internal/hashutil"
 	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/internal/sets"
+	"github.com/pomerium/pomerium/internal/telemetry/metrics"
 	"github.com/pomerium/pomerium/pkg/cryptutil"
 )
 
@@ -80,6 +81,14 @@ func (b *Builder) BuildListeners(ctx context.Context, cfg *config.Config) ([]*en
 		listeners = append(listeners, li)
 	}
 
+	if cfg.Options.EnvoyAdminAddress != "" {
+		li, err := b.buildEnvoyAdminListener(ctx, cfg)
+		if err != nil {
+			return nil, err
+		}
+		listeners = append(listeners, li)
+	}
+
 	li, err := b.buildOutboundListener(cfg)
 	if err != nil {
 		return nil, err
@@ -381,19 +390,35 @@ func (b *Builder) buildMetricsHTTPConnectionManagerFilter() (*envoy_config_liste
 	rc, err := b.buildRouteConfiguration("metrics", []*envoy_config_route_v3.VirtualHost{{
 		Name:    "metrics",
 		Domains: []string{"*"},
-		Routes: []*envoy_config_route_v3.Route{{
-			Name: "metrics",
-			Match: &envoy_config_route_v3.RouteMatch{
-				PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: "/"},
-			},
-			Action: &envoy_config_route_v3.Route_Route{
-				Route: &envoy_config_route_v3.RouteAction{
-					ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
-						Cluster: "pomerium-control-plane-metrics",
+		Routes: []*envoy_config_route_v3.Route{
+			{
+				Name: "envoy-metrics",
+				Match: &envoy_config_route_v3.RouteMatch{
+					PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: metrics.EnvoyMetricsPath},
+				},
+				Action: &envoy_config_route_v3.Route_Route{
+					Route: &envoy_config_route_v3.RouteAction{
+						ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
+							Cluster: envoyAdminClusterName,
+						},
+						PrefixRewrite: "/stats/prometheus",
 					},
 				},
 			},
-		}},
+			{
+				Name: "metrics",
+				Match: &envoy_config_route_v3.RouteMatch{
+					PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: "/"},
+				},
+				Action: &envoy_config_route_v3.Route_Route{
+					Route: &envoy_config_route_v3.RouteAction{
+						ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
+							Cluster: "pomerium-control-plane-metrics",
+						},
+					},
+				},
+			},
+		},
 	}})
 	if err != nil {
 		return nil, err

config/envoyconfig/listeners_envoy_admin.go

@@ -0,0 +1,71 @@
+package envoyconfig
+
+import (
+	"context"
+	"fmt"
+
+	envoy_config_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	envoy_config_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	envoy_http_connection_manager "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+
+	"github.com/pomerium/pomerium/config"
+)
+
+func (b *Builder) buildEnvoyAdminListener(ctx context.Context, cfg *config.Config) (*envoy_config_listener_v3.Listener, error) {
+	filter, err := b.buildEnvoyAdminHTTPConnectionManagerFilter()
+	if err != nil {
+		return nil, err
+	}
+
+	filterChain := &envoy_config_listener_v3.FilterChain{
+		Filters: []*envoy_config_listener_v3.Filter{
+			filter,
+		},
+	}
+
+	addr, err := parseAddress(cfg.Options.EnvoyAdminAddress)
+	if err != nil {
+		return nil, fmt.Errorf("envoy_admin_addr %s: %w", cfg.Options.EnvoyAdminAddress, err)
+	}
+
+	li := newEnvoyListener("envoy-admin")
+	li.Address = addr
+	li.FilterChains = []*envoy_config_listener_v3.FilterChain{filterChain}
+	return li, nil
+}
+
+func (b *Builder) buildEnvoyAdminHTTPConnectionManagerFilter() (*envoy_config_listener_v3.Filter, error) {
+	rc, err := b.buildRouteConfiguration("envoy-admin", []*envoy_config_route_v3.VirtualHost{{
+		Name:    "envoy-admin",
+		Domains: []string{"*"},
+		Routes: []*envoy_config_route_v3.Route{
+			{
+				Name: "envoy-admin",
+				Match: &envoy_config_route_v3.RouteMatch{
+					PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: "/"},
+				},
+				Action: &envoy_config_route_v3.Route_Route{
+					Route: &envoy_config_route_v3.RouteAction{
+						ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
+							Cluster: envoyAdminClusterName,
+						},
+					},
+				},
+			},
+		},
+	}})
+	if err != nil {
+		return nil, err
+	}
+
+	return HTTPConnectionManagerFilter(&envoy_http_connection_manager.HttpConnectionManager{
+		CodecType:  envoy_http_connection_manager.HttpConnectionManager_AUTO,
+		StatPrefix: "envoy-admin",
+		RouteSpecifier: &envoy_http_connection_manager.HttpConnectionManager_RouteConfig{
+			RouteConfig: rc,
+		},
+		HttpFilters: []*envoy_http_connection_manager.HttpFilter{
+			HTTPRouterFilter(),
+		},
+	}), nil
+}

config/envoyconfig/listeners_test.go

@@ -63,15 +63,27 @@ func Test_buildMetricsHTTPConnectionManagerFilter(t *testing.T) {
 					"virtualHosts": [{
 						"name": "metrics",
 						"domains": ["*"],
-						"routes": [{
-							"name": "metrics",
-							"match": {
-								"prefix": "/"
+						"routes": [
+							{
+								"name": "envoy-metrics",
+								"match": {
+									"prefix": "/metrics/envoy"
+								},
+								"route": {
+									"cluster": "pomerium-envoy-admin",
+									"prefixRewrite": "/stats/prometheus"
+								}
 							},
-							"route": {
-								"cluster": "pomerium-control-plane-metrics"
+							{
+								"name": "metrics",
+								"match": {
+									"prefix": "/"
+								},
+								"route": {
+									"cluster": "pomerium-control-plane-metrics"
+								}
 							}
-						}]
+						]
 					}]
 				},
 				"statPrefix": "metrics"

config/metrics.go

@@ -24,14 +24,13 @@ const (
 
 // A MetricsManager manages metrics for a given configuration.
 type MetricsManager struct {
-	mu                sync.RWMutex
-	installationID    string
-	serviceName       string
-	addr              string
-	basicAuth         string
-	envoyAdminAddress string
-	handler           http.Handler
-	endpoints         []MetricsScrapeEndpoint
+	mu             sync.RWMutex
+	installationID string
+	serviceName    string
+	addr           string
+	basicAuth      string
+	handler        http.Handler
+	endpoints      []MetricsScrapeEndpoint
 }
 
 // NewMetricsManager creates a new MetricsManager.
@@ -95,7 +94,6 @@ func (mgr *MetricsManager) updateServer(ctx context.Context, cfg *Config) {
 	mgr.addr = cfg.Options.MetricsAddr
 	mgr.basicAuth = cfg.Options.MetricsBasicAuth
 	mgr.installationID = cfg.Options.InstallationID
-	mgr.envoyAdminAddress = cfg.Options.EnvoyAdminAddress
 	mgr.handler = nil
 
 	if mgr.addr == "" {
@@ -106,7 +104,7 @@ func (mgr *MetricsManager) updateServer(ctx context.Context, cfg *Config) {
 	mgr.endpoints = append(cfg.MetricsScrapeEndpoints,
 		MetricsScrapeEndpoint{
 			Name: "envoy",
-			URL:  url.URL{Scheme: "http", Host: cfg.Options.EnvoyAdminAddress, Path: "/stats/prometheus"},
+			URL:  url.URL{Scheme: "http", Host: cfg.Options.MetricsAddr, Path: "/metrics/envoy"},
 		})
 	handler, err := metrics.PrometheusHandler(toInternalEndpoints(mgr.endpoints), mgr.installationID, defaultMetricsTimeout)
 	if err != nil {
@@ -125,7 +123,6 @@ func (mgr *MetricsManager) configUnchanged(cfg *Config) bool {
 	return cfg.Options.MetricsAddr == mgr.addr &&
 		cfg.Options.MetricsBasicAuth == mgr.basicAuth &&
 		cfg.Options.InstallationID == mgr.installationID &&
-		cfg.Options.EnvoyAdminAddress == mgr.envoyAdminAddress &&
 		reflect.DeepEqual(mgr.endpoints, cfg.MetricsScrapeEndpoints)
 }
 

config/options.go

@@ -345,7 +345,6 @@ var defaultOptions = Options{
 	XffNumTrustedHops:                   0,
 	EnvoyAdminAccessLogPath:             os.DevNull,
 	EnvoyAdminProfilePath:               os.DevNull,
-	EnvoyAdminAddress:                   "127.0.0.1:9901",
 	ProgrammaticRedirectDomainWhitelist: []string{"localhost"},
 }
 

config/options_test.go

@@ -316,7 +316,6 @@ func TestOptionsFromViper(t *testing.T) {
 				DataBrokerStorageType:    "memory",
 				EnvoyAdminAccessLogPath:  os.DevNull,
 				EnvoyAdminProfilePath:    os.DevNull,
-				EnvoyAdminAddress:        "127.0.0.1:9901",
 			},
 			false,
 		},
@@ -337,7 +336,6 @@ func TestOptionsFromViper(t *testing.T) {
 				DataBrokerStorageType:    "memory",
 				EnvoyAdminAccessLogPath:  os.DevNull,
 				EnvoyAdminProfilePath:    os.DevNull,
-				EnvoyAdminAddress:        "127.0.0.1:9901",
 			},
 			false,
 		},
@@ -414,7 +412,7 @@ func Test_AutoCertOptionsFromEnvVar(t *testing.T) {
 		cleanup  func()
 	}
 
-	var tests = map[string]func(t *testing.T) test{
+	tests := map[string]func(t *testing.T) test{
 		"ok/simple": func(t *testing.T) test {
 			envs := map[string]string{
 				"AUTOCERT":             "true",
@@ -689,6 +687,7 @@ func TestOptions_GetOauthOptions(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, u.Hostname(), oauthOptions.RedirectURL.Hostname())
 }
+
 func TestOptions_GetAllRouteableGRPCDomains(t *testing.T) {
 	opts := &Options{
 		AuthenticateURLString: "https://authenticate.example.com",

internal/telemetry/metrics/providers.go

@@ -25,6 +25,9 @@ import (
 	"github.com/pomerium/pomerium/pkg/metrics"
 )
 
+// EnvoyMetricsPath is the path on the metrics listener that retrieves envoy metrics.
+const EnvoyMetricsPath = "/metrics/envoy"
+
 // ScrapeEndpoint external endpoints to scrape and decorate
 type ScrapeEndpoint struct {
 	// Name is the logical name of the endpoint
@@ -208,6 +211,7 @@ func ocExport(name string, exporter *ocprom.Exporter, r *http.Request, labels []
 		}
 	}
 }
+
 func scrapeEndpoints(endpoints []ScrapeEndpoint, labels []*io_prometheus_client.LabelPair) []promProducerFn {
 	out := make([]promProducerFn, 0, len(endpoints))
 	for _, endpoint := range endpoints {