From f44c85880b2f12ddec6685a792862005e1b5b434 Mon Sep 17 00:00:00 2001 From: "backport-actions-token[bot]" <87506591+backport-actions-token[bot]@users.noreply.github.com> Date: Tue, 18 Oct 2022 16:31:56 -0600 Subject: [PATCH] config: disable envoy admin by default, expose stats via envoy route (#3684) config: disable envoy admin by default, expose stats via envoy route (#3677) Co-authored-by: Caleb Doxsey --- config/envoyconfig/bootstrap.go | 18 +++++- config/envoyconfig/bootstrap_test.go | 14 +--- config/envoyconfig/clusters.go | 6 ++ config/envoyconfig/clusters_envoy_admin.go | 36 +++++++++++ config/envoyconfig/listeners.go | 45 ++++++++++--- config/envoyconfig/listeners_envoy_admin.go | 71 +++++++++++++++++++++ config/envoyconfig/listeners_test.go | 26 ++++++-- config/metrics.go | 19 +++--- config/options.go | 1 - config/options_test.go | 5 +- internal/telemetry/metrics/providers.go | 4 ++ 11 files changed, 199 insertions(+), 46 deletions(-) create mode 100644 config/envoyconfig/clusters_envoy_admin.go create mode 100644 config/envoyconfig/listeners_envoy_admin.go diff --git a/config/envoyconfig/bootstrap.go b/config/envoyconfig/bootstrap.go index eecfce132..9166d52fa 100644 --- a/config/envoyconfig/bootstrap.go +++ b/config/envoyconfig/bootstrap.go @@ -3,10 +3,12 @@ package envoyconfig import ( "fmt" "os" + "path/filepath" envoy_config_accesslog_v3 "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3" envoy_config_bootstrap_v3 "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3" envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" envoy_config_metrics_v3 "github.com/envoyproxy/go-control-plane/envoy/config/metrics/v3" envoy_extensions_access_loggers_file_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/file/v3" @@ -17,15 +19,25 @@ import ( "github.com/pomerium/pomerium/internal/telemetry" ) +var ( + envoyAdminAddressPath = filepath.Join(os.TempDir(), "pomerium-envoy-admin.sock") + envoyAdminAddressMode = 0o600 + envoyAdminClusterName = "pomerium-envoy-admin" +) + // BuildBootstrapAdmin builds the admin config for the envoy bootstrap. func (b *Builder) BuildBootstrapAdmin(cfg *config.Config) (admin *envoy_config_bootstrap_v3.Admin, err error) { admin = &envoy_config_bootstrap_v3.Admin{ ProfilePath: cfg.Options.EnvoyAdminProfilePath, } - admin.Address, err = parseAddress(cfg.Options.EnvoyAdminAddress) - if err != nil { - return nil, fmt.Errorf("envoyconfig: invalid envoy admin address: %w", err) + admin.Address = &envoy_config_core_v3.Address{ + Address: &envoy_config_core_v3.Address_Pipe{ + Pipe: &envoy_config_core_v3.Pipe{ + Path: envoyAdminAddressPath, + Mode: uint32(envoyAdminAddressMode), + }, + }, } if cfg.Options.EnvoyAdminAccessLogPath != os.DevNull && cfg.Options.EnvoyAdminAccessLogPath != "" { diff --git a/config/envoyconfig/bootstrap_test.go b/config/envoyconfig/bootstrap_test.go index 4b18056c7..d80caad85 100644 --- a/config/envoyconfig/bootstrap_test.go +++ b/config/envoyconfig/bootstrap_test.go @@ -22,22 +22,14 @@ func TestBuilder_BuildBootstrapAdmin(t *testing.T) { testutil.AssertProtoJSONEqual(t, ` { "address": { - "socketAddress": { - "address": "127.0.0.1", - "portValue": 9901 + "pipe": { + "mode": 384, + "path": "`+envoyAdminAddressPath+`" } } } `, adminCfg) }) - t.Run("bad address", func(t *testing.T) { - _, err := b.BuildBootstrapAdmin(&config.Config{ - Options: &config.Options{ - EnvoyAdminAddress: "xyz1234:zyx4321", - }, - }) - assert.Error(t, err) - }) } func TestBuilder_BuildBootstrapLayeredRuntime(t *testing.T) { diff --git a/config/envoyconfig/clusters.go b/config/envoyconfig/clusters.go index 28888fb55..4357d9ed6 100644 --- a/config/envoyconfig/clusters.go +++ b/config/envoyconfig/clusters.go @@ -79,6 +79,11 @@ func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*env authorizeCluster.OutlierDetection = grpcAuthorizeOutlierDetection() } + envoyAdminCluster, err := b.buildEnvoyAdminCluster(ctx, cfg) + if err != nil { + return nil, err + } + clusters := []*envoy_config_cluster_v3.Cluster{ b.buildACMETLSALPNCluster(cfg), controlGRPC, @@ -86,6 +91,7 @@ func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*env controlMetrics, authorizeCluster, databrokerCluster, + envoyAdminCluster, } tracingCluster, err := buildTracingCluster(cfg.Options) diff --git a/config/envoyconfig/clusters_envoy_admin.go b/config/envoyconfig/clusters_envoy_admin.go new file mode 100644 index 000000000..feda13be0 --- /dev/null +++ b/config/envoyconfig/clusters_envoy_admin.go @@ -0,0 +1,36 @@ +package envoyconfig + +import ( + "context" + + envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + + "github.com/pomerium/pomerium/config" +) + +func (b *Builder) buildEnvoyAdminCluster(ctx context.Context, cfg *config.Config) (*envoy_config_cluster_v3.Cluster, error) { + return &envoy_config_cluster_v3.Cluster{ + Name: envoyAdminClusterName, + ConnectTimeout: defaultConnectionTimeout, + LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{ + ClusterName: envoyAdminClusterName, + Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{{ + LbEndpoints: []*envoy_config_endpoint_v3.LbEndpoint{{ + HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{ + Endpoint: &envoy_config_endpoint_v3.Endpoint{ + Address: &envoy_config_core_v3.Address{ + Address: &envoy_config_core_v3.Address_Pipe{ + Pipe: &envoy_config_core_v3.Pipe{ + Path: envoyAdminAddressPath, + }, + }, + }, + }, + }, + }}, + }}, + }, + }, nil +} diff --git a/config/envoyconfig/listeners.go b/config/envoyconfig/listeners.go index c19cffad4..7c331a0c6 100644 --- a/config/envoyconfig/listeners.go +++ b/config/envoyconfig/listeners.go @@ -25,6 +25,7 @@ import ( "github.com/pomerium/pomerium/internal/hashutil" "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/sets" + "github.com/pomerium/pomerium/internal/telemetry/metrics" "github.com/pomerium/pomerium/pkg/cryptutil" ) @@ -81,6 +82,14 @@ func (b *Builder) BuildListeners(ctx context.Context, cfg *config.Config) ([]*en listeners = append(listeners, li) } + if cfg.Options.EnvoyAdminAddress != "" { + li, err := b.buildEnvoyAdminListener(ctx, cfg) + if err != nil { + return nil, err + } + listeners = append(listeners, li) + } + li, err := b.buildOutboundListener(cfg) if err != nil { return nil, err @@ -394,19 +403,35 @@ func (b *Builder) buildMetricsHTTPConnectionManagerFilter() (*envoy_config_liste rc, err := b.buildRouteConfiguration("metrics", []*envoy_config_route_v3.VirtualHost{{ Name: "metrics", Domains: []string{"*"}, - Routes: []*envoy_config_route_v3.Route{{ - Name: "metrics", - Match: &envoy_config_route_v3.RouteMatch{ - PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: "/"}, - }, - Action: &envoy_config_route_v3.Route_Route{ - Route: &envoy_config_route_v3.RouteAction{ - ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{ - Cluster: "pomerium-control-plane-metrics", + Routes: []*envoy_config_route_v3.Route{ + { + Name: "envoy-metrics", + Match: &envoy_config_route_v3.RouteMatch{ + PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: metrics.EnvoyMetricsPath}, + }, + Action: &envoy_config_route_v3.Route_Route{ + Route: &envoy_config_route_v3.RouteAction{ + ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{ + Cluster: envoyAdminClusterName, + }, + PrefixRewrite: "/stats/prometheus", }, }, }, - }}, + { + Name: "metrics", + Match: &envoy_config_route_v3.RouteMatch{ + PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &envoy_config_route_v3.Route_Route{ + Route: &envoy_config_route_v3.RouteAction{ + ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{ + Cluster: "pomerium-control-plane-metrics", + }, + }, + }, + }, + }, }}) if err != nil { return nil, err diff --git a/config/envoyconfig/listeners_envoy_admin.go b/config/envoyconfig/listeners_envoy_admin.go new file mode 100644 index 000000000..306d0f267 --- /dev/null +++ b/config/envoyconfig/listeners_envoy_admin.go @@ -0,0 +1,71 @@ +package envoyconfig + +import ( + "context" + "fmt" + + envoy_config_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + envoy_config_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + envoy_http_connection_manager "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + + "github.com/pomerium/pomerium/config" +) + +func (b *Builder) buildEnvoyAdminListener(ctx context.Context, cfg *config.Config) (*envoy_config_listener_v3.Listener, error) { + filter, err := b.buildEnvoyAdminHTTPConnectionManagerFilter() + if err != nil { + return nil, err + } + + filterChain := &envoy_config_listener_v3.FilterChain{ + Filters: []*envoy_config_listener_v3.Filter{ + filter, + }, + } + + addr, err := parseAddress(cfg.Options.EnvoyAdminAddress) + if err != nil { + return nil, fmt.Errorf("envoy_admin_addr %s: %w", cfg.Options.EnvoyAdminAddress, err) + } + + li := newEnvoyListener("envoy-admin") + li.Address = addr + li.FilterChains = []*envoy_config_listener_v3.FilterChain{filterChain} + return li, nil +} + +func (b *Builder) buildEnvoyAdminHTTPConnectionManagerFilter() (*envoy_config_listener_v3.Filter, error) { + rc, err := b.buildRouteConfiguration("envoy-admin", []*envoy_config_route_v3.VirtualHost{{ + Name: "envoy-admin", + Domains: []string{"*"}, + Routes: []*envoy_config_route_v3.Route{ + { + Name: "envoy-admin", + Match: &envoy_config_route_v3.RouteMatch{ + PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &envoy_config_route_v3.Route_Route{ + Route: &envoy_config_route_v3.RouteAction{ + ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{ + Cluster: envoyAdminClusterName, + }, + }, + }, + }, + }, + }}) + if err != nil { + return nil, err + } + + return HTTPConnectionManagerFilter(&envoy_http_connection_manager.HttpConnectionManager{ + CodecType: envoy_http_connection_manager.HttpConnectionManager_AUTO, + StatPrefix: "envoy-admin", + RouteSpecifier: &envoy_http_connection_manager.HttpConnectionManager_RouteConfig{ + RouteConfig: rc, + }, + HttpFilters: []*envoy_http_connection_manager.HttpFilter{ + HTTPRouterFilter(), + }, + }), nil +} diff --git a/config/envoyconfig/listeners_test.go b/config/envoyconfig/listeners_test.go index 97cfb516e..1f91e6a19 100644 --- a/config/envoyconfig/listeners_test.go +++ b/config/envoyconfig/listeners_test.go @@ -63,15 +63,27 @@ func Test_buildMetricsHTTPConnectionManagerFilter(t *testing.T) { "virtualHosts": [{ "name": "metrics", "domains": ["*"], - "routes": [{ - "name": "metrics", - "match": { - "prefix": "/" + "routes": [ + { + "name": "envoy-metrics", + "match": { + "prefix": "/metrics/envoy" + }, + "route": { + "cluster": "pomerium-envoy-admin", + "prefixRewrite": "/stats/prometheus" + } }, - "route": { - "cluster": "pomerium-control-plane-metrics" + { + "name": "metrics", + "match": { + "prefix": "/" + }, + "route": { + "cluster": "pomerium-control-plane-metrics" + } } - }] + ] }] }, "statPrefix": "metrics" diff --git a/config/metrics.go b/config/metrics.go index d49756cac..5328d4602 100644 --- a/config/metrics.go +++ b/config/metrics.go @@ -24,14 +24,13 @@ const ( // A MetricsManager manages metrics for a given configuration. type MetricsManager struct { - mu sync.RWMutex - installationID string - serviceName string - addr string - basicAuth string - envoyAdminAddress string - handler http.Handler - endpoints []MetricsScrapeEndpoint + mu sync.RWMutex + installationID string + serviceName string + addr string + basicAuth string + handler http.Handler + endpoints []MetricsScrapeEndpoint } // NewMetricsManager creates a new MetricsManager. @@ -95,7 +94,6 @@ func (mgr *MetricsManager) updateServer(ctx context.Context, cfg *Config) { mgr.addr = cfg.Options.MetricsAddr mgr.basicAuth = cfg.Options.MetricsBasicAuth mgr.installationID = cfg.Options.InstallationID - mgr.envoyAdminAddress = cfg.Options.EnvoyAdminAddress mgr.handler = nil if mgr.addr == "" { @@ -106,7 +104,7 @@ func (mgr *MetricsManager) updateServer(ctx context.Context, cfg *Config) { mgr.endpoints = append(cfg.MetricsScrapeEndpoints, MetricsScrapeEndpoint{ Name: "envoy", - URL: url.URL{Scheme: "http", Host: cfg.Options.EnvoyAdminAddress, Path: "/stats/prometheus"}, + URL: url.URL{Scheme: "http", Host: cfg.Options.MetricsAddr, Path: "/metrics/envoy"}, }) handler, err := metrics.PrometheusHandler(toInternalEndpoints(mgr.endpoints), mgr.installationID, defaultMetricsTimeout) if err != nil { @@ -125,7 +123,6 @@ func (mgr *MetricsManager) configUnchanged(cfg *Config) bool { return cfg.Options.MetricsAddr == mgr.addr && cfg.Options.MetricsBasicAuth == mgr.basicAuth && cfg.Options.InstallationID == mgr.installationID && - cfg.Options.EnvoyAdminAddress == mgr.envoyAdminAddress && reflect.DeepEqual(mgr.endpoints, cfg.MetricsScrapeEndpoints) } diff --git a/config/options.go b/config/options.go index b9c4f5141..96fffa013 100644 --- a/config/options.go +++ b/config/options.go @@ -345,7 +345,6 @@ var defaultOptions = Options{ XffNumTrustedHops: 0, EnvoyAdminAccessLogPath: os.DevNull, EnvoyAdminProfilePath: os.DevNull, - EnvoyAdminAddress: "127.0.0.1:9901", ProgrammaticRedirectDomainWhitelist: []string{"localhost"}, } diff --git a/config/options_test.go b/config/options_test.go index 4cdd3be3c..79c84a619 100644 --- a/config/options_test.go +++ b/config/options_test.go @@ -316,7 +316,6 @@ func TestOptionsFromViper(t *testing.T) { DataBrokerStorageType: "memory", EnvoyAdminAccessLogPath: os.DevNull, EnvoyAdminProfilePath: os.DevNull, - EnvoyAdminAddress: "127.0.0.1:9901", }, false, }, @@ -337,7 +336,6 @@ func TestOptionsFromViper(t *testing.T) { DataBrokerStorageType: "memory", EnvoyAdminAccessLogPath: os.DevNull, EnvoyAdminProfilePath: os.DevNull, - EnvoyAdminAddress: "127.0.0.1:9901", }, false, }, @@ -414,7 +412,7 @@ func Test_AutoCertOptionsFromEnvVar(t *testing.T) { cleanup func() } - var tests = map[string]func(t *testing.T) test{ + tests := map[string]func(t *testing.T) test{ "ok/simple": func(t *testing.T) test { envs := map[string]string{ "AUTOCERT": "true", @@ -689,6 +687,7 @@ func TestOptions_GetOauthOptions(t *testing.T) { require.NoError(t, err) assert.Equal(t, u.Hostname(), oauthOptions.RedirectURL.Hostname()) } + func TestOptions_GetAllRouteableGRPCDomains(t *testing.T) { opts := &Options{ AuthenticateURLString: "https://authenticate.example.com", diff --git a/internal/telemetry/metrics/providers.go b/internal/telemetry/metrics/providers.go index 4487bf5ac..1d4ea17e8 100644 --- a/internal/telemetry/metrics/providers.go +++ b/internal/telemetry/metrics/providers.go @@ -25,6 +25,9 @@ import ( "github.com/pomerium/pomerium/pkg/metrics" ) +// EnvoyMetricsPath is the path on the metrics listener that retrieves envoy metrics. +const EnvoyMetricsPath = "/metrics/envoy" + // ScrapeEndpoint external endpoints to scrape and decorate type ScrapeEndpoint struct { // Name is the logical name of the endpoint @@ -208,6 +211,7 @@ func ocExport(name string, exporter *ocprom.Exporter, r *http.Request, labels [] } } } + func scrapeEndpoints(endpoints []ScrapeEndpoint, labels []*io_prometheus_client.LabelPair) []promProducerFn { out := make([]promProducerFn, 0, len(endpoints)) for _, endpoint := range endpoints {