config: allow customization of envoy boostrap admin options (#1872)

config/options.go

@@ -270,6 +270,11 @@ type Options struct {
 	// SkipXffAppend instructs proxy not to append its IP address to x-forwarded-for header.
 	// see https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers.html?highlight=skip_xff_append#x-forwarded-for
 	SkipXffAppend bool `mapstructure:"skip_xff_append" yaml:"skip_xff_append,omitempty" json:"skip_xff_append,omitempty"`
+
+	// Envoy bootstrap admin options. These do not support dynamic updates.
+	EnvoyAdminAccessLogPath string `mapstructure:"envoy_admin_access_log_path" yaml:"envoy_admin_access_log_path"`
+	EnvoyAdminProfilePath   string `mapstructure:"envoy_admin_profile_path" yaml:"envoy_admin_profile_path"`
+	EnvoyAdminAddress       string `mapstructure:"envoy_admin_address" yaml:"envoy_admin_address"`
 }
 
 type certificateFilePair struct {
@@ -312,8 +317,11 @@ var defaultOptions = Options{
 	AutocertOptions: AutocertOptions{
 		Folder: dataDir(),
 	},
-	DataBrokerStorageType: "memory",
+	DataBrokerStorageType:   "memory",
-	SkipXffAppend:         false,
+	SkipXffAppend:           false,
+	EnvoyAdminAccessLogPath: os.DevNull,
+	EnvoyAdminProfilePath:   os.DevNull,
+	EnvoyAdminAddress:       "127.0.0.1:9901",
 }
 
 // NewDefaultOptions returns a copy the default options. It's the caller's

config/options_test.go

@@ -268,6 +268,9 @@ func TestOptionsFromViper(t *testing.T) {
 				RefreshDirectoryInterval: 10 * time.Minute,
 				QPS:                      1.0,
 				DataBrokerStorageType:    "memory",
+				EnvoyAdminAccessLogPath:  os.DevNull,
+				EnvoyAdminProfilePath:    os.DevNull,
+				EnvoyAdminAddress:        "127.0.0.1:9901",
 			},
 			false,
 		},
@@ -288,6 +291,9 @@ func TestOptionsFromViper(t *testing.T) {
 				RefreshDirectoryInterval:        10 * time.Minute,
 				QPS:                             1.0,
 				DataBrokerStorageType:           "memory",
+				EnvoyAdminAccessLogPath:         os.DevNull,
+				EnvoyAdminProfilePath:           os.DevNull,
+				EnvoyAdminAddress:               "127.0.0.1:9901",
 			},
 			false,
 		},

docs/reference/readme.md

@@ -590,6 +590,15 @@ tracing_zipkin_endpoint | Url to the Zipkin HTTP endpoint. | ✅
 Setting `use_proxy_protocol` will configure Pomerium to require the [HAProxy proxy protocol](https://www.haproxy.org/download/1.9/doc/proxy-protocol.txt) on incoming connections. Versions 1 and 2 of the protocol are supported.
 
 
+### Envoy Admin Options
+- Environment Variable: `ENVOY_ADMIN_ADDRESS`, `ENVOY_ADMIN_ACCESS_LOG_PATH`, `ENVOY_ADMIN_PROFILE_PATH`
+- Config File Keys: `envoy_admin_address`, `envoy_admin_access_log_path`, `envoy_admin_profile_path`
+- Type: `string`
+- Optional
+
+These options customize Envoy's [bootstrap configuration](https://www.envoyproxy.io/docs/envoy/latest/operations/admin#operations-admin-interface). They cannot be modified at runtime.
+
+
 ## Authenticate Service
 
 ### Authenticate Callback Path

docs/reference/settings.yaml

@@ -670,6 +670,15 @@ settings:
           - Optional
         doc: |
           Setting `use_proxy_protocol` will configure Pomerium to require the [HAProxy proxy protocol](https://www.haproxy.org/download/1.9/doc/proxy-protocol.txt) on incoming connections. Versions 1 and 2 of the protocol are supported.
+      - name: "Envoy Admin Options"
+        keys: ["envoy_admin_options"]
+        attributes: |
+          - Environment Variable: `ENVOY_ADMIN_ADDRESS`, `ENVOY_ADMIN_ACCESS_LOG_PATH`, `ENVOY_ADMIN_PROFILE_PATH`
+          - Config File Keys: `envoy_admin_address`, `envoy_admin_access_log_path`, `envoy_admin_profile_path`
+          - Type: `string`
+          - Optional
+        doc: |
+          These options customize Envoy's [bootstrap configuration](https://www.envoyproxy.io/docs/envoy/latest/operations/admin#operations-admin-interface). They cannot be modified at runtime.
   - name: "Authenticate Service"
     settings:
       - name: "Authenticate Callback Path"

internal/envoy/envoy.go

@@ -132,7 +132,7 @@ func (srv *Server) update(cfg *config.Config) {
 	}
 	srv.options = options
 
-	if err := srv.writeConfig(); err != nil {
+	if err := srv.writeConfig(cfg); err != nil {
 		log.Error().Err(err).Str("service", "envoy").Msg("envoy: failed to write envoy config")
 		return
 	}
@@ -195,8 +195,8 @@ func (srv *Server) run() error {
 	return nil
 }
 
-func (srv *Server) writeConfig() error {
+func (srv *Server) writeConfig(cfg *config.Config) error {
-	confBytes, err := srv.buildBootstrapConfig()
+	confBytes, err := srv.buildBootstrapConfig(cfg)
 	if err != nil {
 		return err
 	}
@@ -207,24 +207,20 @@ func (srv *Server) writeConfig() error {
 	return atomic.WriteFile(cfgPath, bytes.NewReader(confBytes))
 }
 
-func (srv *Server) buildBootstrapConfig() ([]byte, error) {
+func (srv *Server) buildBootstrapConfig(cfg *config.Config) ([]byte, error) {
 	nodeCfg := &envoy_config_core_v3.Node{
 		Id:      "proxy",
 		Cluster: "proxy",
 	}
 
+	adminAddr, err := ParseAddress(cfg.Options.EnvoyAdminAddress)
+	if err != nil {
+		return nil, err
+	}
 	adminCfg := &envoy_config_bootstrap_v3.Admin{
-		AccessLogPath: "/tmp/admin_access.log",
+		AccessLogPath: cfg.Options.EnvoyAdminAccessLogPath,
-		Address: &envoy_config_core_v3.Address{
+		ProfilePath:   cfg.Options.EnvoyAdminProfilePath,
-			Address: &envoy_config_core_v3.Address_SocketAddress{
+		Address:       adminAddr,
-				SocketAddress: &envoy_config_core_v3.SocketAddress{
-					Address: "127.0.0.1",
-					PortSpecifier: &envoy_config_core_v3.SocketAddress_PortValue{
-						PortValue: 9901,
-					},
-				},
-			},
-		},
 	}
 
 	dynamicCfg := &envoy_config_bootstrap_v3.Bootstrap_DynamicResources{
@@ -352,7 +348,7 @@ func (srv *Server) buildBootstrapConfig() ([]byte, error) {
 		})
 	}
 
-	cfg := &envoy_config_bootstrap_v3.Bootstrap{
+	bcfg := &envoy_config_bootstrap_v3.Bootstrap{
 		Node:             nodeCfg,
 		Admin:            adminCfg,
 		DynamicResources: dynamicCfg,
@@ -360,11 +356,11 @@ func (srv *Server) buildBootstrapConfig() ([]byte, error) {
 		StatsConfig:      srv.buildStatsConfig(),
 	}
 
-	if err := srv.addTraceConfig(cfg); err != nil {
+	if err := srv.addTraceConfig(bcfg); err != nil {
 		return nil, fmt.Errorf("failed to add tracing config: %w", err)
 	}
 
-	jsonBytes, err := protojson.Marshal(proto.MessageV2(cfg))
+	jsonBytes, err := protojson.Marshal(proto.MessageV2(bcfg))
 	if err != nil {
 		return nil, err
 	}

internal/envoy/misc.go

@@ -1,8 +1,12 @@
 package envoy
 
 import (
+	"fmt"
 	"io/ioutil"
+	"net"
 	"strconv"
+
+	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 )
 
 const baseIDPath = "/tmp/pomerium-envoy-base-id"
@@ -29,3 +33,22 @@ func readBaseID() (int, bool) {
 
 	return baseID, true
 }
+
+// ParseAddress parses a string address into an envoy address.
+func ParseAddress(raw string) (*envoy_config_core_v3.Address, error) {
+	if host, portstr, err := net.SplitHostPort(raw); err == nil {
+		if port, err := strconv.Atoi(portstr); err == nil {
+			return &envoy_config_core_v3.Address{
+				Address: &envoy_config_core_v3.Address_SocketAddress{
+					SocketAddress: &envoy_config_core_v3.SocketAddress{
+						Address: host,
+						PortSpecifier: &envoy_config_core_v3.SocketAddress_PortValue{
+							PortValue: uint32(port),
+						},
+					},
+				},
+			}, nil
+		}
+	}
+	return nil, fmt.Errorf("unknown address format: %s", raw)
+}