autocert: add support for ACME TLS-ALPN (#3590)

* autocert: add support for ACME TLS-ALPN

* always re-create acme tls server

config/config.go

@@ -26,6 +26,8 @@ type Config struct {
 	MetricsPort string
 	// DebugPort is the port the debug listener is running on.
 	DebugPort string
+	// ACMETLSPort is the port that handles the ACME TLS-ALPN challenge.
+	ACMETLSALPNPort string
 
 	// MetricsScrapeEndpoints additional metrics endpoints to scrape and provide part of metrics
 	MetricsScrapeEndpoints []MetricsScrapeEndpoint
@@ -46,11 +48,12 @@ func (cfg *Config) Clone() *Config {
 		AutoCertificates: cfg.AutoCertificates,
 		EnvoyVersion:     cfg.EnvoyVersion,
 
-		GRPCPort:     cfg.GRPCPort,
+		GRPCPort:        cfg.GRPCPort,
-		HTTPPort:     cfg.HTTPPort,
+		HTTPPort:        cfg.HTTPPort,
-		OutboundPort: cfg.OutboundPort,
+		OutboundPort:    cfg.OutboundPort,
-		MetricsPort:  cfg.MetricsPort,
+		MetricsPort:     cfg.MetricsPort,
-		DebugPort:    cfg.DebugPort,
+		DebugPort:       cfg.DebugPort,
+		ACMETLSALPNPort: cfg.ACMETLSALPNPort,
 
 		MetricsScrapeEndpoints: endpoints,
 	}
@@ -75,10 +78,11 @@ func (cfg *Config) Checksum() uint64 {
 }
 
 // AllocatePorts populates
-func (cfg *Config) AllocatePorts(ports [5]string) {
+func (cfg *Config) AllocatePorts(ports [6]string) {
 	cfg.GRPCPort = ports[0]
 	cfg.HTTPPort = ports[1]
 	cfg.OutboundPort = ports[2]
 	cfg.MetricsPort = ports[3]
 	cfg.DebugPort = ports[4]
+	cfg.ACMETLSALPNPort = ports[5]
 }

config/config_source.go

@@ -116,12 +116,12 @@ func NewFileOrEnvironmentSource(
 		EnvoyVersion: envoyVersion,
 	}
 
-	ports, err := netutil.AllocatePorts(5)
+	ports, err := netutil.AllocatePorts(6)
 	if err != nil {
 		return nil, fmt.Errorf("allocating ports: %w", err)
 	}
 
-	cfg.AllocatePorts(*(*[5]string)(ports))
+	cfg.AllocatePorts(*(*[6]string)(ports))
 
 	metrics.SetConfigInfo(ctx, cfg.Options.Services, "local", cfg.Checksum(), true)
 

config/envoyconfig/acmetlsalpn.go

@@ -0,0 +1,53 @@
+package envoyconfig
+
+import (
+	"strconv"
+
+	envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	envoy_config_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+
+	"github.com/pomerium/pomerium/config"
+)
+
+// Pomerium implements the ACME TLS-ALPN protocol by adding a filter chain to the main HTTPS listener
+// that matches the acme-tls/1 application protocol on incoming requests and forwards them to a listener
+// started in the `autocert` package. The proxying is done using TCP so that the Go listener can terminate
+// the TLS connection using the certmagic package.
+
+const (
+	acmeTLSALPNApplicationProtocol = "acme-tls/1"
+	acmeTLSALPNClusterName         = "pomerium-acme-tls-alpn"
+)
+
+func (b *Builder) buildACMETLSALPNCluster(
+	cfg *config.Config,
+) *envoy_config_cluster_v3.Cluster {
+	port, _ := strconv.Atoi(cfg.ACMETLSALPNPort)
+	return &envoy_config_cluster_v3.Cluster{
+		Name: acmeTLSALPNClusterName,
+		LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{
+			ClusterName: acmeTLSALPNClusterName,
+			Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{{
+				LbEndpoints: []*envoy_config_endpoint_v3.LbEndpoint{{
+					HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{
+						Endpoint: &envoy_config_endpoint_v3.Endpoint{
+							Address: buildAddress("127.0.0.1", port),
+						},
+					},
+				}},
+			}},
+		},
+	}
+}
+
+func (b *Builder) buildACMETLSALPNFilterChain() *envoy_config_listener_v3.FilterChain {
+	return &envoy_config_listener_v3.FilterChain{
+		FilterChainMatch: &envoy_config_listener_v3.FilterChainMatch{
+			ApplicationProtocols: []string{acmeTLSALPNApplicationProtocol},
+		},
+		Filters: []*envoy_config_listener_v3.Filter{
+			TCPProxyFilter(acmeTLSALPNClusterName),
+		},
+	}
+}

config/envoyconfig/acmetlsalpn_test.go

@@ -0,0 +1,54 @@
+package envoyconfig
+
+import (
+	"testing"
+
+	"github.com/pomerium/pomerium/config"
+	"github.com/pomerium/pomerium/internal/testutil"
+)
+
+func TestBuilder_buildACMETLSALPNCluster(t *testing.T) {
+	b := New("local-grpc", "local-http", "local-metrics", nil, nil)
+	testutil.AssertProtoJSONEqual(t,
+		`{
+			"name": "pomerium-acme-tls-alpn",
+			"loadAssignment": {
+				"clusterName": "pomerium-acme-tls-alpn",
+				"endpoints": [{
+					"lbEndpoints": [{
+						"endpoint": {
+							"address": {
+								"socketAddress": {
+									"address": "127.0.0.1",
+									"portValue": 1234
+								}
+							}
+						}
+					}]
+				}]
+			}
+		}`,
+		b.buildACMETLSALPNCluster(&config.Config{
+			ACMETLSALPNPort: "1234",
+		}))
+
+}
+
+func TestBuilder_buildACMETLSALPNFilterChain(t *testing.T) {
+	b := New("local-grpc", "local-http", "local-metrics", nil, nil)
+	testutil.AssertProtoJSONEqual(t,
+		`{
+			"filterChainMatch": {
+				"applicationProtocols": ["acme-tls/1"]
+			},
+			"filters": [{
+				"name": "tcp_proxy",
+				"typedConfig": {
+					"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
+					"cluster": "pomerium-acme-tls-alpn",
+					"statPrefix": "acme_tls_alpn"
+				}
+			}]
+		}`,
+		b.buildACMETLSALPNFilterChain())
+}

config/envoyconfig/clusters.go

@@ -80,6 +80,7 @@ func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*env
 	}
 
 	clusters := []*envoy_config_cluster_v3.Cluster{
+		b.buildACMETLSALPNCluster(cfg),
 		controlGRPC,
 		controlHTTP,
 		controlMetrics,

config/envoyconfig/filters.go

@@ -9,6 +9,7 @@ import (
 	envoy_extensions_filters_listener_proxy_protocol_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/listener/proxy_protocol/v3"
 	envoy_extensions_filters_listener_tls_inspector_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/listener/tls_inspector/v3"
 	envoy_extensions_filters_network_http_connection_manager "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	envoy_extensions_filters_network_tcp_proxy_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3"
 	envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3"
 	"google.golang.org/protobuf/types/known/durationpb"
 
@@ -89,6 +90,21 @@ func ProxyProtocolFilter() *envoy_config_listener_v3.ListenerFilter {
 	}
 }
 
+// TCPProxyFilter creates a new TCP Proxy filter.
+func TCPProxyFilter(clusterName string) *envoy_config_listener_v3.Filter {
+	return &envoy_config_listener_v3.Filter{
+		Name: "tcp_proxy",
+		ConfigType: &envoy_config_listener_v3.Filter_TypedConfig{
+			TypedConfig: protoutil.NewAny(&envoy_extensions_filters_network_tcp_proxy_v3.TcpProxy{
+				StatPrefix: "acme_tls_alpn",
+				ClusterSpecifier: &envoy_extensions_filters_network_tcp_proxy_v3.TcpProxy_Cluster{
+					Cluster: clusterName,
+				},
+			}),
+		},
+	}
+}
+
 // TLSInspectorFilter creates a new TLS inspector filter.
 func TLSInspectorFilter() *envoy_config_listener_v3.ListenerFilter {
 	return &envoy_config_listener_v3.ListenerFilter{

config/envoyconfig/listeners.go

@@ -249,6 +249,7 @@ func (b *Builder) buildFilterChains(
 	}
 
 	var chains []*envoy_config_listener_v3.FilterChain
+	chains = append(chains, b.buildACMETLSALPNFilterChain())
 	for _, domain := range tlsDomains {
 		routeableDomains, err := getRouteableDomainsForTLSServerName(options, addr, domain)
 		if err != nil {

internal/autocert/manager.go

@@ -3,9 +3,11 @@ package autocert
 
 import (
 	"context"
+	"crypto/tls"
 	"encoding/base64"
 	"errors"
 	"fmt"
+	"net"
 	"net/http"
 	"sort"
 	"sync"
@@ -43,11 +45,12 @@ type Manager struct {
 	src          config.Source
 	acmeTemplate certmagic.ACMEIssuer
 
-	mu        sync.RWMutex
+	mu                  sync.RWMutex
-	config    *config.Config
+	config              *config.Config
-	certmagic *certmagic.Config
+	certmagic           *certmagic.Config
-	acmeMgr   *atomicutil.Value[*certmagic.ACMEIssuer]
+	acmeMgr             *atomicutil.Value[*certmagic.ACMEIssuer]
-	srv       *http.Server
+	srv                 *http.Server
+	acmeTLSALPNListener net.Listener
 
 	*ocspCache
 
@@ -152,7 +155,6 @@ func (mgr *Manager) getCertMagicConfig(ctx context.Context, cfg *config.Config)
 	if err != nil {
 		return nil, err
 	}
-	acmeMgr.DisableTLSALPNChallenge = true
 	mgr.certmagic.Issuers = []certmagic.Issuer{acmeMgr}
 	mgr.acmeMgr.Store(acmeMgr)
 
@@ -207,6 +209,7 @@ func (mgr *Manager) renewConfigCerts(ctx context.Context) error {
 
 	cfg = mgr.src.GetConfig().Clone()
 	mgr.updateServer(ctx, cfg)
+	mgr.updateACMETLSALPNServer(ctx, cfg)
 	if err := mgr.updateAutocert(ctx, cfg); err != nil {
 		return err
 	}
@@ -224,6 +227,7 @@ func (mgr *Manager) update(ctx context.Context, cfg *config.Config) error {
 	defer func() { mgr.config = cfg }()
 
 	mgr.updateServer(ctx, cfg)
+	mgr.updateACMETLSALPNServer(ctx, cfg)
 	return mgr.updateAutocert(ctx, cfg)
 }
 
@@ -324,6 +328,34 @@ func (mgr *Manager) updateServer(ctx context.Context, cfg *config.Config) {
 	mgr.srv = hsrv
 }
 
+func (mgr *Manager) updateACMETLSALPNServer(ctx context.Context, cfg *config.Config) {
+	addr := net.JoinHostPort("127.0.0.1", cfg.ACMETLSALPNPort)
+	if mgr.acmeTLSALPNListener != nil {
+		_ = mgr.acmeTLSALPNListener.Close()
+		mgr.acmeTLSALPNListener = nil
+	}
+
+	tlsConfig := mgr.certmagic.TLSConfig()
+	ln, err := tls.Listen("tcp", addr, tlsConfig)
+	if err != nil {
+		log.Error(ctx).Err(err).Msg("failed to run acme tls alpn server")
+		return
+	}
+	mgr.acmeTLSALPNListener = ln
+
+	go func() {
+		for {
+			conn, err := ln.Accept()
+			if errors.Is(err, net.ErrClosed) {
+				return
+			} else if err != nil {
+				continue
+			}
+			_ = conn.Close()
+		}
+	}()
+}
+
 func (mgr *Manager) handleHTTPChallenge(w http.ResponseWriter, r *http.Request) bool {
 	return mgr.acmeMgr.Load().HandleHTTPChallenge(w, r)
 }
@@ -347,6 +379,8 @@ func configureCertificateAuthority(acmeMgr *certmagic.ACMEIssuer, opts config.Au
 	}
 	if opts.Email != "" {
 		acmeMgr.Email = opts.Email
+	} else {
+		acmeMgr.Email = " " // intentionally set to a space so that certmagic doesn't prompt for an email address
 	}
 	return nil
 }

internal/autocert/manager_test.go

@@ -395,6 +395,7 @@ func Test_configureCertificateAuthority(t *testing.T) {
 				expected: &certmagic.ACMEIssuer{
 					Agreed: true,
 					CA:     certmagic.DefaultACME.CA,
+					Email:  " ",
 					TestCA: certmagic.DefaultACME.TestCA,
 				},
 				wantErr: false,
@@ -411,6 +412,7 @@ func Test_configureCertificateAuthority(t *testing.T) {
 				expected: &certmagic.ACMEIssuer{
 					Agreed: true,
 					CA:     certmagic.DefaultACME.TestCA,
+					Email:  " ",
 					TestCA: certmagic.DefaultACME.TestCA,
 				},
 				wantErr: false,

pkg/cmd/pomerium/pomerium.go

@@ -80,6 +80,7 @@ func Run(ctx context.Context, src config.Source) error {
 		Str("outbound-port", src.GetConfig().OutboundPort).
 		Str("metrics-port", src.GetConfig().MetricsPort).
 		Str("debug-port", src.GetConfig().DebugPort).
+		Str("acme-tls-alpn-port", src.GetConfig().ACMETLSALPNPort).
 		Msg("server started")
 
 	// create envoy server