From d04de59be50686bd7da1c6f41e9e88c1e60ce713 Mon Sep 17 00:00:00 2001 From: Caleb Doxsey Date: Thu, 19 Dec 2024 13:30:43 -0700 Subject: [PATCH] use envoy for http redirect server --- config/config.go | 8 +- config/config_source.go | 4 +- .../envoyconfig/{acmetlsalpn.go => acme.go} | 42 ++++++++++ .../{acmetlsalpn_test.go => acme_test.go} | 0 config/envoyconfig/clusters.go | 1 + config/envoyconfig/listeners.go | 4 + config/envoyconfig/listeners_redirect.go | 80 +++++++++++++++++++ config/envoyconfig/testdata/clusters.json | 22 +++++ internal/autocert/manager.go | 69 +++++++++------- internal/autocert/manager_test.go | 61 -------------- internal/testenv/environment.go | 15 ++-- internal/zero/bootstrap/new.go | 4 +- 12 files changed, 208 insertions(+), 102 deletions(-) rename config/envoyconfig/{acmetlsalpn.go => acme.go} (58%) rename config/envoyconfig/{acmetlsalpn_test.go => acme_test.go} (100%) create mode 100644 config/envoyconfig/listeners_redirect.go diff --git a/config/config.go b/config/config.go index 14a51b2e4..74f58f9d7 100644 --- a/config/config.go +++ b/config/config.go @@ -44,6 +44,8 @@ type Config struct { MetricsPort string // DebugPort is the port the debug listener is running on. DebugPort string + // ACMEHTTPPort is the port that handles the ACME HTTP challenge. + ACMEHTTPPort string // ACMETLSPort is the port that handles the ACME TLS-ALPN challenge. ACMETLSALPNPort string @@ -78,6 +80,7 @@ func (cfg *Config) Clone() *Config { OutboundPort: cfg.OutboundPort, MetricsPort: cfg.MetricsPort, DebugPort: cfg.DebugPort, + ACMEHTTPPort: cfg.ACMEHTTPPort, ACMETLSALPNPort: cfg.ACMETLSALPNPort, MetricsScrapeEndpoints: endpoints, @@ -134,13 +137,14 @@ func (cfg *Config) Checksum() uint64 { } // AllocatePorts populates -func (cfg *Config) AllocatePorts(ports [6]string) { +func (cfg *Config) AllocatePorts(ports [7]string) { cfg.GRPCPort = ports[0] cfg.HTTPPort = ports[1] cfg.OutboundPort = ports[2] cfg.MetricsPort = ports[3] cfg.DebugPort = ports[4] - cfg.ACMETLSALPNPort = ports[5] + cfg.ACMEHTTPPort = ports[5] + cfg.ACMETLSALPNPort = ports[6] } // GetTLSClientConfig returns TLS configuration that accounts for additional CA entries diff --git a/config/config_source.go b/config/config_source.go index 2de950c1f..3be4cb17b 100644 --- a/config/config_source.go +++ b/config/config_source.go @@ -120,12 +120,12 @@ func NewFileOrEnvironmentSource( EnvoyVersion: envoyVersion, } - ports, err := netutil.AllocatePorts(6) + ports, err := netutil.AllocatePorts(7) if err != nil { return nil, fmt.Errorf("allocating ports: %w", err) } - cfg.AllocatePorts(*(*[6]string)(ports)) + cfg.AllocatePorts(*(*[7]string)(ports)) metrics.SetConfigInfo(ctx, cfg.Options.Services, "local", cfg.Checksum(), true) diff --git a/config/envoyconfig/acmetlsalpn.go b/config/envoyconfig/acme.go similarity index 58% rename from config/envoyconfig/acmetlsalpn.go rename to config/envoyconfig/acme.go index c29185578..3af8a2a0a 100644 --- a/config/envoyconfig/acmetlsalpn.go +++ b/config/envoyconfig/acme.go @@ -6,10 +6,52 @@ import ( envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" envoy_config_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + envoy_config_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" "github.com/pomerium/pomerium/config" ) +const ( + acmeHTTPClusterName = "pomerium-acme-http" + acmeHTTPPrefix = "/.well-known/acme-challenge/" +) + +func (b *Builder) buildACMEHTTPCluster(cfg *config.Config) *envoy_config_cluster_v3.Cluster { + port, _ := strconv.ParseUint(cfg.ACMEHTTPPort, 10, 32) + return &envoy_config_cluster_v3.Cluster{ + Name: acmeHTTPClusterName, + LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{ + ClusterName: acmeHTTPClusterName, + Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{{ + LbEndpoints: []*envoy_config_endpoint_v3.LbEndpoint{{ + HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{ + Endpoint: &envoy_config_endpoint_v3.Endpoint{ + Address: buildTCPAddress("127.0.0.1", uint32(port)), + }, + }, + }}, + }}, + }, + } +} + +func (b *Builder) buildACMEHTTPRoute() *envoy_config_route_v3.Route { + return &envoy_config_route_v3.Route{ + Match: &envoy_config_route_v3.RouteMatch{ + PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{ + Prefix: acmeHTTPPrefix, + }, + }, + Action: &envoy_config_route_v3.Route_Route{ + Route: &envoy_config_route_v3.RouteAction{ + ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{ + Cluster: acmeHTTPClusterName, + }, + }, + }, + } +} + // Pomerium implements the ACME TLS-ALPN protocol by adding a filter chain to the main HTTPS listener // that matches the acme-tls/1 application protocol on incoming requests and forwards them to a listener // started in the `autocert` package. The proxying is done using TCP so that the Go listener can terminate diff --git a/config/envoyconfig/acmetlsalpn_test.go b/config/envoyconfig/acme_test.go similarity index 100% rename from config/envoyconfig/acmetlsalpn_test.go rename to config/envoyconfig/acme_test.go diff --git a/config/envoyconfig/clusters.go b/config/envoyconfig/clusters.go index d0d1fd7f9..4d0718f95 100644 --- a/config/envoyconfig/clusters.go +++ b/config/envoyconfig/clusters.go @@ -96,6 +96,7 @@ func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*env clusters := []*envoy_config_cluster_v3.Cluster{ b.buildACMETLSALPNCluster(cfg), + b.buildACMEHTTPCluster(cfg), controlGRPC, controlHTTP, controlMetrics, diff --git a/config/envoyconfig/listeners.go b/config/envoyconfig/listeners.go index f017c0834..0d88e4088 100644 --- a/config/envoyconfig/listeners.go +++ b/config/envoyconfig/listeners.go @@ -65,6 +65,10 @@ func (b *Builder) BuildListeners( listeners = append(listeners, li) } + if shouldStartRedirectListener(cfg.Options) { + listeners = append(listeners, b.buildRedirectListener(cfg)) + } + li, err := b.buildOutboundListener(cfg) if err != nil { return nil, err diff --git a/config/envoyconfig/listeners_redirect.go b/config/envoyconfig/listeners_redirect.go new file mode 100644 index 000000000..8613216b5 --- /dev/null +++ b/config/envoyconfig/listeners_redirect.go @@ -0,0 +1,80 @@ +package envoyconfig + +import ( + "net" + "strconv" + + envoy_config_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + envoy_config_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + envoy_extensions_filters_network_http_connection_manager "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + + "github.com/pomerium/pomerium/config" +) + +func (b *Builder) buildRedirectListener( + cfg *config.Config, +) *envoy_config_listener_v3.Listener { + li := newTCPListener("http-redirect", buildTCPAddress(cfg.Options.HTTPRedirectAddr, 80)) + + // listener filters + if cfg.Options.UseProxyProtocol { + li.ListenerFilters = append(li.ListenerFilters, ProxyProtocolFilter()) + } + + li.FilterChains = append(li.FilterChains, &envoy_config_listener_v3.FilterChain{ + Filters: []*envoy_config_listener_v3.Filter{ + b.buildRedirectHTTPConnectionManagerFilter(cfg), + }, + }) + + return li +} + +func (b *Builder) buildRedirectHTTPConnectionManagerFilter( + cfg *config.Config, +) *envoy_config_listener_v3.Filter { + _, strport, _ := net.SplitHostPort(cfg.Options.Addr) + port, _ := strconv.ParseUint(strport, 10, 32) + if port == 0 { + port = 443 + } + + return HTTPConnectionManagerFilter(&envoy_extensions_filters_network_http_connection_manager.HttpConnectionManager{ + AlwaysSetRequestIdInResponse: true, + StatPrefix: "http-redirect", + RouteSpecifier: &envoy_extensions_filters_network_http_connection_manager.HttpConnectionManager_RouteConfig{ + RouteConfig: &envoy_config_route_v3.RouteConfiguration{ + Name: "http-redirect", + VirtualHosts: []*envoy_config_route_v3.VirtualHost{{ + Name: "http-redirect", + Domains: []string{"*"}, + Routes: []*envoy_config_route_v3.Route{ + b.buildACMEHTTPRoute(), + { + Match: &envoy_config_route_v3.RouteMatch{ + PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{ + Prefix: "/", + }, + }, + Action: &envoy_config_route_v3.Route_Redirect{ + Redirect: &envoy_config_route_v3.RedirectAction{ + SchemeRewriteSpecifier: &envoy_config_route_v3.RedirectAction_HttpsRedirect{ + HttpsRedirect: true, + }, + PortRedirect: uint32(port), + }, + }, + }, + }, + }}, + }, + }, + HttpFilters: []*envoy_extensions_filters_network_http_connection_manager.HttpFilter{ + HTTPRouterFilter(), + }, + }) +} + +func shouldStartRedirectListener(options *config.Options) bool { + return !options.InsecureServer && options.HTTPRedirectAddr != "" +} diff --git a/config/envoyconfig/testdata/clusters.json b/config/envoyconfig/testdata/clusters.json index e42f5b5ee..1f140ef4a 100644 --- a/config/envoyconfig/testdata/clusters.json +++ b/config/envoyconfig/testdata/clusters.json @@ -21,6 +21,28 @@ }, "name": "pomerium-acme-tls-alpn" }, + { + "loadAssignment": { + "clusterName": "pomerium-acme-http", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 0 + } + } + } + } + ] + } + ] + }, + "name": "pomerium-acme-http" + }, { "connectTimeout": "10s", "dnsLookupFamily": "V4_PREFERRED", diff --git a/internal/autocert/manager.go b/internal/autocert/manager.go index 8cd21e7da..b07c25c87 100644 --- a/internal/autocert/manager.go +++ b/internal/autocert/manager.go @@ -20,7 +20,6 @@ import ( "github.com/rs/zerolog" "github.com/pomerium/pomerium/config" - "github.com/pomerium/pomerium/internal/httputil" "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/telemetry/metrics" "github.com/pomerium/pomerium/internal/urlutil" @@ -50,13 +49,16 @@ type Manager struct { config *config.Config certmagic *certmagic.Config acmeMgr atomic.Pointer[certmagic.ACMEIssuer] - srv *http.Server acmeTLSALPNLock sync.Mutex acmeTLSALPNPort string acmeTLSALPNListener net.Listener acmeTLSALPNConfig *tls.Config + acmeHTTPLock sync.Mutex + acmeHTTPPort string + acmeHTTPListener net.Listener + *ocspCache config.ChangeDispatcher @@ -228,7 +230,7 @@ func (mgr *Manager) renewConfigCerts(ctx context.Context) error { log.Ctx(ctx).Info().Msg("updating certificates") cfg = mgr.src.GetConfig().Clone() - mgr.updateServer(ctx, cfg) + mgr.updateACMEHTTPServer(ctx, cfg) mgr.updateACMETLSALPNServer(ctx, cfg) if err := mgr.updateAutocert(ctx, cfg); err != nil { return err @@ -246,7 +248,7 @@ func (mgr *Manager) update(ctx context.Context, cfg *config.Config) error { defer mgr.mu.Unlock() defer func() { mgr.config = cfg }() - mgr.updateServer(ctx, cfg) + mgr.updateACMEHTTPServer(ctx, cfg) mgr.updateACMETLSALPNServer(ctx, cfg) return mgr.updateAutocert(ctx, cfg) } @@ -313,40 +315,51 @@ func (mgr *Manager) updateAutocert(ctx context.Context, cfg *config.Config) erro return nil } -func (mgr *Manager) updateServer(ctx context.Context, cfg *config.Config) { - if mgr.srv != nil { - // nothing to do if the address hasn't changed - if mgr.srv.Addr == cfg.Options.HTTPRedirectAddr { - return - } - // close immediately, don't care about the error - _ = mgr.srv.Close() - mgr.srv = nil - } +func (mgr *Manager) updateACMEHTTPServer(ctx context.Context, cfg *config.Config) { + mgr.acmeHTTPLock.Lock() + defer mgr.acmeHTTPLock.Unlock() - if cfg.Options.HTTPRedirectAddr == "" { + // if the port hasn't changed, we're done + if mgr.acmeHTTPPort == cfg.ACMEHTTPPort { return } - redirect := httputil.RedirectHandler() + // store the updated port + mgr.acmeHTTPPort = cfg.ACMEHTTPPort - hsrv := &http.Server{ - Addr: cfg.Options.HTTPRedirectAddr, - Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if mgr.handleHTTPChallenge(w, r) { - return - } - redirect.ServeHTTP(w, r) - }), + // close the existing listener + if mgr.acmeHTTPListener != nil { + _ = mgr.acmeHTTPListener.Close() + mgr.acmeHTTPListener = nil } + + // start a new listener + addr := net.JoinHostPort("127.0.0.1", cfg.ACMEHTTPPort) + ln, err := net.Listen("tcp", addr) + if err != nil { + log.Ctx(ctx).Error().Err(err).Msg("failed to run acme http server") + return + } + mgr.acmeHTTPListener = ln + go func() { - log.Ctx(ctx).Info().Str("addr", hsrv.Addr).Msg("starting http redirect server") - err := hsrv.ListenAndServe() + srv := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if mgr.handleHTTPChallenge(w, r) { + return + } + // just serve not found + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + }), + ReadTimeout: time.Minute, + WriteTimeout: time.Minute, + IdleTimeout: 5 * time.Minute, + } + err := srv.Serve(ln) if err != nil { - log.Ctx(ctx).Error().Err(err).Msg("failed to run http redirect server") + log.Ctx(ctx).Error().Err(err).Msg("error serving acme http server") } }() - mgr.srv = hsrv } func (mgr *Manager) updateACMETLSALPNServer(ctx context.Context, cfg *config.Config) { diff --git a/internal/autocert/manager_test.go b/internal/autocert/manager_test.go index 5538a246c..8c0c22093 100644 --- a/internal/autocert/manager_test.go +++ b/internal/autocert/manager_test.go @@ -11,7 +11,6 @@ import ( "encoding/base64" "encoding/json" "encoding/pem" - "fmt" "io" "math/big" "net" @@ -298,66 +297,6 @@ func TestConfig(t *testing.T) { } } -func TestRedirect(t *testing.T) { - li, err := net.Listen("tcp", "127.0.0.1:0") - if !assert.NoError(t, err) { - return - } - addr := li.Addr().String() - _ = li.Close() - - src := config.NewStaticSource(&config.Config{ - Options: &config.Options{ - HTTPRedirectAddr: addr, - SetResponseHeaders: map[string]string{ - "X-Frame-Options": "SAMEORIGIN", - "X-XSS-Protection": "1; mode=block", - "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", - }, - }, - }) - _, err = New(context.Background(), src) - if !assert.NoError(t, err) { - return - } - err = waitFor(addr) - if !assert.NoError(t, err) { - return - } - - client := &http.Client{ - CheckRedirect: func(_ *http.Request, _ []*http.Request) error { - return http.ErrUseLastResponse - }, - } - - res, err := client.Get(fmt.Sprintf("http://%s", addr)) - if !assert.NoError(t, err) { - return - } - defer res.Body.Close() - - assert.Equal(t, http.StatusMovedPermanently, res.StatusCode, "should redirect to https") - for k, v := range src.GetConfig().Options.SetResponseHeaders { - assert.NotEqual(t, v, res.Header.Get(k), "should ignore options header") - } -} - -func waitFor(addr string) error { - var err error - deadline := time.Now().Add(time.Second * 30) - for time.Now().Before(deadline) { - var conn net.Conn - conn, err = net.Dial("tcp", addr) - if err == nil { - conn.Close() - return nil - } - time.Sleep(time.Second) - } - return err -} - func readJWSPayload(r io.Reader, dst any) { var req struct { Protected string `json:"protected"` diff --git a/internal/testenv/environment.go b/internal/testenv/environment.go index d5d8f88e1..5294f5e2b 100644 --- a/internal/testenv/environment.go +++ b/internal/testenv/environment.go @@ -30,6 +30,12 @@ import ( "testing" "time" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/grpclog" + "github.com/pomerium/pomerium/config" "github.com/pomerium/pomerium/config/envoyconfig/filemgr" "github.com/pomerium/pomerium/internal/log" @@ -39,11 +45,6 @@ import ( "github.com/pomerium/pomerium/pkg/health" "github.com/pomerium/pomerium/pkg/netutil" "github.com/pomerium/pomerium/pkg/slices" - "github.com/rs/zerolog" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "golang.org/x/sync/errgroup" - "google.golang.org/grpc/grpclog" ) // Environment is a lightweight integration test fixture that runs Pomerium @@ -450,7 +451,7 @@ func (e *environment) Start() { cfg := &config.Config{ Options: config.NewDefaultOptions(), } - ports, err := netutil.AllocatePorts(9) + ports, err := netutil.AllocatePorts(10) require.NoError(e.t, err) atoi := func(str string) int { p, err := strconv.Atoi(str) @@ -468,7 +469,7 @@ func (e *environment) Start() { e.ports.Metrics.Resolve(atoi(ports[6])) e.ports.Debug.Resolve(atoi(ports[7])) e.ports.ALPN.Resolve(atoi(ports[8])) - cfg.AllocatePorts(*(*[6]string)(ports[3:])) + cfg.AllocatePorts(*(*[7]string)(ports[3:])) cfg.Options.AutocertOptions = config.AutocertOptions{Enable: false} cfg.Options.Services = "all" diff --git a/internal/zero/bootstrap/new.go b/internal/zero/bootstrap/new.go index c80c6530f..eb90f3694 100644 --- a/internal/zero/bootstrap/new.go +++ b/internal/zero/bootstrap/new.go @@ -78,12 +78,12 @@ func New(secret []byte, fileCachePath *string, writer writers.ConfigWriter, api func setConfigDefaults(cfg *config.Config) error { cfg.Options = config.NewDefaultOptions() - ports, err := netutil.AllocatePorts(6) + ports, err := netutil.AllocatePorts(7) if err != nil { return fmt.Errorf("allocating ports: %w", err) } - cfg.AllocatePorts(*(*[6]string)(ports[:6])) + cfg.AllocatePorts(*(*[7]string)(ports[:7])) return nil }