reduce memory usage by handling http/2 coalescing via a lua script (#1779)

* add support for proxy protocol on HTTP listener (#1777)

* add support for proxy protocol on HTTP listener

* rename option, add doc

* reduce memory usage by handling http/2 coalescing via a lua script

* move script to file

* use wellknown

* fix integration test

integration/policy_test.go

@@ -405,7 +405,7 @@ func TestSNIMismatch(t *testing.T) {
 	}
 	defer res.Body.Close()
 
-	assert.Equal(t, http.StatusOK, res.StatusCode)
+	assert.Equal(t, http.StatusMisdirectedRequest, res.StatusCode)
 }
 
 func TestAttestationJWT(t *testing.T) {

internal/controlplane/luascripts/fix-misdirected.lua

@@ -0,0 +1,23 @@
+function envoy_on_request(request_handle)
+    local headers = request_handle:headers()
+    local dynamic_meta = request_handle:streamInfo():dynamicMetadata()
+
+    local authority = headers:get(":authority")
+
+    -- store the authority header in the metadata so we can retrieve it in the response
+    dynamic_meta:set("envoy.filters.http.lua", "request.authority", authority)
+end
+
+function envoy_on_response(response_handle)
+    local headers = response_handle:headers()
+    local dynamic_meta = response_handle:streamInfo():dynamicMetadata()
+
+    local authority =
+        dynamic_meta:get("envoy.filters.http.lua")["request.authority"]
+
+    -- if we got a 404 (no route found) and the authority header doens't match
+    -- assume we've coalesced http/2 connections and return a 421
+    if headers:get(":status") == "404" and authority ~= "%s" then
+        headers:replace(":status", "421")
+    end
+end

internal/controlplane/xds_filters.go

@@ -0,0 +1,116 @@
+package controlplane
+
+import (
+	"fmt"
+	"time"
+
+	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	envoy_extensions_filters_http_ext_authz_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_authz/v3"
+	envoy_extensions_filters_http_lua_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/lua/v3"
+	envoy_http_connection_manager "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3"
+	"github.com/envoyproxy/go-control-plane/pkg/wellknown"
+	"google.golang.org/protobuf/types/known/durationpb"
+
+	"github.com/pomerium/pomerium/config"
+)
+
+func getHTTPConnectionManagerFilters(options *config.Options, tlsDomain string) []*envoy_http_connection_manager.HttpFilter {
+	fs := []*envoy_http_connection_manager.HttpFilter{
+		getRemoveImpersonateHeadersFilter(),
+		getExtAuthZFilter(options),
+		getExtAuthZSetCookieFilter(),
+		getCleanUpstreamFilter(),
+	}
+
+	if tlsDomain != "" && tlsDomain != "*" {
+		fs = append(fs, getFixMisdirectedFilter(tlsDomain))
+	}
+
+	fs = append(fs, &envoy_http_connection_manager.HttpFilter{
+		Name: wellknown.Router,
+	})
+
+	return fs
+}
+
+func getRemoveImpersonateHeadersFilter() *envoy_http_connection_manager.HttpFilter {
+	data := marshalAny(&envoy_extensions_filters_http_lua_v3.Lua{
+		InlineCode: luascripts.RemoveImpersonateHeaders,
+	})
+	return &envoy_http_connection_manager.HttpFilter{
+		Name: wellknown.Lua,
+		ConfigType: &envoy_http_connection_manager.HttpFilter_TypedConfig{
+			TypedConfig: data,
+		},
+	}
+}
+
+func getExtAuthZFilter(options *config.Options) *envoy_http_connection_manager.HttpFilter {
+	var grpcClientTimeout *durationpb.Duration
+	if options.GRPCClientTimeout != 0 {
+		grpcClientTimeout = durationpb.New(options.GRPCClientTimeout)
+	} else {
+		grpcClientTimeout = durationpb.New(30 * time.Second)
+	}
+	data := marshalAny(&envoy_extensions_filters_http_ext_authz_v3.ExtAuthz{
+		StatusOnError: &envoy_type_v3.HttpStatus{
+			Code: envoy_type_v3.StatusCode_InternalServerError,
+		},
+		Services: &envoy_extensions_filters_http_ext_authz_v3.ExtAuthz_GrpcService{
+			GrpcService: &envoy_config_core_v3.GrpcService{
+				Timeout: grpcClientTimeout,
+				TargetSpecifier: &envoy_config_core_v3.GrpcService_EnvoyGrpc_{
+					EnvoyGrpc: &envoy_config_core_v3.GrpcService_EnvoyGrpc{
+						ClusterName: options.GetAuthorizeURL().Host,
+					},
+				},
+			},
+		},
+		IncludePeerCertificate: true,
+	})
+	return &envoy_http_connection_manager.HttpFilter{
+		Name: wellknown.HTTPExternalAuthorization,
+		ConfigType: &envoy_http_connection_manager.HttpFilter_TypedConfig{
+			TypedConfig: data,
+		},
+	}
+}
+
+func getExtAuthZSetCookieFilter() *envoy_http_connection_manager.HttpFilter {
+	data := marshalAny(&envoy_extensions_filters_http_lua_v3.Lua{
+		InlineCode: luascripts.ExtAuthzSetCookie,
+	})
+	return &envoy_http_connection_manager.HttpFilter{
+		Name: wellknown.Lua,
+		ConfigType: &envoy_http_connection_manager.HttpFilter_TypedConfig{
+			TypedConfig: data,
+		},
+	}
+}
+
+func getCleanUpstreamFilter() *envoy_http_connection_manager.HttpFilter {
+	data := marshalAny(&envoy_extensions_filters_http_lua_v3.Lua{
+		InlineCode: luascripts.CleanUpstream,
+	})
+	return &envoy_http_connection_manager.HttpFilter{
+		Name: wellknown.Lua,
+		ConfigType: &envoy_http_connection_manager.HttpFilter_TypedConfig{
+			TypedConfig: data,
+		},
+	}
+}
+
+func getFixMisdirectedFilter(fqdn string) *envoy_http_connection_manager.HttpFilter {
+	// based on https://github.com/projectcontour/contour/pull/2483/files#diff-7b5eca045986ae5cb249a53591b132b2db720095fa9fa24715178f660383b6c6R303
+	code := fmt.Sprintf(luascripts.FixMisdirected, fqdn)
+	data := marshalAny(&envoy_extensions_filters_http_lua_v3.Lua{
+		InlineCode: code,
+	})
+	return &envoy_http_connection_manager.HttpFilter{
+		Name: wellknown.Lua,
+		ConfigType: &envoy_http_connection_manager.HttpFilter_TypedConfig{
+			TypedConfig: data,
+		},
+	}
+}

internal/controlplane/xds_listeners.go

@@ -5,13 +5,11 @@ import (
 	"net"
 	"net/url"
 	"sort"
-	"time"
 
 	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	envoy_config_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	envoy_config_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
 	envoy_extensions_filters_http_ext_authz_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_authz/v3"
-	envoy_extensions_filters_http_lua_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/lua/v3"
 	envoy_extensions_filters_listener_proxy_protocol_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/listener/proxy_protocol/v3"
 	envoy_http_connection_manager "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
 	envoy_extensions_transport_sockets_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
@@ -66,7 +64,7 @@ func (srv *Server) buildMainListener(options *config.Options) *envoy_config_list
 
 	if options.InsecureServer {
 		filter := buildMainHTTPConnectionManagerFilter(options,
-			getAllRouteableDomains(options, options.Addr))
+			getAllRouteableDomains(options, options.Addr), "")
 
 		return &envoy_config_listener_v3.Listener{
 			Name:            "http-ingress",
@@ -94,7 +92,7 @@ func (srv *Server) buildMainListener(options *config.Options) *envoy_config_list
 		ListenerFilters: listenerFilters,
 		FilterChains: buildFilterChains(options, options.Addr,
 			func(tlsDomain string, httpDomains []string) *envoy_config_listener_v3.FilterChain {
-				filter := buildMainHTTPConnectionManagerFilter(options, httpDomains)
+				filter := buildMainHTTPConnectionManagerFilter(options, httpDomains, tlsDomain)
 				filterChain := &envoy_config_listener_v3.FilterChain{
 					Filters: []*envoy_config_listener_v3.Filter{filter},
 				}
@@ -128,14 +126,14 @@ func buildFilterChains(
 	var chains []*envoy_config_listener_v3.FilterChain
 	for _, domain := range tlsDomains {
 		// first we match on SNI
-		chains = append(chains, callback(domain, allDomains))
+		chains = append(chains, callback(domain, []string{domain}))
 	}
 	// if there are no SNI matches we match on HTTP host
 	chains = append(chains, callback("*", allDomains))
 	return chains
 }
 
-func buildMainHTTPConnectionManagerFilter(options *config.Options, domains []string) *envoy_config_listener_v3.Filter {
+func buildMainHTTPConnectionManagerFilter(options *config.Options, domains []string, tlsDomain string) *envoy_config_listener_v3.Filter {
 	var virtualHosts []*envoy_config_route_v3.VirtualHost
 	for _, domain := range domains {
 		vh := &envoy_config_route_v3.VirtualHost{
@@ -163,46 +161,14 @@ func buildMainHTTPConnectionManagerFilter(options *config.Options, domains []str
 			virtualHosts = append(virtualHosts, vh)
 		}
 	}
+	if tlsDomain == "*" {
 		virtualHosts = append(virtualHosts, &envoy_config_route_v3.VirtualHost{
 			Name:    "catch-all",
 			Domains: []string{"*"},
 			Routes:  buildPomeriumHTTPRoutes(options, "*"),
 		})
-
-	var grpcClientTimeout *durationpb.Duration
-	if options.GRPCClientTimeout != 0 {
-		grpcClientTimeout = ptypes.DurationProto(options.GRPCClientTimeout)
-	} else {
-		grpcClientTimeout = ptypes.DurationProto(30 * time.Second)
 	}
 
-	extAuthZ := marshalAny(&envoy_extensions_filters_http_ext_authz_v3.ExtAuthz{
-		StatusOnError: &envoy_type_v3.HttpStatus{
-			Code: envoy_type_v3.StatusCode_InternalServerError,
-		},
-		Services: &envoy_extensions_filters_http_ext_authz_v3.ExtAuthz_GrpcService{
-			GrpcService: &envoy_config_core_v3.GrpcService{
-				Timeout: grpcClientTimeout,
-				TargetSpecifier: &envoy_config_core_v3.GrpcService_EnvoyGrpc_{
-					EnvoyGrpc: &envoy_config_core_v3.GrpcService_EnvoyGrpc{
-						ClusterName: options.GetAuthorizeURL().Host,
-					},
-				},
-			},
-		},
-		IncludePeerCertificate: true,
-	})
-
-	extAuthzSetCookieLua := marshalAny(&envoy_extensions_filters_http_lua_v3.Lua{
-		InlineCode: luascripts.ExtAuthzSetCookie,
-	})
-	cleanUpstreamLua := marshalAny(&envoy_extensions_filters_http_lua_v3.Lua{
-		InlineCode: luascripts.CleanUpstream,
-	})
-	removeImpersonateHeadersLua := marshalAny(&envoy_extensions_filters_http_lua_v3.Lua{
-		InlineCode: luascripts.RemoveImpersonateHeaders,
-	})
-
 	var maxStreamDuration *durationpb.Duration
 	if options.WriteTimeout > 0 {
 		maxStreamDuration = ptypes.DurationProto(options.WriteTimeout)
@@ -214,35 +180,7 @@ func buildMainHTTPConnectionManagerFilter(options *config.Options, domains []str
 		RouteSpecifier: &envoy_http_connection_manager.HttpConnectionManager_RouteConfig{
 			RouteConfig: buildRouteConfiguration("main", virtualHosts),
 		},
-		HttpFilters: []*envoy_http_connection_manager.HttpFilter{
+		HttpFilters: getHTTPConnectionManagerFilters(options, tlsDomain),
-			{
-				Name: "envoy.filters.http.lua",
-				ConfigType: &envoy_http_connection_manager.HttpFilter_TypedConfig{
-					TypedConfig: removeImpersonateHeadersLua,
-				},
-			},
-			{
-				Name: "envoy.filters.http.ext_authz",
-				ConfigType: &envoy_http_connection_manager.HttpFilter_TypedConfig{
-					TypedConfig: extAuthZ,
-				},
-			},
-			{
-				Name: "envoy.filters.http.lua",
-				ConfigType: &envoy_http_connection_manager.HttpFilter_TypedConfig{
-					TypedConfig: extAuthzSetCookieLua,
-				},
-			},
-			{
-				Name: "envoy.filters.http.lua",
-				ConfigType: &envoy_http_connection_manager.HttpFilter_TypedConfig{
-					TypedConfig: cleanUpstreamLua,
-				},
-			},
-			{
-				Name: "envoy.filters.http.router",
-			},
-		},
 		AccessLog:   buildAccessLogs(options),
 		CommonHttpProtocolOptions: &envoy_config_core_v3.HttpProtocolOptions{
 			IdleTimeout:       ptypes.DurationProto(options.IdleTimeout),

internal/controlplane/xds_listeners_test.go

@@ -22,7 +22,7 @@ const (
 
 func Test_buildMainHTTPConnectionManagerFilter(t *testing.T) {
 	options := config.NewDefaultOptions()
-	filter := buildMainHTTPConnectionManagerFilter(options, []string{"example.com"})
+	filter := buildMainHTTPConnectionManagerFilter(options, []string{"example.com"}, "*")
 	testutil.AssertProtoJSONEqual(t, `{
 		"name": "envoy.filters.network.http_connection_manager",
 		"typedConfig": {

internal/controlplane/xds_lua.go

@@ -16,6 +16,7 @@ var luascripts struct {
 	ExtAuthzSetCookie        string
 	CleanUpstream            string
 	RemoveImpersonateHeaders string
+	FixMisdirected           string
 }
 
 func init() {
@@ -28,6 +29,7 @@ func init() {
 		"/clean-upstream.lua":             &luascripts.CleanUpstream,
 		"/ext-authz-set-cookie.lua":       &luascripts.ExtAuthzSetCookie,
 		"/remove-impersonate-headers.lua": &luascripts.RemoveImpersonateHeaders,
+		"/fix-misdirected.lua":            &luascripts.FixMisdirected,
 	}
 
 	err = fs.Walk(hfs, "/", func(p string, fi os.FileInfo, err error) error {