From 22f6a2207beaa2f1dfd094b1036e615b39554476 Mon Sep 17 00:00:00 2001 From: Caleb Doxsey Date: Fri, 23 Apr 2021 14:54:13 -0600 Subject: [PATCH] envoy: re-implement recommended defaults (#2123) --- config/envoyconfig/clusters_test.go | 36 ++++++++++-- config/envoyconfig/envoyconfig.go | 8 ++- config/envoyconfig/listeners.go | 84 ++++++++++++++-------------- config/envoyconfig/listeners_test.go | 1 + config/envoyconfig/protocols.go | 24 ++++++-- 5 files changed, 99 insertions(+), 54 deletions(-) diff --git a/config/envoyconfig/clusters_test.go b/config/envoyconfig/clusters_test.go index dec052ca2..cd49dbd13 100644 --- a/config/envoyconfig/clusters_test.go +++ b/config/envoyconfig/clusters_test.go @@ -241,12 +241,16 @@ func Test_buildCluster(t *testing.T) { "connectTimeout": "10s", "respectDnsTtl": true, "dnsLookupFamily": "V4_ONLY", + "perConnectionBufferLimitBytes": 32768, "typedExtensionProtocolOptions": { "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", "explicitHttpConfig": { "http2ProtocolOptions": { - "allowConnect": true + "allowConnect": true, + "initialConnectionWindowSize": 1048576, + "initialStreamWindowSize": 65536, + "maxConcurrentStreams": 100 } } } @@ -297,6 +301,7 @@ func Test_buildCluster(t *testing.T) { "type": "STRICT_DNS", "connectTimeout": "10s", "respectDnsTtl": true, + "perConnectionBufferLimitBytes": 32768, "transportSocketMatches": [{ "name": "`+endpoints[0].TransportSocketName()+`", "match": { @@ -360,7 +365,10 @@ func Test_buildCluster(t *testing.T) { "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", "explicitHttpConfig": { "http2ProtocolOptions": { - "allowConnect": true + "allowConnect": true, + "initialConnectionWindowSize": 1048576, + "initialStreamWindowSize": 65536, + "maxConcurrentStreams": 100 } } } @@ -422,12 +430,16 @@ func Test_buildCluster(t *testing.T) { "type": "STATIC", "connectTimeout": "10s", "respectDnsTtl": true, + "perConnectionBufferLimitBytes": 32768, "typedExtensionProtocolOptions": { "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", "explicitHttpConfig": { "http2ProtocolOptions": { - "allowConnect": true + "allowConnect": true, + "initialConnectionWindowSize": 1048576, + "initialStreamWindowSize": 65536, + "maxConcurrentStreams": 100 } } } @@ -475,12 +487,16 @@ func Test_buildCluster(t *testing.T) { "type": "STATIC", "connectTimeout": "10s", "respectDnsTtl": true, + "perConnectionBufferLimitBytes": 32768, "typedExtensionProtocolOptions": { "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", "explicitHttpConfig": { "http2ProtocolOptions": { - "allowConnect": true + "allowConnect": true, + "initialConnectionWindowSize": 1048576, + "initialStreamWindowSize": 65536, + "maxConcurrentStreams": 100 } } } @@ -530,12 +546,16 @@ func Test_buildCluster(t *testing.T) { "type": "STATIC", "connectTimeout": "10s", "respectDnsTtl": true, + "perConnectionBufferLimitBytes": 32768, "typedExtensionProtocolOptions": { "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", "explicitHttpConfig": { "http2ProtocolOptions": { - "allowConnect": true + "allowConnect": true, + "initialConnectionWindowSize": 1048576, + "initialStreamWindowSize": 65536, + "maxConcurrentStreams": 100 } } } @@ -578,12 +598,16 @@ func Test_buildCluster(t *testing.T) { "type": "STRICT_DNS", "connectTimeout": "10s", "respectDnsTtl": true, + "perConnectionBufferLimitBytes": 32768, "typedExtensionProtocolOptions": { "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", "explicitHttpConfig": { "http2ProtocolOptions": { - "allowConnect": true + "allowConnect": true, + "initialConnectionWindowSize": 1048576, + "initialStreamWindowSize": 65536, + "maxConcurrentStreams": 100 } } } diff --git a/config/envoyconfig/envoyconfig.go b/config/envoyconfig/envoyconfig.go index 7c97e31f0..be33d3ad2 100644 --- a/config/envoyconfig/envoyconfig.go +++ b/config/envoyconfig/envoyconfig.go @@ -27,6 +27,7 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" "github.com/pomerium/pomerium/config" "github.com/pomerium/pomerium/internal/log" @@ -66,9 +67,10 @@ func (e Endpoint) TransportSocketName() string { // newDefaultEnvoyClusterConfig creates envoy cluster with certain default values func newDefaultEnvoyClusterConfig() *envoy_config_cluster_v3.Cluster { return &envoy_config_cluster_v3.Cluster{ - ConnectTimeout: defaultConnectionTimeout, - RespectDnsTtl: true, - DnsLookupFamily: envoy_config_cluster_v3.Cluster_AUTO, + ConnectTimeout: defaultConnectionTimeout, + RespectDnsTtl: true, + DnsLookupFamily: envoy_config_cluster_v3.Cluster_AUTO, + PerConnectionBufferLimitBytes: wrapperspb.UInt32(connectionBufferLimit), } } diff --git a/config/envoyconfig/listeners.go b/config/envoyconfig/listeners.go index a702b62f5..8bf823af8 100644 --- a/config/envoyconfig/listeners.go +++ b/config/envoyconfig/listeners.go @@ -31,6 +31,8 @@ import ( "github.com/pomerium/pomerium/pkg/cryptutil" ) +const listenerBufferLimit uint32 = 32 * 1024 + var ( disableExtAuthz *any.Any tlsParams = &envoy_extensions_transport_sockets_tls_v3.TlsParameters{ @@ -108,16 +110,15 @@ func (b *Builder) buildMainListener(cfg *config.Config) (*envoy_config_listener_ return nil, err } - return &envoy_config_listener_v3.Listener{ - Name: "http-ingress", - Address: buildAddress(cfg.Options.Addr, 80), - ListenerFilters: listenerFilters, - FilterChains: []*envoy_config_listener_v3.FilterChain{{ - Filters: []*envoy_config_listener_v3.Filter{ - filter, - }, - }}, - }, nil + li := newEnvoyListener("http-ingress") + li.Address = buildAddress(cfg.Options.Addr, 80) + li.ListenerFilters = listenerFilters + li.FilterChains = []*envoy_config_listener_v3.FilterChain{{ + Filters: []*envoy_config_listener_v3.Filter{ + filter, + }, + }} + return li, nil } tlsInspectorCfg := marshalAny(new(emptypb.Empty)) @@ -158,12 +159,10 @@ func (b *Builder) buildMainListener(cfg *config.Config) (*envoy_config_listener_ return nil, err } - li := &envoy_config_listener_v3.Listener{ - Name: "https-ingress", - Address: buildAddress(cfg.Options.Addr, 443), - ListenerFilters: listenerFilters, - FilterChains: chains, - } + li := newEnvoyListener("https-ingress") + li.Address = buildAddress(cfg.Options.Addr, 443) + li.ListenerFilters = listenerFilters + li.FilterChains = chains return li, nil } @@ -239,11 +238,9 @@ func (b *Builder) buildMetricsListener(cfg *config.Config) (*envoy_config_listen host = "" } - li := &envoy_config_listener_v3.Listener{ - Name: "metrics-ingress", - Address: buildAddress(fmt.Sprintf("%s:%s", host, port), 9902), - FilterChains: []*envoy_config_listener_v3.FilterChain{filterChain}, - } + li := newEnvoyListener("metrics-ingress") + li.Address = buildAddress(fmt.Sprintf("%s:%s", host, port), 9902) + li.FilterChains = []*envoy_config_listener_v3.FilterChain{filterChain} return li, nil } @@ -532,15 +529,14 @@ func (b *Builder) buildGRPCListener(cfg *config.Config) (*envoy_config_listener_ } if cfg.Options.GetGRPCInsecure() { - return &envoy_config_listener_v3.Listener{ - Name: "grpc-ingress", - Address: buildAddress(cfg.Options.GetGRPCAddr(), 80), - FilterChains: []*envoy_config_listener_v3.FilterChain{{ - Filters: []*envoy_config_listener_v3.Filter{ - filter, - }, - }}, - }, nil + li := newEnvoyListener("grpc-ingress") + li.Address = buildAddress(cfg.Options.GetGRPCAddr(), 80) + li.FilterChains = []*envoy_config_listener_v3.FilterChain{{ + Filters: []*envoy_config_listener_v3.Filter{ + filter, + }, + }} + return li, nil } chains, err := b.buildFilterChains(cfg.Options, cfg.Options.Addr, @@ -570,17 +566,15 @@ func (b *Builder) buildGRPCListener(cfg *config.Config) (*envoy_config_listener_ } tlsInspectorCfg := marshalAny(new(emptypb.Empty)) - li := &envoy_config_listener_v3.Listener{ - Name: "grpc-ingress", - Address: buildAddress(cfg.Options.GetGRPCAddr(), 443), - ListenerFilters: []*envoy_config_listener_v3.ListenerFilter{{ - Name: "envoy.filters.listener.tls_inspector", - ConfigType: &envoy_config_listener_v3.ListenerFilter_TypedConfig{ - TypedConfig: tlsInspectorCfg, - }, - }}, - FilterChains: chains, - } + li := newEnvoyListener("grpc-ingress") + li.Address = buildAddress(cfg.Options.GetGRPCAddr(), 443) + li.ListenerFilters = []*envoy_config_listener_v3.ListenerFilter{{ + Name: "envoy.filters.listener.tls_inspector", + ConfigType: &envoy_config_listener_v3.ListenerFilter_TypedConfig{ + TypedConfig: tlsInspectorCfg, + }, + }} + li.FilterChains = chains return li, nil } @@ -848,3 +842,11 @@ func getPoliciesForDomain(options *config.Options, domain string) []config.Polic } return policies } + +// newEnvoyListener creates envoy listener with certain default values +func newEnvoyListener(name string) *envoy_config_listener_v3.Listener { + return &envoy_config_listener_v3.Listener{ + Name: name, + PerConnectionBufferLimitBytes: wrapperspb.UInt32(listenerBufferLimit), + } +} diff --git a/config/envoyconfig/listeners_test.go b/config/envoyconfig/listeners_test.go index 0e22b4829..12dfaaf52 100644 --- a/config/envoyconfig/listeners_test.go +++ b/config/envoyconfig/listeners_test.go @@ -36,6 +36,7 @@ func Test_buildMetricsHTTPConnectionManagerFilter(t *testing.T) { testutil.AssertProtoJSONEqual(t, ` { "name": "metrics-ingress", + "perConnectionBufferLimitBytes": 32768, "address": { "socketAddress": { "address": "127.0.0.1", diff --git a/config/envoyconfig/protocols.go b/config/envoyconfig/protocols.go index 6deadd1e1..a8502d312 100644 --- a/config/envoyconfig/protocols.go +++ b/config/envoyconfig/protocols.go @@ -3,8 +3,24 @@ package envoyconfig import ( envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_extensions_upstreams_http_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" + "google.golang.org/protobuf/types/known/wrapperspb" ) +// recommended defaults: https://www.envoyproxy.io/docs/envoy/latest/configuration/best_practices/edge +const ( + connectionBufferLimit uint32 = 32 * 1024 + maxConcurrentStreams uint32 = 100 + initialStreamWindowSizeLimit uint32 = 64 * 1024 + initialConnectionWindowSizeLimit uint32 = 1 * 1024 * 1024 +) + +var http2ProtocolOptions = &envoy_config_core_v3.Http2ProtocolOptions{ + AllowConnect: true, + MaxConcurrentStreams: wrapperspb.UInt32(maxConcurrentStreams), + InitialStreamWindowSize: wrapperspb.UInt32(initialStreamWindowSizeLimit), + InitialConnectionWindowSize: wrapperspb.UInt32(initialConnectionWindowSizeLimit), +} + func buildUpstreamProtocolOptions(endpoints []Endpoint, forceHTTP2 bool) *envoy_extensions_upstreams_http_v3.HttpProtocolOptions { // if forcing http/2, use that explicitly if forceHTTP2 { @@ -12,9 +28,7 @@ func buildUpstreamProtocolOptions(endpoints []Endpoint, forceHTTP2 bool) *envoy_ UpstreamProtocolOptions: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_{ ExplicitHttpConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig{ ProtocolConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_Http2ProtocolOptions{ - Http2ProtocolOptions: &envoy_config_core_v3.Http2ProtocolOptions{ - AllowConnect: true, - }, + Http2ProtocolOptions: http2ProtocolOptions, }, }, }, @@ -31,7 +45,9 @@ func buildUpstreamProtocolOptions(endpoints []Endpoint, forceHTTP2 bool) *envoy_ if tlsCount > 0 && tlsCount == len(endpoints) { return &envoy_extensions_upstreams_http_v3.HttpProtocolOptions{ UpstreamProtocolOptions: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_AutoConfig{ - AutoConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_AutoHttpConfig{}, + AutoConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_AutoHttpConfig{ + Http2ProtocolOptions: http2ProtocolOptions, + }, }, } }