From e3e7de741c8cc113f11d1578258f79dc32275997 Mon Sep 17 00:00:00 2001 From: Joe Kralicky Date: Tue, 13 Aug 2024 13:40:44 -0400 Subject: [PATCH] envoy: support http2 prior knowledge for insecure upstream targets (#5205) This allows using the scheme 'h2c' to indicate http2 prior knowledge for insecure upstream servers. This can be used to perform TLS termination for GRPC servers configured with insecure credentials. As an example, this allows the following route configuration: routes: - from: https://grpc.localhost.pomerium.io to: h2c://localhost:9090 --- config/envoyconfig/protocols.go | 25 +++-- config/envoyconfig/protocols_int_test.go | 102 +++++++++++++++++++ config/envoyconfig/protocols_test.go | 122 +++++++++++++++++++---- 3 files changed, 224 insertions(+), 25 deletions(-) create mode 100644 config/envoyconfig/protocols_int_test.go diff --git a/config/envoyconfig/protocols.go b/config/envoyconfig/protocols.go index 05eb941ae..2e2c8168a 100644 --- a/config/envoyconfig/protocols.go +++ b/config/envoyconfig/protocols.go @@ -54,6 +54,7 @@ var http2ProtocolOptions = &envoy_config_core_v3.Http2ProtocolOptions{ InitialStreamWindowSize: wrapperspb.UInt32(initialStreamWindowSizeLimit), InitialConnectionWindowSize: wrapperspb.UInt32(initialConnectionWindowSizeLimit), } +var http2ProtocolOptionsWithKeepalive = WithKeepalive(http2ProtocolOptions) func WithKeepalive(src *envoy_config_core_v3.Http2ProtocolOptions) *envoy_config_core_v3.Http2ProtocolOptions { dst := proto.Clone(src).(*envoy_config_core_v3.Http2ProtocolOptions) @@ -81,12 +82,12 @@ func buildUpstreamProtocolOptions( upstreamProtocol upstreamProtocolConfig, keepalive Keepalive, ) *envoy_extensions_upstreams_http_v3.HttpProtocolOptions { + h2opt := http2ProtocolOptions + if keepalive { + h2opt = http2ProtocolOptionsWithKeepalive + } switch upstreamProtocol { case upstreamProtocolHTTP2: - h2opt := http2ProtocolOptions - if keepalive { - h2opt = WithKeepalive(http2ProtocolOptions) - } // when explicitly configured, force HTTP/2 return &envoy_extensions_upstreams_http_v3.HttpProtocolOptions{ UpstreamProtocolOptions: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_{ @@ -99,10 +100,12 @@ func buildUpstreamProtocolOptions( } case upstreamProtocolAuto: // when using TLS use ALPN auto config - tlsCount := 0 + var tlsCount, h2cCount int for _, e := range endpoints { if e.transportSocket != nil { tlsCount++ + } else if e.url.Scheme == "h2c" { + h2cCount++ } } if tlsCount > 0 && tlsCount == len(endpoints) { @@ -110,7 +113,17 @@ func buildUpstreamProtocolOptions( UpstreamProtocolOptions: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_AutoConfig{ AutoConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_AutoHttpConfig{ HttpProtocolOptions: http1ProtocolOptions, - Http2ProtocolOptions: http2ProtocolOptions, + Http2ProtocolOptions: h2opt, + }, + }, + } + } else if h2cCount > 0 && h2cCount == len(endpoints) { + return &envoy_extensions_upstreams_http_v3.HttpProtocolOptions{ + UpstreamProtocolOptions: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_{ + ExplicitHttpConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig{ + ProtocolConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_Http2ProtocolOptions{ + Http2ProtocolOptions: h2opt, + }, }, }, } diff --git a/config/envoyconfig/protocols_int_test.go b/config/envoyconfig/protocols_int_test.go new file mode 100644 index 000000000..19f2833a6 --- /dev/null +++ b/config/envoyconfig/protocols_int_test.go @@ -0,0 +1,102 @@ +package envoyconfig_test + +import ( + "context" + "fmt" + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/interop" + "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + "github.com/pomerium/pomerium/config" + "github.com/pomerium/pomerium/pkg/cmd/pomerium" + "github.com/pomerium/pomerium/pkg/netutil" +) + +func TestH2C(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + + ctx, ca := context.WithCancel(context.Background()) + + opts := config.NewDefaultOptions() + listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", "127.0.0.1:0") + require.NoError(t, err) + ports, err := netutil.AllocatePorts(7) + require.NoError(t, err) + urls, err := config.ParseWeightedUrls("http://"+listener.Addr().String(), "h2c://"+listener.Addr().String()) + require.NoError(t, err) + opts.Addr = fmt.Sprintf("127.0.0.1:%s", ports[0]) + opts.Routes = []config.Policy{ + { + From: fmt.Sprintf("https://grpc-http.localhost.pomerium.io:%s", ports[0]), + To: urls[:1], + AllowPublicUnauthenticatedAccess: true, + }, + { + From: fmt.Sprintf("https://grpc-h2c.localhost.pomerium.io:%s", ports[0]), + To: urls[1:], + AllowPublicUnauthenticatedAccess: true, + }, + } + opts.CertFile = "../../integration/tpl/files/trusted.pem" + opts.KeyFile = "../../integration/tpl/files/trusted-key.pem" + cfg := &config.Config{Options: opts} + cfg.AllocatePorts(*(*[6]string)(ports[1:])) + + server := grpc.NewServer(grpc.Creds(insecure.NewCredentials())) + grpc_testing.RegisterTestServiceServer(server, interop.NewTestServer()) + go server.Serve(listener) + + errC := make(chan error, 1) + go func() { + errC <- pomerium.Run(ctx, config.NewStaticSource(cfg)) + }() + + t.Cleanup(func() { + ca() + assert.ErrorIs(t, context.Canceled, <-errC) + }) + + tlsConfig, err := credentials.NewClientTLSFromFile("../../integration/tpl/files/ca.pem", "") + require.NoError(t, err) + + t.Run("h2c", func(t *testing.T) { + t.Parallel() + + cc, err := grpc.Dial(fmt.Sprintf("grpc-h2c.localhost.pomerium.io:%s", ports[0]), grpc.WithTransportCredentials(tlsConfig)) + require.NoError(t, err) + client := grpc_testing.NewTestServiceClient(cc) + var md metadata.MD + _, err = client.EmptyCall(ctx, &grpc_testing.Empty{}, grpc.WaitForReady(true), grpc.Header(&md)) + cc.Close() + assert.NoError(t, err) + assert.Contains(t, md, "x-envoy-upstream-service-time") + }) + t.Run("http", func(t *testing.T) { + t.Parallel() + + cc, err := grpc.Dial(fmt.Sprintf("grpc-http.localhost.pomerium.io:%s", ports[0]), grpc.WithTransportCredentials(tlsConfig)) + require.NoError(t, err) + client := grpc_testing.NewTestServiceClient(cc) + var md metadata.MD + _, err = client.EmptyCall(ctx, &grpc_testing.Empty{}, grpc.WaitForReady(true), grpc.Trailer(&md)) + cc.Close() + stat := status.Convert(err) + assert.NotNil(t, stat) + assert.Equal(t, stat.Code(), codes.Unavailable) + assert.NotContains(t, md, "x-envoy-upstream-service-time") + assert.Contains(t, stat.Message(), "") + assert.Contains(t, stat.Message(), "upstream_reset_before_response_started{protocol_error}") + }) +} diff --git a/config/envoyconfig/protocols_test.go b/config/envoyconfig/protocols_test.go index f65f507fa..e2b4877c7 100644 --- a/config/envoyconfig/protocols_test.go +++ b/config/envoyconfig/protocols_test.go @@ -4,33 +4,117 @@ import ( "testing" envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - envoy_extensions_http_header_formatters_preserve_case_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/http/header_formatters/preserve_case/v3" envoy_extensions_upstreams_http_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" - "github.com/google/go-cmp/cmp" - "github.com/stretchr/testify/assert" - "google.golang.org/protobuf/testing/protocmp" + + "github.com/pomerium/pomerium/internal/testutil" ) -func Test_buildUpstreamProtocolOptions(t *testing.T) { +func TestBuildUpstreamProtocolOptions(t *testing.T) { t.Parallel() - assert.Empty(t, - cmp.Diff(&envoy_extensions_upstreams_http_v3.HttpProtocolOptions{ + var ( + explicitH2 = &envoy_extensions_upstreams_http_v3.HttpProtocolOptions{ UpstreamProtocolOptions: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_{ ExplicitHttpConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig{ - ProtocolConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_HttpProtocolOptions{ - HttpProtocolOptions: &envoy_config_core_v3.Http1ProtocolOptions{ - HeaderKeyFormat: &envoy_config_core_v3.Http1ProtocolOptions_HeaderKeyFormat{ - HeaderFormat: &envoy_config_core_v3.Http1ProtocolOptions_HeaderKeyFormat_StatefulFormatter{ - StatefulFormatter: &envoy_config_core_v3.TypedExtensionConfig{ - Name: "preserve_case", - TypedConfig: marshalAny(&envoy_extensions_http_header_formatters_preserve_case_v3.PreserveCaseFormatterConfig{}), - }, - }, - }, - }, + ProtocolConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_Http2ProtocolOptions{ + Http2ProtocolOptions: http2ProtocolOptions, }, }, }, - }, buildUpstreamProtocolOptions(nil, upstreamProtocolHTTP1, Keepalive(false)), protocmp.Transform())) + } + explicitH2Keepalive = &envoy_extensions_upstreams_http_v3.HttpProtocolOptions{ + UpstreamProtocolOptions: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_{ + ExplicitHttpConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig{ + ProtocolConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_Http2ProtocolOptions{ + Http2ProtocolOptions: http2ProtocolOptionsWithKeepalive, + }, + }, + }, + } + explicitH1 = &envoy_extensions_upstreams_http_v3.HttpProtocolOptions{ + UpstreamProtocolOptions: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_{ + ExplicitHttpConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig{ + ProtocolConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_HttpProtocolOptions{ + HttpProtocolOptions: http1ProtocolOptions, + }, + }, + }, + } + explicitH1Keepalive = &envoy_extensions_upstreams_http_v3.HttpProtocolOptions{ + UpstreamProtocolOptions: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_{ + ExplicitHttpConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig{ + ProtocolConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_HttpProtocolOptions{ + HttpProtocolOptions: http1ProtocolOptions, + }, + }, + }, + } + auto = &envoy_extensions_upstreams_http_v3.HttpProtocolOptions{ + UpstreamProtocolOptions: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_AutoConfig{ + AutoConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_AutoHttpConfig{ + HttpProtocolOptions: http1ProtocolOptions, + Http2ProtocolOptions: http2ProtocolOptions, + }, + }, + } + autoKeepalive = &envoy_extensions_upstreams_http_v3.HttpProtocolOptions{ + UpstreamProtocolOptions: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_AutoConfig{ + AutoConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_AutoHttpConfig{ + HttpProtocolOptions: http1ProtocolOptions, + Http2ProtocolOptions: http2ProtocolOptionsWithKeepalive, + }, + }, + } + ) + cases := []struct { + endpoints []string + protocol upstreamProtocolConfig + keepalive bool + expected *envoy_extensions_upstreams_http_v3.HttpProtocolOptions + }{ + {[]string{"https://foo", "https://bar"}, upstreamProtocolHTTP1, false, explicitH1}, + {[]string{"https://foo", "https://bar"}, upstreamProtocolHTTP1, true, explicitH1Keepalive}, + {[]string{"http://foo", "https://bar"}, upstreamProtocolHTTP1, false, explicitH1}, + {[]string{"http://foo", "https://bar"}, upstreamProtocolHTTP1, true, explicitH1Keepalive}, + {[]string{"http://foo", "http://bar"}, upstreamProtocolHTTP1, false, explicitH1}, + {[]string{"http://foo", "http://bar"}, upstreamProtocolHTTP1, true, explicitH1Keepalive}, + + {[]string{"https://foo", "https://bar"}, upstreamProtocolHTTP2, false, explicitH2}, + {[]string{"https://foo", "https://bar"}, upstreamProtocolHTTP2, true, explicitH2Keepalive}, + {[]string{"http://foo", "https://bar"}, upstreamProtocolHTTP2, false, explicitH2}, + {[]string{"http://foo", "https://bar"}, upstreamProtocolHTTP2, true, explicitH2Keepalive}, + {[]string{"http://foo", "http://bar"}, upstreamProtocolHTTP2, false, explicitH2}, + {[]string{"http://foo", "http://bar"}, upstreamProtocolHTTP2, true, explicitH2Keepalive}, + + {[]string{"https://foo", "https://bar"}, upstreamProtocolAuto, false, auto}, + {[]string{"https://foo", "https://bar"}, upstreamProtocolAuto, true, autoKeepalive}, + {[]string{"http://foo", "https://bar"}, upstreamProtocolAuto, false, explicitH1}, + {[]string{"http://foo", "https://bar"}, upstreamProtocolAuto, true, explicitH1Keepalive}, + {[]string{"http://foo", "http://bar"}, upstreamProtocolAuto, false, explicitH1}, + {[]string{"http://foo", "http://bar"}, upstreamProtocolAuto, true, explicitH1Keepalive}, + + {[]string{"h2c://foo", "http://bar"}, upstreamProtocolAuto, false, explicitH1}, + {[]string{"h2c://foo", "http://bar"}, upstreamProtocolAuto, true, explicitH1Keepalive}, + {[]string{"h2c://foo", "https://bar"}, upstreamProtocolAuto, false, explicitH1}, + {[]string{"h2c://foo", "https://bar"}, upstreamProtocolAuto, true, explicitH1Keepalive}, + {[]string{"h2c://foo", "h2c://bar"}, upstreamProtocolAuto, false, explicitH2}, + {[]string{"h2c://foo", "h2c://bar"}, upstreamProtocolAuto, true, explicitH2Keepalive}, + } + + for _, tc := range cases { + t.Run("", func(t *testing.T) { + endpoints := []Endpoint{} + for _, e := range tc.endpoints { + endpoint := Endpoint{url: *mustParseURL(t, e)} + // match logic from buildInternalTransportSocket + if endpoint.url.Scheme == "https" { + // buildUpstreamProtocolOptions only checks for the presence of + // transportSocket, and does not inspect any of its contents + endpoint.transportSocket = &envoy_config_core_v3.TransportSocket{} + } + endpoints = append(endpoints, endpoint) + } + testutil.AssertProtoEqual(t, tc.expected, buildUpstreamProtocolOptions(endpoints, tc.protocol, Keepalive(tc.keepalive))) + }) + } }