envoy: support http2 prior knowledge for insecure upstream targets (#5205)

This allows using the scheme 'h2c' to indicate http2 prior knowledge for
insecure upstream servers. This can be used to perform TLS termination for
GRPC servers configured with insecure credentials.

As an example, this allows the following route configuration:

routes:
  - from: https://grpc.localhost.pomerium.io
    to: h2c://localhost:9090

config/envoyconfig/protocols.go

@@ -54,6 +54,7 @@ var http2ProtocolOptions = &envoy_config_core_v3.Http2ProtocolOptions{
 	InitialStreamWindowSize:     wrapperspb.UInt32(initialStreamWindowSizeLimit),
 	InitialConnectionWindowSize: wrapperspb.UInt32(initialConnectionWindowSizeLimit),
 }
+var http2ProtocolOptionsWithKeepalive = WithKeepalive(http2ProtocolOptions)
 
 func WithKeepalive(src *envoy_config_core_v3.Http2ProtocolOptions) *envoy_config_core_v3.Http2ProtocolOptions {
 	dst := proto.Clone(src).(*envoy_config_core_v3.Http2ProtocolOptions)
@@ -81,12 +82,12 @@ func buildUpstreamProtocolOptions(
 	upstreamProtocol upstreamProtocolConfig,
 	keepalive Keepalive,
 ) *envoy_extensions_upstreams_http_v3.HttpProtocolOptions {
-	switch upstreamProtocol {
-	case upstreamProtocolHTTP2:
 	h2opt := http2ProtocolOptions
 	if keepalive {
-			h2opt = WithKeepalive(http2ProtocolOptions)
+		h2opt = http2ProtocolOptionsWithKeepalive
 	}
+	switch upstreamProtocol {
+	case upstreamProtocolHTTP2:
 		// when explicitly configured, force HTTP/2
 		return &envoy_extensions_upstreams_http_v3.HttpProtocolOptions{
 			UpstreamProtocolOptions: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_{
@@ -99,10 +100,12 @@ func buildUpstreamProtocolOptions(
 		}
 	case upstreamProtocolAuto:
 		// when using TLS use ALPN auto config
-		tlsCount := 0
+		var tlsCount, h2cCount int
 		for _, e := range endpoints {
 			if e.transportSocket != nil {
 				tlsCount++
+			} else if e.url.Scheme == "h2c" {
+				h2cCount++
 			}
 		}
 		if tlsCount > 0 && tlsCount == len(endpoints) {
@@ -110,7 +113,17 @@ func buildUpstreamProtocolOptions(
 				UpstreamProtocolOptions: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_AutoConfig{
 					AutoConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_AutoHttpConfig{
 						HttpProtocolOptions:  http1ProtocolOptions,
-						Http2ProtocolOptions: http2ProtocolOptions,
+						Http2ProtocolOptions: h2opt,
+					},
+				},
+			}
+		} else if h2cCount > 0 && h2cCount == len(endpoints) {
+			return &envoy_extensions_upstreams_http_v3.HttpProtocolOptions{
+				UpstreamProtocolOptions: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_{
+					ExplicitHttpConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig{
+						ProtocolConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_Http2ProtocolOptions{
+							Http2ProtocolOptions: h2opt,
+						},
 					},
 				},
 			}

config/envoyconfig/protocols_int_test.go

@@ -0,0 +1,102 @@
+package envoyconfig_test
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/interop"
+	"google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
+
+	"github.com/pomerium/pomerium/config"
+	"github.com/pomerium/pomerium/pkg/cmd/pomerium"
+	"github.com/pomerium/pomerium/pkg/netutil"
+)
+
+func TestH2C(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+	}
+
+	ctx, ca := context.WithCancel(context.Background())
+
+	opts := config.NewDefaultOptions()
+	listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	ports, err := netutil.AllocatePorts(7)
+	require.NoError(t, err)
+	urls, err := config.ParseWeightedUrls("http://"+listener.Addr().String(), "h2c://"+listener.Addr().String())
+	require.NoError(t, err)
+	opts.Addr = fmt.Sprintf("127.0.0.1:%s", ports[0])
+	opts.Routes = []config.Policy{
+		{
+			From:                             fmt.Sprintf("https://grpc-http.localhost.pomerium.io:%s", ports[0]),
+			To:                               urls[:1],
+			AllowPublicUnauthenticatedAccess: true,
+		},
+		{
+			From:                             fmt.Sprintf("https://grpc-h2c.localhost.pomerium.io:%s", ports[0]),
+			To:                               urls[1:],
+			AllowPublicUnauthenticatedAccess: true,
+		},
+	}
+	opts.CertFile = "../../integration/tpl/files/trusted.pem"
+	opts.KeyFile = "../../integration/tpl/files/trusted-key.pem"
+	cfg := &config.Config{Options: opts}
+	cfg.AllocatePorts(*(*[6]string)(ports[1:]))
+
+	server := grpc.NewServer(grpc.Creds(insecure.NewCredentials()))
+	grpc_testing.RegisterTestServiceServer(server, interop.NewTestServer())
+	go server.Serve(listener)
+
+	errC := make(chan error, 1)
+	go func() {
+		errC <- pomerium.Run(ctx, config.NewStaticSource(cfg))
+	}()
+
+	t.Cleanup(func() {
+		ca()
+		assert.ErrorIs(t, context.Canceled, <-errC)
+	})
+
+	tlsConfig, err := credentials.NewClientTLSFromFile("../../integration/tpl/files/ca.pem", "")
+	require.NoError(t, err)
+
+	t.Run("h2c", func(t *testing.T) {
+		t.Parallel()
+
+		cc, err := grpc.Dial(fmt.Sprintf("grpc-h2c.localhost.pomerium.io:%s", ports[0]), grpc.WithTransportCredentials(tlsConfig))
+		require.NoError(t, err)
+		client := grpc_testing.NewTestServiceClient(cc)
+		var md metadata.MD
+		_, err = client.EmptyCall(ctx, &grpc_testing.Empty{}, grpc.WaitForReady(true), grpc.Header(&md))
+		cc.Close()
+		assert.NoError(t, err)
+		assert.Contains(t, md, "x-envoy-upstream-service-time")
+	})
+	t.Run("http", func(t *testing.T) {
+		t.Parallel()
+
+		cc, err := grpc.Dial(fmt.Sprintf("grpc-http.localhost.pomerium.io:%s", ports[0]), grpc.WithTransportCredentials(tlsConfig))
+		require.NoError(t, err)
+		client := grpc_testing.NewTestServiceClient(cc)
+		var md metadata.MD
+		_, err = client.EmptyCall(ctx, &grpc_testing.Empty{}, grpc.WaitForReady(true), grpc.Trailer(&md))
+		cc.Close()
+		stat := status.Convert(err)
+		assert.NotNil(t, stat)
+		assert.Equal(t, stat.Code(), codes.Unavailable)
+		assert.NotContains(t, md, "x-envoy-upstream-service-time")
+		assert.Contains(t, stat.Message(), "<!DOCTYPE html>")
+		assert.Contains(t, stat.Message(), "upstream_reset_before_response_started{protocol_error}")
+	})
+}

config/envoyconfig/protocols_test.go

@@ -4,33 +4,117 @@ import (
 	"testing"
 
 	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
-	envoy_extensions_http_header_formatters_preserve_case_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/http/header_formatters/preserve_case/v3"
 	envoy_extensions_upstreams_http_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3"
-	"github.com/google/go-cmp/cmp"
+
-	"github.com/stretchr/testify/assert"
+	"github.com/pomerium/pomerium/internal/testutil"
-	"google.golang.org/protobuf/testing/protocmp"
 )
 
-func Test_buildUpstreamProtocolOptions(t *testing.T) {
+func TestBuildUpstreamProtocolOptions(t *testing.T) {
 	t.Parallel()
 
-	assert.Empty(t,
+	var (
-		cmp.Diff(&envoy_extensions_upstreams_http_v3.HttpProtocolOptions{
+		explicitH2 = &envoy_extensions_upstreams_http_v3.HttpProtocolOptions{
+			UpstreamProtocolOptions: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_{
+				ExplicitHttpConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig{
+					ProtocolConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_Http2ProtocolOptions{
+						Http2ProtocolOptions: http2ProtocolOptions,
+					},
+				},
+			},
+		}
+		explicitH2Keepalive = &envoy_extensions_upstreams_http_v3.HttpProtocolOptions{
+			UpstreamProtocolOptions: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_{
+				ExplicitHttpConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig{
+					ProtocolConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_Http2ProtocolOptions{
+						Http2ProtocolOptions: http2ProtocolOptionsWithKeepalive,
+					},
+				},
+			},
+		}
+		explicitH1 = &envoy_extensions_upstreams_http_v3.HttpProtocolOptions{
 			UpstreamProtocolOptions: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_{
 				ExplicitHttpConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig{
 					ProtocolConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_HttpProtocolOptions{
-						HttpProtocolOptions: &envoy_config_core_v3.Http1ProtocolOptions{
+						HttpProtocolOptions: http1ProtocolOptions,
-							HeaderKeyFormat: &envoy_config_core_v3.Http1ProtocolOptions_HeaderKeyFormat{
-								HeaderFormat: &envoy_config_core_v3.Http1ProtocolOptions_HeaderKeyFormat_StatefulFormatter{
-									StatefulFormatter: &envoy_config_core_v3.TypedExtensionConfig{
-										Name:        "preserve_case",
-										TypedConfig: marshalAny(&envoy_extensions_http_header_formatters_preserve_case_v3.PreserveCaseFormatterConfig{}),
 					},
 				},
 			},
+		}
+		explicitH1Keepalive = &envoy_extensions_upstreams_http_v3.HttpProtocolOptions{
+			UpstreamProtocolOptions: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_{
+				ExplicitHttpConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig{
+					ProtocolConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_ExplicitHttpConfig_HttpProtocolOptions{
+						HttpProtocolOptions: http1ProtocolOptions,
 					},
 				},
 			},
+		}
+		auto = &envoy_extensions_upstreams_http_v3.HttpProtocolOptions{
+			UpstreamProtocolOptions: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_AutoConfig{
+				AutoConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_AutoHttpConfig{
+					HttpProtocolOptions:  http1ProtocolOptions,
+					Http2ProtocolOptions: http2ProtocolOptions,
 				},
-		}, buildUpstreamProtocolOptions(nil, upstreamProtocolHTTP1, Keepalive(false)), protocmp.Transform()))
+			},
+		}
+		autoKeepalive = &envoy_extensions_upstreams_http_v3.HttpProtocolOptions{
+			UpstreamProtocolOptions: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_AutoConfig{
+				AutoConfig: &envoy_extensions_upstreams_http_v3.HttpProtocolOptions_AutoHttpConfig{
+					HttpProtocolOptions:  http1ProtocolOptions,
+					Http2ProtocolOptions: http2ProtocolOptionsWithKeepalive,
+				},
+			},
+		}
+	)
+	cases := []struct {
+		endpoints []string
+		protocol  upstreamProtocolConfig
+		keepalive bool
+		expected  *envoy_extensions_upstreams_http_v3.HttpProtocolOptions
+	}{
+		{[]string{"https://foo", "https://bar"}, upstreamProtocolHTTP1, false, explicitH1},
+		{[]string{"https://foo", "https://bar"}, upstreamProtocolHTTP1, true, explicitH1Keepalive},
+		{[]string{"http://foo", "https://bar"}, upstreamProtocolHTTP1, false, explicitH1},
+		{[]string{"http://foo", "https://bar"}, upstreamProtocolHTTP1, true, explicitH1Keepalive},
+		{[]string{"http://foo", "http://bar"}, upstreamProtocolHTTP1, false, explicitH1},
+		{[]string{"http://foo", "http://bar"}, upstreamProtocolHTTP1, true, explicitH1Keepalive},
+
+		{[]string{"https://foo", "https://bar"}, upstreamProtocolHTTP2, false, explicitH2},
+		{[]string{"https://foo", "https://bar"}, upstreamProtocolHTTP2, true, explicitH2Keepalive},
+		{[]string{"http://foo", "https://bar"}, upstreamProtocolHTTP2, false, explicitH2},
+		{[]string{"http://foo", "https://bar"}, upstreamProtocolHTTP2, true, explicitH2Keepalive},
+		{[]string{"http://foo", "http://bar"}, upstreamProtocolHTTP2, false, explicitH2},
+		{[]string{"http://foo", "http://bar"}, upstreamProtocolHTTP2, true, explicitH2Keepalive},
+
+		{[]string{"https://foo", "https://bar"}, upstreamProtocolAuto, false, auto},
+		{[]string{"https://foo", "https://bar"}, upstreamProtocolAuto, true, autoKeepalive},
+		{[]string{"http://foo", "https://bar"}, upstreamProtocolAuto, false, explicitH1},
+		{[]string{"http://foo", "https://bar"}, upstreamProtocolAuto, true, explicitH1Keepalive},
+		{[]string{"http://foo", "http://bar"}, upstreamProtocolAuto, false, explicitH1},
+		{[]string{"http://foo", "http://bar"}, upstreamProtocolAuto, true, explicitH1Keepalive},
+
+		{[]string{"h2c://foo", "http://bar"}, upstreamProtocolAuto, false, explicitH1},
+		{[]string{"h2c://foo", "http://bar"}, upstreamProtocolAuto, true, explicitH1Keepalive},
+		{[]string{"h2c://foo", "https://bar"}, upstreamProtocolAuto, false, explicitH1},
+		{[]string{"h2c://foo", "https://bar"}, upstreamProtocolAuto, true, explicitH1Keepalive},
+		{[]string{"h2c://foo", "h2c://bar"}, upstreamProtocolAuto, false, explicitH2},
+		{[]string{"h2c://foo", "h2c://bar"}, upstreamProtocolAuto, true, explicitH2Keepalive},
+	}
+
+	for _, tc := range cases {
+		t.Run("", func(t *testing.T) {
+			endpoints := []Endpoint{}
+			for _, e := range tc.endpoints {
+				endpoint := Endpoint{url: *mustParseURL(t, e)}
+				// match logic from buildInternalTransportSocket
+				if endpoint.url.Scheme == "https" {
+					// buildUpstreamProtocolOptions only checks for the presence of
+					// transportSocket, and does not inspect any of its contents
+					endpoint.transportSocket = &envoy_config_core_v3.TransportSocket{}
+				}
+				endpoints = append(endpoints, endpoint)
+			}
+			testutil.AssertProtoEqual(t, tc.expected, buildUpstreamProtocolOptions(endpoints, tc.protocol, Keepalive(tc.keepalive)))
+		})
+	}
 }