envoyconfig: move most bootstrap config to shared package (#2088)

config/envoyconfig/bootstrap.go

@@ -0,0 +1,126 @@
+package envoyconfig
+
+import (
+	"fmt"
+
+	envoy_config_bootstrap_v3 "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
+	envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	envoy_config_metrics_v3 "github.com/envoyproxy/go-control-plane/envoy/config/metrics/v3"
+	"google.golang.org/protobuf/types/known/durationpb"
+
+	"github.com/pomerium/pomerium/config"
+	"github.com/pomerium/pomerium/internal/telemetry"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
+)
+
+// BuildBootstrapAdmin builds the admin config for the envoy bootstrap.
+func (b *Builder) BuildBootstrapAdmin(cfg *config.Config) (*envoy_config_bootstrap_v3.Admin, error) {
+	adminAddr, err := parseAddress(cfg.Options.EnvoyAdminAddress)
+	if err != nil {
+		return nil, fmt.Errorf("envoyconfig: invalid envoy admin address: %w", err)
+	}
+	return &envoy_config_bootstrap_v3.Admin{
+		AccessLogPath: cfg.Options.EnvoyAdminAccessLogPath,
+		ProfilePath:   cfg.Options.EnvoyAdminProfilePath,
+		Address:       adminAddr,
+	}, nil
+}
+
+// BuildBootstrapStaticResources builds the static resources for the envoy bootstrap. It includes the control plane
+// cluster as well as a datadog-apm cluster (if datadog is used).
+func (b *Builder) BuildBootstrapStaticResources(cfg *config.Config) (*envoy_config_bootstrap_v3.Bootstrap_StaticResources, error) {
+	grpcAddr, err := parseAddress(b.localGRPCAddress)
+	if err != nil {
+		return nil, fmt.Errorf("envoyconfig: invalid local gRPC address: %w", err)
+	}
+
+	controlPlaneEndpoint := &envoy_config_endpoint_v3.LbEndpoint_Endpoint{
+		Endpoint: &envoy_config_endpoint_v3.Endpoint{
+			Address: grpcAddr,
+		},
+	}
+
+	controlPlaneCluster := &envoy_config_cluster_v3.Cluster{
+		Name: "pomerium-control-plane-grpc",
+		ConnectTimeout: &durationpb.Duration{
+			Seconds: 5,
+		},
+		ClusterDiscoveryType: &envoy_config_cluster_v3.Cluster_Type{
+			Type: envoy_config_cluster_v3.Cluster_STATIC,
+		},
+		LbPolicy: envoy_config_cluster_v3.Cluster_ROUND_ROBIN,
+		LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{
+			ClusterName: "pomerium-control-plane-grpc",
+			Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{
+				{
+					LbEndpoints: []*envoy_config_endpoint_v3.LbEndpoint{
+						{
+							HostIdentifier: controlPlaneEndpoint,
+						},
+					},
+				},
+			},
+		},
+		Http2ProtocolOptions: &envoy_config_core_v3.Http2ProtocolOptions{},
+	}
+
+	staticCfg := &envoy_config_bootstrap_v3.Bootstrap_StaticResources{
+		Clusters: []*envoy_config_cluster_v3.Cluster{
+			controlPlaneCluster,
+		},
+	}
+
+	if cfg.Options.TracingProvider == trace.DatadogTracingProviderName {
+		addr, _ := parseAddress("127.0.0.1:8126")
+
+		if cfg.Options.TracingDatadogAddress != "" {
+			addr, err = parseAddress(cfg.Options.TracingDatadogAddress)
+			if err != nil {
+				return nil, fmt.Errorf("envoyconfig: invalid tracing datadog address: %w", err)
+			}
+		}
+
+		staticCfg.Clusters = append(staticCfg.Clusters, &envoy_config_cluster_v3.Cluster{
+			Name: "datadog-apm",
+			ConnectTimeout: &durationpb.Duration{
+				Seconds: 5,
+			},
+			ClusterDiscoveryType: &envoy_config_cluster_v3.Cluster_Type{
+				Type: envoy_config_cluster_v3.Cluster_STATIC,
+			},
+			LbPolicy: envoy_config_cluster_v3.Cluster_ROUND_ROBIN,
+			LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{
+				ClusterName: "datadog-apm",
+				Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{
+					{
+						LbEndpoints: []*envoy_config_endpoint_v3.LbEndpoint{
+							{
+								HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{
+									Endpoint: &envoy_config_endpoint_v3.Endpoint{
+										Address: addr,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		})
+	}
+
+	return staticCfg, nil
+}
+
+// BuildBootstrapStatsConfig builds a the stats config the envoy bootstrap.
+func (b *Builder) BuildBootstrapStatsConfig(cfg *config.Config) (*envoy_config_metrics_v3.StatsConfig, error) {
+	statsCfg := &envoy_config_metrics_v3.StatsConfig{}
+	statsCfg.StatsTags = []*envoy_config_metrics_v3.TagSpecifier{{
+		TagName: "service",
+		TagValue: &envoy_config_metrics_v3.TagSpecifier_FixedValue{
+			FixedValue: telemetry.ServiceName(cfg.Options.Services),
+		},
+	}}
+	return statsCfg, nil
+}

config/envoyconfig/bootstrap_test.go

@@ -0,0 +1,138 @@
+package envoyconfig
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/pomerium/pomerium/config"
+	"github.com/pomerium/pomerium/config/envoyconfig/filemgr"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
+	"github.com/pomerium/pomerium/internal/testutil"
+)
+
+func TestBuilder_BuildBootstrapAdmin(t *testing.T) {
+	b := New("local-grpc", "local-http", filemgr.NewManager(), nil)
+	t.Run("valid", func(t *testing.T) {
+		adminCfg, err := b.BuildBootstrapAdmin(&config.Config{
+			Options: &config.Options{
+				EnvoyAdminAddress: "localhost:9901",
+			},
+		})
+		assert.NoError(t, err)
+		testutil.AssertProtoJSONEqual(t, `
+			{
+				"address": {
+					"socketAddress": {
+						"address": "127.0.0.1",
+						"portValue": 9901
+					}
+				}
+			}
+		`, adminCfg)
+	})
+	t.Run("bad address", func(t *testing.T) {
+		_, err := b.BuildBootstrapAdmin(&config.Config{
+			Options: &config.Options{
+				EnvoyAdminAddress: "xyz1234:zyx4321",
+			},
+		})
+		assert.Error(t, err)
+	})
+}
+
+func TestBuilder_BuildBootstrapStaticResources(t *testing.T) {
+	t.Run("valid", func(t *testing.T) {
+		b := New("localhost:1111", "localhost:2222", filemgr.NewManager(), nil)
+		staticCfg, err := b.BuildBootstrapStaticResources(&config.Config{
+			Options: &config.Options{
+				TracingProvider: trace.DatadogTracingProviderName,
+			},
+		})
+		assert.NoError(t, err)
+		testutil.AssertProtoJSONEqual(t, `
+			{
+				"clusters": [
+					{
+						"name": "pomerium-control-plane-grpc",
+						"type": "STATIC",
+						"connectTimeout": "5s",
+						"http2ProtocolOptions": {},
+						"loadAssignment": {
+							"clusterName": "pomerium-control-plane-grpc",
+							"endpoints": [{
+								"lbEndpoints": [{
+									"endpoint": {
+										"address": {
+											"socketAddress":{
+												"address": "127.0.0.1",
+												"portValue": 1111
+											}
+										}
+									}
+								}]
+							}]
+						}
+					},
+					{
+						"name": "datadog-apm",
+						"type": "STATIC",
+						"connectTimeout": "5s",
+						"loadAssignment": {
+							"clusterName": "datadog-apm",
+							"endpoints": [{
+								"lbEndpoints": [{
+									"endpoint": {
+										"address": {
+											"socketAddress":{
+												"address": "127.0.0.1",
+												"portValue": 8126
+											}
+										}
+									}
+								}]
+							}]
+						}
+					}
+				]
+			}
+		`, staticCfg)
+	})
+	t.Run("bad gRPC address", func(t *testing.T) {
+		b := New("xyz:zyx", "localhost:2222", filemgr.NewManager(), nil)
+		_, err := b.BuildBootstrapStaticResources(&config.Config{
+			Options: &config.Options{},
+		})
+		assert.Error(t, err)
+	})
+	t.Run("bad datadog address", func(t *testing.T) {
+		b := New("localhost:1111", "localhost:2222", filemgr.NewManager(), nil)
+		_, err := b.BuildBootstrapStaticResources(&config.Config{
+			Options: &config.Options{
+				TracingProvider:       trace.DatadogTracingProviderName,
+				TracingDatadogAddress: "not-valid:zyx",
+			},
+		})
+		assert.Error(t, err)
+	})
+}
+
+func TestBuilder_BuildBootstrapStatsConfig(t *testing.T) {
+	b := New("local-grpc", "local-http", filemgr.NewManager(), nil)
+	t.Run("valid", func(t *testing.T) {
+		statsCfg, err := b.BuildBootstrapStatsConfig(&config.Config{
+			Options: &config.Options{
+				Services: "all",
+			},
+		})
+		assert.NoError(t, err)
+		testutil.AssertProtoJSONEqual(t, `
+			{
+				"statsTags": [{
+					"tagName": "service",
+					"fixedValue": "pomerium"
+				}]
+			}
+		`, statsCfg)
+	})
+}

config/envoyconfig/envoyconfig.go

@@ -205,3 +205,26 @@ func marshalAny(msg proto.Message) *anypb.Any {
 	})
 	return any
 }
+
+// parseAddress parses a string address into an envoy address.
+func parseAddress(raw string) (*envoy_config_core_v3.Address, error) {
+	if host, portstr, err := net.SplitHostPort(raw); err == nil {
+		if host == "localhost" {
+			host = "127.0.0.1"
+		}
+
+		if port, err := strconv.Atoi(portstr); err == nil {
+			return &envoy_config_core_v3.Address{
+				Address: &envoy_config_core_v3.Address_SocketAddress{
+					SocketAddress: &envoy_config_core_v3.SocketAddress{
+						Address: host,
+						PortSpecifier: &envoy_config_core_v3.SocketAddress_PortValue{
+							PortValue: uint32(port),
+						},
+					},
+				},
+			}, nil
+		}
+	}
+	return nil, fmt.Errorf("unknown address format: %s", raw)
+}

internal/cmd/pomerium/pomerium.go

@@ -83,7 +83,7 @@ func Run(ctx context.Context, configFile string) error {
 	log.Info().Str("port", httpPort).Msg("HTTP server started")
 
 	// create envoy server
-	envoyServer, err := envoy.NewServer(src, grpcPort, httpPort)
+	envoyServer, err := envoy.NewServer(src, grpcPort, httpPort, controlPlane.Builder)
 	if err != nil {
 		return fmt.Errorf("error creating envoy server: %w", err)
 	}

internal/controlplane/server.go

@@ -49,6 +49,7 @@ type Server struct {
 	GRPCServer   *grpc.Server
 	HTTPListener net.Listener
 	HTTPRouter   *mux.Router
+	Builder      *envoyconfig.Builder
 
 	currentConfig atomicVersionedConfig
 	name          string
@@ -56,7 +57,6 @@ type Server struct {
 	filemgr       *filemgr.Manager
 	metricsMgr    *config.MetricsManager
 	reproxy       *reproxy.Handler
-	builder       *envoyconfig.Builder
 }
 
 // NewServer creates a new Server. Listener ports are chosen by the OS.
@@ -99,7 +99,7 @@ func NewServer(name string, metricsMgr *config.MetricsManager) (*Server, error)
 	srv.filemgr = filemgr.NewManager()
 	srv.filemgr.ClearCache()
 
-	srv.builder = envoyconfig.New(
+	srv.Builder = envoyconfig.New(
 		srv.GRPCListener.Addr().String(),
 		srv.HTTPListener.Addr().String(),
 		srv.filemgr,

internal/controlplane/xds.go

@@ -18,7 +18,7 @@ func (srv *Server) buildDiscoveryResources() (map[string][]*envoy_service_discov
 	resources := map[string][]*envoy_service_discovery_v3.Resource{}
 	cfg := srv.currentConfig.Load()
 
-	clusters, err := srv.builder.BuildClusters(cfg.Config)
+	clusters, err := srv.Builder.BuildClusters(cfg.Config)
 	if err != nil {
 		return nil, err
 	}
@@ -31,7 +31,7 @@ func (srv *Server) buildDiscoveryResources() (map[string][]*envoy_service_discov
 		})
 	}
 
-	listeners, err := srv.builder.BuildListeners(cfg.Config)
+	listeners, err := srv.Builder.BuildListeners(cfg.Config)
 	if err != nil {
 		return nil, err
 	}

internal/envoy/envoy.go

@@ -11,7 +11,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -24,21 +23,17 @@ import (
 
 	"github.com/cenkalti/backoff/v4"
 	envoy_config_bootstrap_v3 "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
-	envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
-	envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
-	envoy_config_metrics_v3 "github.com/envoyproxy/go-control-plane/envoy/config/metrics/v3"
 	"github.com/golang/protobuf/proto"
 	"github.com/google/go-cmp/cmp"
 	"github.com/natefinch/atomic"
 	"github.com/rs/zerolog"
 	"go.opencensus.io/stats/view"
 	"google.golang.org/protobuf/encoding/protojson"
-	"google.golang.org/protobuf/types/known/durationpb"
 
 	"github.com/pomerium/pomerium/config"
+	"github.com/pomerium/pomerium/config/envoyconfig"
 	"github.com/pomerium/pomerium/internal/log"
-	"github.com/pomerium/pomerium/internal/telemetry"
 	"github.com/pomerium/pomerium/internal/telemetry/metrics"
 	"github.com/pomerium/pomerium/internal/telemetry/trace"
 )
@@ -62,6 +57,7 @@ type Server struct {
 	wd  string
 	cmd *exec.Cmd
 
+	builder            *envoyconfig.Builder
 	grpcPort, httpPort string
 	envoyPath          string
 	restartEpoch       int
@@ -71,7 +67,7 @@ type Server struct {
 }
 
 // NewServer creates a new server with traffic routed by envoy.
-func NewServer(src config.Source, grpcPort, httpPort string) (*Server, error) {
+func NewServer(src config.Source, grpcPort, httpPort string, builder *envoyconfig.Builder) (*Server, error) {
 	wd := filepath.Join(os.TempDir(), workingDirectoryName)
 	err := os.MkdirAll(wd, embeddedEnvoyPermissions)
 	if err != nil {
@@ -107,6 +103,7 @@ func NewServer(src config.Source, grpcPort, httpPort string) (*Server, error) {
 
 	srv := &Server{
 		wd:        wd,
+		builder:   builder,
 		grpcPort:  grpcPort,
 		httpPort:  httpPort,
 		envoyPath: envoyPath,
@@ -248,15 +245,10 @@ func (srv *Server) buildBootstrapConfig(cfg *config.Config) ([]byte, error) {
 		Cluster: "proxy",
 	}
 
-	adminAddr, err := ParseAddress(cfg.Options.EnvoyAdminAddress)
+	adminCfg, err := srv.builder.BuildBootstrapAdmin(cfg)
 	if err != nil {
 		return nil, err
 	}
-	adminCfg := &envoy_config_bootstrap_v3.Admin{
-		AccessLogPath: cfg.Options.EnvoyAdminAccessLogPath,
-		ProfilePath:   cfg.Options.EnvoyAdminProfilePath,
-		Address:       adminAddr,
-	}
 
 	dynamicCfg := &envoy_config_bootstrap_v3.Bootstrap_DynamicResources{
 		AdsConfig: &envoy_config_core_v3.ApiConfigSource{
@@ -282,136 +274,31 @@ func (srv *Server) buildBootstrapConfig(cfg *config.Config) ([]byte, error) {
 		},
 	}
 
-	controlPlanePort, err := strconv.Atoi(srv.grpcPort)
+	staticCfg, err := srv.builder.BuildBootstrapStaticResources(cfg)
 	if err != nil {
-		return nil, fmt.Errorf("invalid control plane port: %w", err)
+		return nil, err
 	}
 
-	controlPlaneEndpoint := &envoy_config_endpoint_v3.LbEndpoint_Endpoint{
+	statsCfg, err := srv.builder.BuildBootstrapStatsConfig(cfg)
-		Endpoint: &envoy_config_endpoint_v3.Endpoint{
+	if err != nil {
-			Address: &envoy_config_core_v3.Address{
+		return nil, err
-				Address: &envoy_config_core_v3.Address_SocketAddress{
-					SocketAddress: &envoy_config_core_v3.SocketAddress{
-						Address: "127.0.0.1",
-						PortSpecifier: &envoy_config_core_v3.SocketAddress_PortValue{
-							PortValue: uint32(controlPlanePort),
-						},
-					},
-				},
-			},
-		},
 	}
 
-	controlPlaneCluster := &envoy_config_cluster_v3.Cluster{
+	bootstrapCfg := &envoy_config_bootstrap_v3.Bootstrap{
-		Name: "pomerium-control-plane-grpc",
-		ConnectTimeout: &durationpb.Duration{
-			Seconds: 5,
-		},
-		ClusterDiscoveryType: &envoy_config_cluster_v3.Cluster_Type{
-			Type: envoy_config_cluster_v3.Cluster_STATIC,
-		},
-		LbPolicy: envoy_config_cluster_v3.Cluster_ROUND_ROBIN,
-		LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{
-			ClusterName: "pomerium-control-plane-grpc",
-			Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{
-				{
-					LbEndpoints: []*envoy_config_endpoint_v3.LbEndpoint{
-						{
-							HostIdentifier: controlPlaneEndpoint,
-						},
-					},
-				},
-			},
-		},
-		Http2ProtocolOptions: &envoy_config_core_v3.Http2ProtocolOptions{},
-	}
-
-	staticCfg := &envoy_config_bootstrap_v3.Bootstrap_StaticResources{
-		Clusters: []*envoy_config_cluster_v3.Cluster{
-			controlPlaneCluster,
-		},
-	}
-
-	if srv.options.tracingOptions.Provider == trace.DatadogTracingProviderName {
-		addr := &envoy_config_core_v3.SocketAddress{
-			Address: "127.0.0.1",
-			PortSpecifier: &envoy_config_core_v3.SocketAddress_PortValue{
-				PortValue: 8126,
-			},
-		}
-		if srv.options.tracingOptions.DatadogAddress != "" {
-			a, p, err := net.SplitHostPort(srv.options.tracingOptions.DatadogAddress)
-			if err == nil {
-				addr.Address = a
-				if pv, err := strconv.ParseUint(p, 10, 32); err == nil {
-					addr.PortSpecifier = &envoy_config_core_v3.SocketAddress_PortValue{
-						PortValue: uint32(pv),
-					}
-				}
-			}
-		}
-
-		staticCfg.Clusters = append(staticCfg.Clusters, &envoy_config_cluster_v3.Cluster{
-			Name: "datadog-apm",
-			ConnectTimeout: &durationpb.Duration{
-				Seconds: 5,
-			},
-			ClusterDiscoveryType: &envoy_config_cluster_v3.Cluster_Type{
-				Type: envoy_config_cluster_v3.Cluster_STATIC,
-			},
-			LbPolicy: envoy_config_cluster_v3.Cluster_ROUND_ROBIN,
-			LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{
-				ClusterName: "datadog-apm",
-				Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{
-					{
-						LbEndpoints: []*envoy_config_endpoint_v3.LbEndpoint{
-							{
-								HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{
-									Endpoint: &envoy_config_endpoint_v3.Endpoint{
-										Address: &envoy_config_core_v3.Address{
-											Address: &envoy_config_core_v3.Address_SocketAddress{
-												SocketAddress: addr,
-											},
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		})
-	}
-
-	bcfg := &envoy_config_bootstrap_v3.Bootstrap{
 		Node:             nodeCfg,
 		Admin:            adminCfg,
 		DynamicResources: dynamicCfg,
 		StaticResources:  staticCfg,
-		StatsConfig:      srv.buildStatsConfig(),
+		StatsConfig:      statsCfg,
 	}
 
-	jsonBytes, err := protojson.Marshal(proto.MessageV2(bcfg))
+	jsonBytes, err := protojson.Marshal(proto.MessageV2(bootstrapCfg))
 	if err != nil {
 		return nil, err
 	}
 	return jsonBytes, nil
 }
 
-func (srv *Server) buildStatsConfig() *envoy_config_metrics_v3.StatsConfig {
-	cfg := &envoy_config_metrics_v3.StatsConfig{}
-
-	cfg.StatsTags = []*envoy_config_metrics_v3.TagSpecifier{
-		{
-			TagName: "service",
-			TagValue: &envoy_config_metrics_v3.TagSpecifier_FixedValue{
-				FixedValue: telemetry.ServiceName(srv.options.services),
-			},
-		},
-	}
-	return cfg
-}
-
 var fileNameAndNumberRE = regexp.MustCompile(`^(\[[a-zA-Z0-9/-_.]+:[0-9]+])\s(.*)$`)
 
 func (srv *Server) parseLog(line string) (name string, logLevel string, msg string) {

internal/envoy/envoy_test.go

@@ -7,31 +7,8 @@ import (
 	"testing"
 
 	"github.com/rs/zerolog"
-
-	"github.com/pomerium/pomerium/config"
-	"github.com/pomerium/pomerium/internal/testutil"
 )
 
-func Test_buildStatsConfig(t *testing.T) {
-	tests := []struct {
-		name string
-		opts *config.Options
-		want string
-	}{
-		{"all-in-one", &config.Options{Services: config.ServiceAll}, `{"statsTags":[{"tagName":"service","fixedValue":"pomerium"}]}`},
-		{"authorize", &config.Options{Services: config.ServiceAuthorize}, `{"statsTags":[{"tagName":"service","fixedValue":"pomerium-authorize"}]}`},
-		{"proxy", &config.Options{Services: config.ServiceProxy}, `{"statsTags":[{"tagName":"service","fixedValue":"pomerium-proxy"}]}`},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			srv := &Server{options: serverOptions{services: tt.opts.Services}}
-
-			statsCfg := srv.buildStatsConfig()
-			testutil.AssertProtoJSONEqual(t, tt.want, statsCfg)
-		})
-	}
-}
-
 func TestServer_handleLogs(t *testing.T) {
 	logFormatRE := regexp.MustCompile(`^[[]LOG_FORMAT[]](.*?)--(.*?)--(.*?)$`)
 	line := "[LOG_FORMAT]debug--filter--[external/envoy/source/extensions/filters/listener/tls_inspector/tls_inspector.cc:78] tls inspector: new connection accepted"

internal/testutil/testutil.go

@@ -23,10 +23,17 @@ func AssertProtoJSONEqual(t *testing.T, expected string, protoMsg interface{}, m
 			protoMsgs = append(protoMsgs, toProtoJSON(protoMsgVal.Index(i).Interface()))
 		}
 		bs, _ := json.Marshal(protoMsgs)
-		return assert.JSONEq(t, expected, string(bs), msgAndArgs...)
+		return assert.Equal(t, reformatJSON(json.RawMessage(expected)), reformatJSON(bs), msgAndArgs...)
 	}
 
-	return assert.JSONEq(t, expected, string(toProtoJSON(protoMsg)), msgAndArgs...)
+	return assert.Equal(t, reformatJSON(json.RawMessage(expected)), reformatJSON(toProtoJSON(protoMsg)), msgAndArgs...)
+}
+
+func reformatJSON(raw json.RawMessage) string {
+	var obj interface{}
+	_ = json.Unmarshal(raw, &obj)
+	bs, _ := json.MarshalIndent(obj, "", "  ")
+	return string(bs)
 }
 
 func toProtoJSON(protoMsg interface{}) json.RawMessage {