diff --git a/config/envoyconfig/bootstrap.go b/config/envoyconfig/bootstrap.go new file mode 100644 index 000000000..328232bc5 --- /dev/null +++ b/config/envoyconfig/bootstrap.go @@ -0,0 +1,126 @@ +package envoyconfig + +import ( + "fmt" + + envoy_config_bootstrap_v3 "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3" + envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + envoy_config_metrics_v3 "github.com/envoyproxy/go-control-plane/envoy/config/metrics/v3" + "google.golang.org/protobuf/types/known/durationpb" + + "github.com/pomerium/pomerium/config" + "github.com/pomerium/pomerium/internal/telemetry" + "github.com/pomerium/pomerium/internal/telemetry/trace" +) + +// BuildBootstrapAdmin builds the admin config for the envoy bootstrap. +func (b *Builder) BuildBootstrapAdmin(cfg *config.Config) (*envoy_config_bootstrap_v3.Admin, error) { + adminAddr, err := parseAddress(cfg.Options.EnvoyAdminAddress) + if err != nil { + return nil, fmt.Errorf("envoyconfig: invalid envoy admin address: %w", err) + } + return &envoy_config_bootstrap_v3.Admin{ + AccessLogPath: cfg.Options.EnvoyAdminAccessLogPath, + ProfilePath: cfg.Options.EnvoyAdminProfilePath, + Address: adminAddr, + }, nil +} + +// BuildBootstrapStaticResources builds the static resources for the envoy bootstrap. It includes the control plane +// cluster as well as a datadog-apm cluster (if datadog is used). +func (b *Builder) BuildBootstrapStaticResources(cfg *config.Config) (*envoy_config_bootstrap_v3.Bootstrap_StaticResources, error) { + grpcAddr, err := parseAddress(b.localGRPCAddress) + if err != nil { + return nil, fmt.Errorf("envoyconfig: invalid local gRPC address: %w", err) + } + + controlPlaneEndpoint := &envoy_config_endpoint_v3.LbEndpoint_Endpoint{ + Endpoint: &envoy_config_endpoint_v3.Endpoint{ + Address: grpcAddr, + }, + } + + controlPlaneCluster := &envoy_config_cluster_v3.Cluster{ + Name: "pomerium-control-plane-grpc", + ConnectTimeout: &durationpb.Duration{ + Seconds: 5, + }, + ClusterDiscoveryType: &envoy_config_cluster_v3.Cluster_Type{ + Type: envoy_config_cluster_v3.Cluster_STATIC, + }, + LbPolicy: envoy_config_cluster_v3.Cluster_ROUND_ROBIN, + LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{ + ClusterName: "pomerium-control-plane-grpc", + Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{ + { + LbEndpoints: []*envoy_config_endpoint_v3.LbEndpoint{ + { + HostIdentifier: controlPlaneEndpoint, + }, + }, + }, + }, + }, + Http2ProtocolOptions: &envoy_config_core_v3.Http2ProtocolOptions{}, + } + + staticCfg := &envoy_config_bootstrap_v3.Bootstrap_StaticResources{ + Clusters: []*envoy_config_cluster_v3.Cluster{ + controlPlaneCluster, + }, + } + + if cfg.Options.TracingProvider == trace.DatadogTracingProviderName { + addr, _ := parseAddress("127.0.0.1:8126") + + if cfg.Options.TracingDatadogAddress != "" { + addr, err = parseAddress(cfg.Options.TracingDatadogAddress) + if err != nil { + return nil, fmt.Errorf("envoyconfig: invalid tracing datadog address: %w", err) + } + } + + staticCfg.Clusters = append(staticCfg.Clusters, &envoy_config_cluster_v3.Cluster{ + Name: "datadog-apm", + ConnectTimeout: &durationpb.Duration{ + Seconds: 5, + }, + ClusterDiscoveryType: &envoy_config_cluster_v3.Cluster_Type{ + Type: envoy_config_cluster_v3.Cluster_STATIC, + }, + LbPolicy: envoy_config_cluster_v3.Cluster_ROUND_ROBIN, + LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{ + ClusterName: "datadog-apm", + Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{ + { + LbEndpoints: []*envoy_config_endpoint_v3.LbEndpoint{ + { + HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{ + Endpoint: &envoy_config_endpoint_v3.Endpoint{ + Address: addr, + }, + }, + }, + }, + }, + }, + }, + }) + } + + return staticCfg, nil +} + +// BuildBootstrapStatsConfig builds a the stats config the envoy bootstrap. +func (b *Builder) BuildBootstrapStatsConfig(cfg *config.Config) (*envoy_config_metrics_v3.StatsConfig, error) { + statsCfg := &envoy_config_metrics_v3.StatsConfig{} + statsCfg.StatsTags = []*envoy_config_metrics_v3.TagSpecifier{{ + TagName: "service", + TagValue: &envoy_config_metrics_v3.TagSpecifier_FixedValue{ + FixedValue: telemetry.ServiceName(cfg.Options.Services), + }, + }} + return statsCfg, nil +} diff --git a/config/envoyconfig/bootstrap_test.go b/config/envoyconfig/bootstrap_test.go new file mode 100644 index 000000000..97282f797 --- /dev/null +++ b/config/envoyconfig/bootstrap_test.go @@ -0,0 +1,138 @@ +package envoyconfig + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/pomerium/pomerium/config" + "github.com/pomerium/pomerium/config/envoyconfig/filemgr" + "github.com/pomerium/pomerium/internal/telemetry/trace" + "github.com/pomerium/pomerium/internal/testutil" +) + +func TestBuilder_BuildBootstrapAdmin(t *testing.T) { + b := New("local-grpc", "local-http", filemgr.NewManager(), nil) + t.Run("valid", func(t *testing.T) { + adminCfg, err := b.BuildBootstrapAdmin(&config.Config{ + Options: &config.Options{ + EnvoyAdminAddress: "localhost:9901", + }, + }) + assert.NoError(t, err) + testutil.AssertProtoJSONEqual(t, ` + { + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 9901 + } + } + } + `, adminCfg) + }) + t.Run("bad address", func(t *testing.T) { + _, err := b.BuildBootstrapAdmin(&config.Config{ + Options: &config.Options{ + EnvoyAdminAddress: "xyz1234:zyx4321", + }, + }) + assert.Error(t, err) + }) +} + +func TestBuilder_BuildBootstrapStaticResources(t *testing.T) { + t.Run("valid", func(t *testing.T) { + b := New("localhost:1111", "localhost:2222", filemgr.NewManager(), nil) + staticCfg, err := b.BuildBootstrapStaticResources(&config.Config{ + Options: &config.Options{ + TracingProvider: trace.DatadogTracingProviderName, + }, + }) + assert.NoError(t, err) + testutil.AssertProtoJSONEqual(t, ` + { + "clusters": [ + { + "name": "pomerium-control-plane-grpc", + "type": "STATIC", + "connectTimeout": "5s", + "http2ProtocolOptions": {}, + "loadAssignment": { + "clusterName": "pomerium-control-plane-grpc", + "endpoints": [{ + "lbEndpoints": [{ + "endpoint": { + "address": { + "socketAddress":{ + "address": "127.0.0.1", + "portValue": 1111 + } + } + } + }] + }] + } + }, + { + "name": "datadog-apm", + "type": "STATIC", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "datadog-apm", + "endpoints": [{ + "lbEndpoints": [{ + "endpoint": { + "address": { + "socketAddress":{ + "address": "127.0.0.1", + "portValue": 8126 + } + } + } + }] + }] + } + } + ] + } + `, staticCfg) + }) + t.Run("bad gRPC address", func(t *testing.T) { + b := New("xyz:zyx", "localhost:2222", filemgr.NewManager(), nil) + _, err := b.BuildBootstrapStaticResources(&config.Config{ + Options: &config.Options{}, + }) + assert.Error(t, err) + }) + t.Run("bad datadog address", func(t *testing.T) { + b := New("localhost:1111", "localhost:2222", filemgr.NewManager(), nil) + _, err := b.BuildBootstrapStaticResources(&config.Config{ + Options: &config.Options{ + TracingProvider: trace.DatadogTracingProviderName, + TracingDatadogAddress: "not-valid:zyx", + }, + }) + assert.Error(t, err) + }) +} + +func TestBuilder_BuildBootstrapStatsConfig(t *testing.T) { + b := New("local-grpc", "local-http", filemgr.NewManager(), nil) + t.Run("valid", func(t *testing.T) { + statsCfg, err := b.BuildBootstrapStatsConfig(&config.Config{ + Options: &config.Options{ + Services: "all", + }, + }) + assert.NoError(t, err) + testutil.AssertProtoJSONEqual(t, ` + { + "statsTags": [{ + "tagName": "service", + "fixedValue": "pomerium" + }] + } + `, statsCfg) + }) +} diff --git a/config/envoyconfig/envoyconfig.go b/config/envoyconfig/envoyconfig.go index ed355d9a7..31e117829 100644 --- a/config/envoyconfig/envoyconfig.go +++ b/config/envoyconfig/envoyconfig.go @@ -205,3 +205,26 @@ func marshalAny(msg proto.Message) *anypb.Any { }) return any } + +// parseAddress parses a string address into an envoy address. +func parseAddress(raw string) (*envoy_config_core_v3.Address, error) { + if host, portstr, err := net.SplitHostPort(raw); err == nil { + if host == "localhost" { + host = "127.0.0.1" + } + + if port, err := strconv.Atoi(portstr); err == nil { + return &envoy_config_core_v3.Address{ + Address: &envoy_config_core_v3.Address_SocketAddress{ + SocketAddress: &envoy_config_core_v3.SocketAddress{ + Address: host, + PortSpecifier: &envoy_config_core_v3.SocketAddress_PortValue{ + PortValue: uint32(port), + }, + }, + }, + }, nil + } + } + return nil, fmt.Errorf("unknown address format: %s", raw) +} diff --git a/internal/cmd/pomerium/pomerium.go b/internal/cmd/pomerium/pomerium.go index c3c7f2a44..d954350d9 100644 --- a/internal/cmd/pomerium/pomerium.go +++ b/internal/cmd/pomerium/pomerium.go @@ -83,7 +83,7 @@ func Run(ctx context.Context, configFile string) error { log.Info().Str("port", httpPort).Msg("HTTP server started") // create envoy server - envoyServer, err := envoy.NewServer(src, grpcPort, httpPort) + envoyServer, err := envoy.NewServer(src, grpcPort, httpPort, controlPlane.Builder) if err != nil { return fmt.Errorf("error creating envoy server: %w", err) } diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go index 1fd242012..fea2ace1e 100644 --- a/internal/controlplane/server.go +++ b/internal/controlplane/server.go @@ -49,6 +49,7 @@ type Server struct { GRPCServer *grpc.Server HTTPListener net.Listener HTTPRouter *mux.Router + Builder *envoyconfig.Builder currentConfig atomicVersionedConfig name string @@ -56,7 +57,6 @@ type Server struct { filemgr *filemgr.Manager metricsMgr *config.MetricsManager reproxy *reproxy.Handler - builder *envoyconfig.Builder } // NewServer creates a new Server. Listener ports are chosen by the OS. @@ -99,7 +99,7 @@ func NewServer(name string, metricsMgr *config.MetricsManager) (*Server, error) srv.filemgr = filemgr.NewManager() srv.filemgr.ClearCache() - srv.builder = envoyconfig.New( + srv.Builder = envoyconfig.New( srv.GRPCListener.Addr().String(), srv.HTTPListener.Addr().String(), srv.filemgr, diff --git a/internal/controlplane/xds.go b/internal/controlplane/xds.go index 4029fcda3..676165829 100644 --- a/internal/controlplane/xds.go +++ b/internal/controlplane/xds.go @@ -18,7 +18,7 @@ func (srv *Server) buildDiscoveryResources() (map[string][]*envoy_service_discov resources := map[string][]*envoy_service_discovery_v3.Resource{} cfg := srv.currentConfig.Load() - clusters, err := srv.builder.BuildClusters(cfg.Config) + clusters, err := srv.Builder.BuildClusters(cfg.Config) if err != nil { return nil, err } @@ -31,7 +31,7 @@ func (srv *Server) buildDiscoveryResources() (map[string][]*envoy_service_discov }) } - listeners, err := srv.builder.BuildListeners(cfg.Config) + listeners, err := srv.Builder.BuildListeners(cfg.Config) if err != nil { return nil, err } diff --git a/internal/envoy/envoy.go b/internal/envoy/envoy.go index 4a3a64a76..3a8585096 100644 --- a/internal/envoy/envoy.go +++ b/internal/envoy/envoy.go @@ -11,7 +11,6 @@ import ( "fmt" "io" "io/ioutil" - "net" "os" "os/exec" "path/filepath" @@ -24,21 +23,17 @@ import ( "github.com/cenkalti/backoff/v4" envoy_config_bootstrap_v3 "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3" - envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" - envoy_config_metrics_v3 "github.com/envoyproxy/go-control-plane/envoy/config/metrics/v3" "github.com/golang/protobuf/proto" "github.com/google/go-cmp/cmp" "github.com/natefinch/atomic" "github.com/rs/zerolog" "go.opencensus.io/stats/view" "google.golang.org/protobuf/encoding/protojson" - "google.golang.org/protobuf/types/known/durationpb" "github.com/pomerium/pomerium/config" + "github.com/pomerium/pomerium/config/envoyconfig" "github.com/pomerium/pomerium/internal/log" - "github.com/pomerium/pomerium/internal/telemetry" "github.com/pomerium/pomerium/internal/telemetry/metrics" "github.com/pomerium/pomerium/internal/telemetry/trace" ) @@ -62,6 +57,7 @@ type Server struct { wd string cmd *exec.Cmd + builder *envoyconfig.Builder grpcPort, httpPort string envoyPath string restartEpoch int @@ -71,7 +67,7 @@ type Server struct { } // NewServer creates a new server with traffic routed by envoy. -func NewServer(src config.Source, grpcPort, httpPort string) (*Server, error) { +func NewServer(src config.Source, grpcPort, httpPort string, builder *envoyconfig.Builder) (*Server, error) { wd := filepath.Join(os.TempDir(), workingDirectoryName) err := os.MkdirAll(wd, embeddedEnvoyPermissions) if err != nil { @@ -107,6 +103,7 @@ func NewServer(src config.Source, grpcPort, httpPort string) (*Server, error) { srv := &Server{ wd: wd, + builder: builder, grpcPort: grpcPort, httpPort: httpPort, envoyPath: envoyPath, @@ -248,15 +245,10 @@ func (srv *Server) buildBootstrapConfig(cfg *config.Config) ([]byte, error) { Cluster: "proxy", } - adminAddr, err := ParseAddress(cfg.Options.EnvoyAdminAddress) + adminCfg, err := srv.builder.BuildBootstrapAdmin(cfg) if err != nil { return nil, err } - adminCfg := &envoy_config_bootstrap_v3.Admin{ - AccessLogPath: cfg.Options.EnvoyAdminAccessLogPath, - ProfilePath: cfg.Options.EnvoyAdminProfilePath, - Address: adminAddr, - } dynamicCfg := &envoy_config_bootstrap_v3.Bootstrap_DynamicResources{ AdsConfig: &envoy_config_core_v3.ApiConfigSource{ @@ -282,136 +274,31 @@ func (srv *Server) buildBootstrapConfig(cfg *config.Config) ([]byte, error) { }, } - controlPlanePort, err := strconv.Atoi(srv.grpcPort) + staticCfg, err := srv.builder.BuildBootstrapStaticResources(cfg) if err != nil { - return nil, fmt.Errorf("invalid control plane port: %w", err) + return nil, err } - controlPlaneEndpoint := &envoy_config_endpoint_v3.LbEndpoint_Endpoint{ - Endpoint: &envoy_config_endpoint_v3.Endpoint{ - Address: &envoy_config_core_v3.Address{ - Address: &envoy_config_core_v3.Address_SocketAddress{ - SocketAddress: &envoy_config_core_v3.SocketAddress{ - Address: "127.0.0.1", - PortSpecifier: &envoy_config_core_v3.SocketAddress_PortValue{ - PortValue: uint32(controlPlanePort), - }, - }, - }, - }, - }, + statsCfg, err := srv.builder.BuildBootstrapStatsConfig(cfg) + if err != nil { + return nil, err } - controlPlaneCluster := &envoy_config_cluster_v3.Cluster{ - Name: "pomerium-control-plane-grpc", - ConnectTimeout: &durationpb.Duration{ - Seconds: 5, - }, - ClusterDiscoveryType: &envoy_config_cluster_v3.Cluster_Type{ - Type: envoy_config_cluster_v3.Cluster_STATIC, - }, - LbPolicy: envoy_config_cluster_v3.Cluster_ROUND_ROBIN, - LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{ - ClusterName: "pomerium-control-plane-grpc", - Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{ - { - LbEndpoints: []*envoy_config_endpoint_v3.LbEndpoint{ - { - HostIdentifier: controlPlaneEndpoint, - }, - }, - }, - }, - }, - Http2ProtocolOptions: &envoy_config_core_v3.Http2ProtocolOptions{}, - } - - staticCfg := &envoy_config_bootstrap_v3.Bootstrap_StaticResources{ - Clusters: []*envoy_config_cluster_v3.Cluster{ - controlPlaneCluster, - }, - } - - if srv.options.tracingOptions.Provider == trace.DatadogTracingProviderName { - addr := &envoy_config_core_v3.SocketAddress{ - Address: "127.0.0.1", - PortSpecifier: &envoy_config_core_v3.SocketAddress_PortValue{ - PortValue: 8126, - }, - } - if srv.options.tracingOptions.DatadogAddress != "" { - a, p, err := net.SplitHostPort(srv.options.tracingOptions.DatadogAddress) - if err == nil { - addr.Address = a - if pv, err := strconv.ParseUint(p, 10, 32); err == nil { - addr.PortSpecifier = &envoy_config_core_v3.SocketAddress_PortValue{ - PortValue: uint32(pv), - } - } - } - } - - staticCfg.Clusters = append(staticCfg.Clusters, &envoy_config_cluster_v3.Cluster{ - Name: "datadog-apm", - ConnectTimeout: &durationpb.Duration{ - Seconds: 5, - }, - ClusterDiscoveryType: &envoy_config_cluster_v3.Cluster_Type{ - Type: envoy_config_cluster_v3.Cluster_STATIC, - }, - LbPolicy: envoy_config_cluster_v3.Cluster_ROUND_ROBIN, - LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{ - ClusterName: "datadog-apm", - Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{ - { - LbEndpoints: []*envoy_config_endpoint_v3.LbEndpoint{ - { - HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{ - Endpoint: &envoy_config_endpoint_v3.Endpoint{ - Address: &envoy_config_core_v3.Address{ - Address: &envoy_config_core_v3.Address_SocketAddress{ - SocketAddress: addr, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }) - } - - bcfg := &envoy_config_bootstrap_v3.Bootstrap{ + bootstrapCfg := &envoy_config_bootstrap_v3.Bootstrap{ Node: nodeCfg, Admin: adminCfg, DynamicResources: dynamicCfg, StaticResources: staticCfg, - StatsConfig: srv.buildStatsConfig(), + StatsConfig: statsCfg, } - jsonBytes, err := protojson.Marshal(proto.MessageV2(bcfg)) + jsonBytes, err := protojson.Marshal(proto.MessageV2(bootstrapCfg)) if err != nil { return nil, err } return jsonBytes, nil } -func (srv *Server) buildStatsConfig() *envoy_config_metrics_v3.StatsConfig { - cfg := &envoy_config_metrics_v3.StatsConfig{} - - cfg.StatsTags = []*envoy_config_metrics_v3.TagSpecifier{ - { - TagName: "service", - TagValue: &envoy_config_metrics_v3.TagSpecifier_FixedValue{ - FixedValue: telemetry.ServiceName(srv.options.services), - }, - }, - } - return cfg -} - var fileNameAndNumberRE = regexp.MustCompile(`^(\[[a-zA-Z0-9/-_.]+:[0-9]+])\s(.*)$`) func (srv *Server) parseLog(line string) (name string, logLevel string, msg string) { diff --git a/internal/envoy/envoy_test.go b/internal/envoy/envoy_test.go index 7e181ae69..01acf4abe 100644 --- a/internal/envoy/envoy_test.go +++ b/internal/envoy/envoy_test.go @@ -7,31 +7,8 @@ import ( "testing" "github.com/rs/zerolog" - - "github.com/pomerium/pomerium/config" - "github.com/pomerium/pomerium/internal/testutil" ) -func Test_buildStatsConfig(t *testing.T) { - tests := []struct { - name string - opts *config.Options - want string - }{ - {"all-in-one", &config.Options{Services: config.ServiceAll}, `{"statsTags":[{"tagName":"service","fixedValue":"pomerium"}]}`}, - {"authorize", &config.Options{Services: config.ServiceAuthorize}, `{"statsTags":[{"tagName":"service","fixedValue":"pomerium-authorize"}]}`}, - {"proxy", &config.Options{Services: config.ServiceProxy}, `{"statsTags":[{"tagName":"service","fixedValue":"pomerium-proxy"}]}`}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - srv := &Server{options: serverOptions{services: tt.opts.Services}} - - statsCfg := srv.buildStatsConfig() - testutil.AssertProtoJSONEqual(t, tt.want, statsCfg) - }) - } -} - func TestServer_handleLogs(t *testing.T) { logFormatRE := regexp.MustCompile(`^[[]LOG_FORMAT[]](.*?)--(.*?)--(.*?)$`) line := "[LOG_FORMAT]debug--filter--[external/envoy/source/extensions/filters/listener/tls_inspector/tls_inspector.cc:78] tls inspector: new connection accepted" diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index 06b31a750..90652b387 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -23,10 +23,17 @@ func AssertProtoJSONEqual(t *testing.T, expected string, protoMsg interface{}, m protoMsgs = append(protoMsgs, toProtoJSON(protoMsgVal.Index(i).Interface())) } bs, _ := json.Marshal(protoMsgs) - return assert.JSONEq(t, expected, string(bs), msgAndArgs...) + return assert.Equal(t, reformatJSON(json.RawMessage(expected)), reformatJSON(bs), msgAndArgs...) } - return assert.JSONEq(t, expected, string(toProtoJSON(protoMsg)), msgAndArgs...) + return assert.Equal(t, reformatJSON(json.RawMessage(expected)), reformatJSON(toProtoJSON(protoMsg)), msgAndArgs...) +} + +func reformatJSON(raw json.RawMessage) string { + var obj interface{} + _ = json.Unmarshal(raw, &obj) + bs, _ := json.MarshalIndent(obj, "", " ") + return string(bs) } func toProtoJSON(protoMsg interface{}) json.RawMessage {