3pp-mirror/pomerium
1
0
Fork 0
mirror of https://github.com/pomerium/pomerium.git synced 2025-08-04 01:09:36 +02:00
Code Issues Projects Releases Packages Wiki Activity

envoyconfig: move most bootstrap config to shared package (#2088)

Browse source
This commit is contained in:
Caleb Doxsey 2021-04-14 12:07:49 -06:00 • committed by GitHub
parent c12c0aab49
commit f760cdece5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 314 additions and 156 deletions
Download patch file Download diff file Expand all files Collapse all files

2
internal/cmd/pomerium/pomerium.go
View file

@ -83,7 +83,7 @@ func Run(ctx context.Context, configFile string) error {
log.Info().Str("port", httpPort).Msg("HTTP server started")
// create envoy server
envoyServer, err := envoy.NewServer(src, grpcPort, httpPort)
envoyServer, err := envoy.NewServer(src, grpcPort, httpPort, controlPlane.Builder)
if err != nil {
return fmt.Errorf("error creating envoy server: %w", err)
}

4
internal/controlplane/server.go
View file

@ -49,6 +49,7 @@ type Server struct {
GRPCServer *grpc.Server
HTTPListener net.Listener
HTTPRouter *mux.Router
Builder *envoyconfig.Builder
currentConfig atomicVersionedConfig
name string
@ -56,7 +57,6 @@ type Server struct {
filemgr *filemgr.Manager
metricsMgr *config.MetricsManager
reproxy *reproxy.Handler
builder *envoyconfig.Builder
}
// NewServer creates a new Server. Listener ports are chosen by the OS.
@ -99,7 +99,7 @@ func NewServer(name string, metricsMgr *config.MetricsManager) (*Server, error)
srv.filemgr = filemgr.NewManager()
srv.filemgr.ClearCache()
srv.builder = envoyconfig.New(
srv.Builder = envoyconfig.New(
srv.GRPCListener.Addr().String(),
srv.HTTPListener.Addr().String(),
srv.filemgr,

4
internal/controlplane/xds.go
View file

@ -18,7 +18,7 @@ func (srv *Server) buildDiscoveryResources() (map[string][]*envoy_service_discov
resources := map[string][]*envoy_service_discovery_v3.Resource{}
cfg := srv.currentConfig.Load()
clusters, err := srv.builder.BuildClusters(cfg.Config)
clusters, err := srv.Builder.BuildClusters(cfg.Config)
if err != nil {
return nil, err
}
@ -31,7 +31,7 @@ func (srv *Server) buildDiscoveryResources() (map[string][]*envoy_service_discov
})
}
listeners, err := srv.builder.BuildListeners(cfg.Config)
listeners, err := srv.Builder.BuildListeners(cfg.Config)
if err != nil {
return nil, err
}

139
internal/envoy/envoy.go
View file

@ -11,7 +11,6 @@ import (
"fmt"
"io"
"io/ioutil"
"net"
"os"
"os/exec"
"path/filepath"
@ -24,21 +23,17 @@ import (
"github.com/cenkalti/backoff/v4"
envoy_config_bootstrap_v3 "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
envoy_config_metrics_v3 "github.com/envoyproxy/go-control-plane/envoy/config/metrics/v3"
"github.com/golang/protobuf/proto"
"github.com/google/go-cmp/cmp"
"github.com/natefinch/atomic"
"github.com/rs/zerolog"
"go.opencensus.io/stats/view"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/types/known/durationpb"
"github.com/pomerium/pomerium/config"
"github.com/pomerium/pomerium/config/envoyconfig"
"github.com/pomerium/pomerium/internal/log"
"github.com/pomerium/pomerium/internal/telemetry"
"github.com/pomerium/pomerium/internal/telemetry/metrics"
"github.com/pomerium/pomerium/internal/telemetry/trace"
)
@ -62,6 +57,7 @@ type Server struct {
wd string
cmd *exec.Cmd
builder *envoyconfig.Builder
grpcPort, httpPort string
envoyPath string
restartEpoch int
@ -71,7 +67,7 @@ type Server struct {
}
// NewServer creates a new server with traffic routed by envoy.
func NewServer(src config.Source, grpcPort, httpPort string) (*Server, error) {
func NewServer(src config.Source, grpcPort, httpPort string, builder *envoyconfig.Builder) (*Server, error) {
wd := filepath.Join(os.TempDir(), workingDirectoryName)
err := os.MkdirAll(wd, embeddedEnvoyPermissions)
if err != nil {
@ -107,6 +103,7 @@ func NewServer(src config.Source, grpcPort, httpPort string) (*Server, error) {
srv := &Server{
wd: wd,
builder: builder,
grpcPort: grpcPort,
httpPort: httpPort,
envoyPath: envoyPath,
@ -248,15 +245,10 @@ func (srv *Server) buildBootstrapConfig(cfg *config.Config) ([]byte, error) {
Cluster: "proxy",
}
adminAddr, err := ParseAddress(cfg.Options.EnvoyAdminAddress)
adminCfg, err := srv.builder.BuildBootstrapAdmin(cfg)
if err != nil {
return nil, err
}
adminCfg := &envoy_config_bootstrap_v3.Admin{
AccessLogPath: cfg.Options.EnvoyAdminAccessLogPath,
ProfilePath: cfg.Options.EnvoyAdminProfilePath,
Address: adminAddr,
}
dynamicCfg := &envoy_config_bootstrap_v3.Bootstrap_DynamicResources{
AdsConfig: &envoy_config_core_v3.ApiConfigSource{
@ -282,136 +274,31 @@ func (srv *Server) buildBootstrapConfig(cfg *config.Config) ([]byte, error) {
},
}
controlPlanePort, err := strconv.Atoi(srv.grpcPort)
staticCfg, err := srv.builder.BuildBootstrapStaticResources(cfg)
if err != nil {
return nil, fmt.Errorf("invalid control plane port: %w", err)
return nil, err
}
controlPlaneEndpoint := &envoy_config_endpoint_v3.LbEndpoint_Endpoint{
Endpoint: &envoy_config_endpoint_v3.Endpoint{
Address: &envoy_config_core_v3.Address{
Address: &envoy_config_core_v3.Address_SocketAddress{
SocketAddress: &envoy_config_core_v3.SocketAddress{
Address: "127.0.0.1",
PortSpecifier: &envoy_config_core_v3.SocketAddress_PortValue{
PortValue: uint32(controlPlanePort),
},
},
},
},
},
statsCfg, err := srv.builder.BuildBootstrapStatsConfig(cfg)
if err != nil {
return nil, err
}
controlPlaneCluster := &envoy_config_cluster_v3.Cluster{
Name: "pomerium-control-plane-grpc",
ConnectTimeout: &durationpb.Duration{
Seconds: 5,
},
ClusterDiscoveryType: &envoy_config_cluster_v3.Cluster_Type{
Type: envoy_config_cluster_v3.Cluster_STATIC,
},
LbPolicy: envoy_config_cluster_v3.Cluster_ROUND_ROBIN,
LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{
ClusterName: "pomerium-control-plane-grpc",
Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{
{
LbEndpoints: []*envoy_config_endpoint_v3.LbEndpoint{
{
HostIdentifier: controlPlaneEndpoint,
},
},
},
},
},
Http2ProtocolOptions: &envoy_config_core_v3.Http2ProtocolOptions{},
}
staticCfg := &envoy_config_bootstrap_v3.Bootstrap_StaticResources{
Clusters: []*envoy_config_cluster_v3.Cluster{
controlPlaneCluster,
},
}
if srv.options.tracingOptions.Provider == trace.DatadogTracingProviderName {
addr := &envoy_config_core_v3.SocketAddress{
Address: "127.0.0.1",
PortSpecifier: &envoy_config_core_v3.SocketAddress_PortValue{
PortValue: 8126,
},
}
if srv.options.tracingOptions.DatadogAddress != "" {
a, p, err := net.SplitHostPort(srv.options.tracingOptions.DatadogAddress)
if err == nil {
addr.Address = a
if pv, err := strconv.ParseUint(p, 10, 32); err == nil {
addr.PortSpecifier = &envoy_config_core_v3.SocketAddress_PortValue{
PortValue: uint32(pv),
}
}
}
}
staticCfg.Clusters = append(staticCfg.Clusters, &envoy_config_cluster_v3.Cluster{
Name: "datadog-apm",
ConnectTimeout: &durationpb.Duration{
Seconds: 5,
},
ClusterDiscoveryType: &envoy_config_cluster_v3.Cluster_Type{
Type: envoy_config_cluster_v3.Cluster_STATIC,
},
LbPolicy: envoy_config_cluster_v3.Cluster_ROUND_ROBIN,
LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{
ClusterName: "datadog-apm",
Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{
{
LbEndpoints: []*envoy_config_endpoint_v3.LbEndpoint{
{
HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{
Endpoint: &envoy_config_endpoint_v3.Endpoint{
Address: &envoy_config_core_v3.Address{
Address: &envoy_config_core_v3.Address_SocketAddress{
SocketAddress: addr,
},
},
},
},
},
},
},
},
},
})
}
bcfg := &envoy_config_bootstrap_v3.Bootstrap{
bootstrapCfg := &envoy_config_bootstrap_v3.Bootstrap{
Node: nodeCfg,
Admin: adminCfg,
DynamicResources: dynamicCfg,
StaticResources: staticCfg,
StatsConfig: srv.buildStatsConfig(),
StatsConfig: statsCfg,
}
jsonBytes, err := protojson.Marshal(proto.MessageV2(bcfg))
jsonBytes, err := protojson.Marshal(proto.MessageV2(bootstrapCfg))
if err != nil {
return nil, err
}
return jsonBytes, nil
}
func (srv *Server) buildStatsConfig() *envoy_config_metrics_v3.StatsConfig {
cfg := &envoy_config_metrics_v3.StatsConfig{}
cfg.StatsTags = []*envoy_config_metrics_v3.TagSpecifier{
{
TagName: "service",
TagValue: &envoy_config_metrics_v3.TagSpecifier_FixedValue{
FixedValue: telemetry.ServiceName(srv.options.services),
},
},
}
return cfg
}
var fileNameAndNumberRE = regexp.MustCompile(`^(\[[a-zA-Z0-9/-_.]+:[0-9]+])\s(.*)$`)
func (srv *Server) parseLog(line string) (name string, logLevel string, msg string) {

23
internal/envoy/envoy_test.go
View file

@ -7,31 +7,8 @@ import (
"testing"
"github.com/rs/zerolog"
"github.com/pomerium/pomerium/config"
"github.com/pomerium/pomerium/internal/testutil"
)
func Test_buildStatsConfig(t *testing.T) {
tests := []struct {
name string
opts *config.Options
want string
}{
{"all-in-one", &config.Options{Services: config.ServiceAll}, `{"statsTags":[{"tagName":"service","fixedValue":"pomerium"}]}`},
{"authorize", &config.Options{Services: config.ServiceAuthorize}, `{"statsTags":[{"tagName":"service","fixedValue":"pomerium-authorize"}]}`},
{"proxy", &config.Options{Services: config.ServiceProxy}, `{"statsTags":[{"tagName":"service","fixedValue":"pomerium-proxy"}]}`},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
srv := &Server{options: serverOptions{services: tt.opts.Services}}
statsCfg := srv.buildStatsConfig()
testutil.AssertProtoJSONEqual(t, tt.want, statsCfg)
})
}
}
func TestServer_handleLogs(t *testing.T) {
logFormatRE := regexp.MustCompile(`^[[]LOG_FORMAT[]](.*?)--(.*?)--(.*?)$`)
line := "[LOG_FORMAT]debug--filter--[external/envoy/source/extensions/filters/listener/tls_inspector/tls_inspector.cc:78] tls inspector: new connection accepted"

11
internal/testutil/testutil.go
View file

@ -23,10 +23,17 @@ func AssertProtoJSONEqual(t *testing.T, expected string, protoMsg interface{}, m
protoMsgs = append(protoMsgs, toProtoJSON(protoMsgVal.Index(i).Interface()))
}
bs, _ := json.Marshal(protoMsgs)
return assert.JSONEq(t, expected, string(bs), msgAndArgs...)
return assert.Equal(t, reformatJSON(json.RawMessage(expected)), reformatJSON(bs), msgAndArgs...)
}
return assert.JSONEq(t, expected, string(toProtoJSON(protoMsg)), msgAndArgs...)
return assert.Equal(t, reformatJSON(json.RawMessage(expected)), reformatJSON(toProtoJSON(protoMsg)), msgAndArgs...)
}
func reformatJSON(raw json.RawMessage) string {
var obj interface{}
_ = json.Unmarshal(raw, &obj)
bs, _ := json.MarshalIndent(obj, "", " ")
return string(bs)
}
func toProtoJSON(protoMsg interface{}) json.RawMessage {