authenticate: fix debug and metrics endpoints (#3212)

This commit is contained in:
Caleb Doxsey 2022-03-30 09:37:37 -06:00 committed by GitHub
parent b83bb8f2f7
commit b435f73e2b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 167 additions and 128 deletions

View file

@ -18,6 +18,10 @@ type Config struct {
HTTPPort string HTTPPort string
// OutboundPort is the port the outbound gRPC listener is running on. // OutboundPort is the port the outbound gRPC listener is running on.
OutboundPort string OutboundPort string
// MetricsPort is the port the metrics listener is running on.
MetricsPort string
// DebugPort is the port the debug listener is running on.
DebugPort string
} }
// Clone creates a clone of the config. // Clone creates a clone of the config.
@ -32,6 +36,8 @@ func (cfg *Config) Clone() *Config {
GRPCPort: cfg.GRPCPort, GRPCPort: cfg.GRPCPort,
HTTPPort: cfg.HTTPPort, HTTPPort: cfg.HTTPPort,
OutboundPort: cfg.OutboundPort, OutboundPort: cfg.OutboundPort,
MetricsPort: cfg.MetricsPort,
DebugPort: cfg.DebugPort,
} }
} }

View file

@ -110,13 +110,15 @@ func NewFileOrEnvironmentSource(
return nil, err return nil, err
} }
ports, err := netutil.AllocatePorts(3) ports, err := netutil.AllocatePorts(5)
if err != nil { if err != nil {
return nil, err return nil, err
} }
grpcPort := ports[0] grpcPort := ports[0]
httpPort := ports[1] httpPort := ports[1]
outboundPort := ports[2] outboundPort := ports[2]
metricsPort := ports[3]
debugPort := ports[4]
cfg := &Config{ cfg := &Config{
Options: options, Options: options,
@ -125,6 +127,8 @@ func NewFileOrEnvironmentSource(
GRPCPort: grpcPort, GRPCPort: grpcPort,
HTTPPort: httpPort, HTTPPort: httpPort,
OutboundPort: outboundPort, OutboundPort: outboundPort,
MetricsPort: metricsPort,
DebugPort: debugPort,
} }
metrics.SetConfigInfo(ctx, cfg.Options.Services, "local", cfg.Checksum(), true) metrics.SetConfigInfo(ctx, cfg.Options.Services, "local", cfg.Checksum(), true)

View file

@ -11,7 +11,7 @@ import (
) )
func TestBuilder_BuildBootstrapAdmin(t *testing.T) { func TestBuilder_BuildBootstrapAdmin(t *testing.T) {
b := New("local-grpc", "local-http", filemgr.NewManager(), nil) b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
t.Run("valid", func(t *testing.T) { t.Run("valid", func(t *testing.T) {
adminCfg, err := b.BuildBootstrapAdmin(&config.Config{ adminCfg, err := b.BuildBootstrapAdmin(&config.Config{
Options: &config.Options{ Options: &config.Options{
@ -41,7 +41,7 @@ func TestBuilder_BuildBootstrapAdmin(t *testing.T) {
} }
func TestBuilder_BuildBootstrapLayeredRuntime(t *testing.T) { func TestBuilder_BuildBootstrapLayeredRuntime(t *testing.T) {
b := New("localhost:1111", "localhost:2222", filemgr.NewManager(), nil) b := New("localhost:1111", "localhost:2222", "localhost:3333", filemgr.NewManager(), nil)
staticCfg, err := b.BuildBootstrapLayeredRuntime() staticCfg, err := b.BuildBootstrapLayeredRuntime()
assert.NoError(t, err) assert.NoError(t, err)
testutil.AssertProtoJSONEqual(t, ` testutil.AssertProtoJSONEqual(t, `
@ -58,7 +58,7 @@ func TestBuilder_BuildBootstrapLayeredRuntime(t *testing.T) {
func TestBuilder_BuildBootstrapStaticResources(t *testing.T) { func TestBuilder_BuildBootstrapStaticResources(t *testing.T) {
t.Run("valid", func(t *testing.T) { t.Run("valid", func(t *testing.T) {
b := New("localhost:1111", "localhost:2222", filemgr.NewManager(), nil) b := New("localhost:1111", "localhost:2222", "localhost:3333", filemgr.NewManager(), nil)
staticCfg, err := b.BuildBootstrapStaticResources() staticCfg, err := b.BuildBootstrapStaticResources()
assert.NoError(t, err) assert.NoError(t, err)
testutil.AssertProtoJSONEqual(t, ` testutil.AssertProtoJSONEqual(t, `
@ -90,14 +90,14 @@ func TestBuilder_BuildBootstrapStaticResources(t *testing.T) {
`, staticCfg) `, staticCfg)
}) })
t.Run("bad gRPC address", func(t *testing.T) { t.Run("bad gRPC address", func(t *testing.T) {
b := New("xyz:zyx", "localhost:2222", filemgr.NewManager(), nil) b := New("xyz:zyx", "localhost:2222", "localhost:3333", filemgr.NewManager(), nil)
_, err := b.BuildBootstrapStaticResources() _, err := b.BuildBootstrapStaticResources()
assert.Error(t, err) assert.Error(t, err)
}) })
} }
func TestBuilder_BuildBootstrapStatsConfig(t *testing.T) { func TestBuilder_BuildBootstrapStatsConfig(t *testing.T) {
b := New("local-grpc", "local-http", filemgr.NewManager(), nil) b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
t.Run("valid", func(t *testing.T) { t.Run("valid", func(t *testing.T) {
statsCfg, err := b.BuildBootstrapStatsConfig(&config.Config{ statsCfg, err := b.BuildBootstrapStatsConfig(&config.Config{
Options: &config.Options{ Options: &config.Options{

View file

@ -7,23 +7,26 @@ import (
// A Builder builds envoy config from pomerium config. // A Builder builds envoy config from pomerium config.
type Builder struct { type Builder struct {
localGRPCAddress string localGRPCAddress string
localHTTPAddress string localHTTPAddress string
filemgr *filemgr.Manager localMetricsAddress string
reproxy *reproxy.Handler filemgr *filemgr.Manager
reproxy *reproxy.Handler
} }
// New creates a new Builder. // New creates a new Builder.
func New( func New(
localGRPCAddress string, localGRPCAddress string,
localHTTPAddress string, localHTTPAddress string,
localMetricsAddress string,
fileManager *filemgr.Manager, fileManager *filemgr.Manager,
reproxyHandler *reproxy.Handler, reproxyHandler *reproxy.Handler,
) *Builder { ) *Builder {
return &Builder{ return &Builder{
localGRPCAddress: localGRPCAddress, localGRPCAddress: localGRPCAddress,
localHTTPAddress: localHTTPAddress, localHTTPAddress: localHTTPAddress,
filemgr: fileManager, localMetricsAddress: localMetricsAddress,
reproxy: reproxyHandler, filemgr: fileManager,
reproxy: reproxyHandler,
} }
} }

View file

@ -34,6 +34,10 @@ func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*env
Scheme: "http", Scheme: "http",
Host: b.localHTTPAddress, Host: b.localHTTPAddress,
} }
metricsURL := &url.URL{
Scheme: "http",
Host: b.localMetricsAddress,
}
authorizeURLs, err := cfg.Options.GetInternalAuthorizeURLs() authorizeURLs, err := cfg.Options.GetInternalAuthorizeURLs()
if err != nil { if err != nil {
return nil, err return nil, err
@ -53,6 +57,11 @@ func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*env
return nil, err return nil, err
} }
controlMetrics, err := b.buildInternalCluster(ctx, cfg.Options, "pomerium-control-plane-metrics", []*url.URL{metricsURL}, upstreamProtocolAuto)
if err != nil {
return nil, err
}
authorizeCluster, err := b.buildInternalCluster(ctx, cfg.Options, "pomerium-authorize", authorizeURLs, upstreamProtocolHTTP2) authorizeCluster, err := b.buildInternalCluster(ctx, cfg.Options, "pomerium-authorize", authorizeURLs, upstreamProtocolHTTP2)
if err != nil { if err != nil {
return nil, err return nil, err
@ -74,6 +83,7 @@ func (b *Builder) BuildClusters(ctx context.Context, cfg *config.Config) ([]*env
clusters := []*envoy_config_cluster_v3.Cluster{ clusters := []*envoy_config_cluster_v3.Cluster{
controlGRPC, controlGRPC,
controlHTTP, controlHTTP,
controlMetrics,
authorizeCluster, authorizeCluster,
databrokerCluster, databrokerCluster,
} }

View file

@ -25,7 +25,7 @@ func Test_buildPolicyTransportSocket(t *testing.T) {
cacheDir, _ := os.UserCacheDir() cacheDir, _ := os.UserCacheDir()
customCA := filepath.Join(cacheDir, "pomerium", "envoy", "files", "custom-ca-32484c314b584447463735303142374c31414145374650305a525539554938594d524855353757313942494d473847535231.pem") customCA := filepath.Join(cacheDir, "pomerium", "envoy", "files", "custom-ca-32484c314b584447463735303142374c31414145374650305a525539554938594d524855353757313942494d473847535231.pem")
b := New("local-grpc", "local-http", filemgr.NewManager(), nil) b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
rootCABytes, _ := getCombinedCertificateAuthority("", "") rootCABytes, _ := getCombinedCertificateAuthority("", "")
rootCA := b.filemgr.BytesDataSource("ca.pem", rootCABytes).GetFilename() rootCA := b.filemgr.BytesDataSource("ca.pem", rootCABytes).GetFilename()
@ -379,7 +379,7 @@ func Test_buildPolicyTransportSocket(t *testing.T) {
func Test_buildCluster(t *testing.T) { func Test_buildCluster(t *testing.T) {
ctx := context.Background() ctx := context.Background()
b := New("local-grpc", "local-http", filemgr.NewManager(), nil) b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
rootCABytes, _ := getCombinedCertificateAuthority("", "") rootCABytes, _ := getCombinedCertificateAuthority("", "")
rootCA := b.filemgr.BytesDataSource("ca.pem", rootCABytes).GetFilename() rootCA := b.filemgr.BytesDataSource("ca.pem", rootCABytes).GetFilename()
o1 := config.NewDefaultOptions() o1 := config.NewDefaultOptions()
@ -858,7 +858,7 @@ func Test_bindConfig(t *testing.T) {
ctx, clearTimeout := context.WithTimeout(context.Background(), time.Second*10) ctx, clearTimeout := context.WithTimeout(context.Background(), time.Second*10)
defer clearTimeout() defer clearTimeout()
b := New("local-grpc", "local-http", filemgr.NewManager(), nil) b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
t.Run("no bind config", func(t *testing.T) { t.Run("no bind config", func(t *testing.T) {
cluster, err := b.buildPolicyCluster(ctx, &config.Options{}, &config.Policy{ cluster, err := b.buildPolicyCluster(ctx, &config.Options{}, &config.Policy{
From: "https://from.example.com", From: "https://from.example.com",

View file

@ -486,7 +486,7 @@ func (b *Builder) buildMetricsHTTPConnectionManagerFilter() (*envoy_config_liste
Action: &envoy_config_route_v3.Route_Route{ Action: &envoy_config_route_v3.Route_Route{
Route: &envoy_config_route_v3.RouteAction{ Route: &envoy_config_route_v3.RouteAction{
ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{ ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
Cluster: "pomerium-control-plane-http", Cluster: "pomerium-control-plane-metrics",
}, },
}, },
}, },

View file

@ -25,7 +25,7 @@ func Test_buildMetricsHTTPConnectionManagerFilter(t *testing.T) {
certFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-crt-354e49305a5a39414a545530374e58454e48334148524c4e324258463837364355564c4e4532464b54355139495547514a38.pem") certFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-crt-354e49305a5a39414a545530374e58454e48334148524c4e324258463837364355564c4e4532464b54355139495547514a38.pem")
keyFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-key-3350415a38414e4e4a4655424e55393430474147324651433949384e485341334b5157364f424b4c5856365a545937383735.pem") keyFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-key-3350415a38414e4e4a4655424e55393430474147324651433949384e485341334b5157364f424b4c5856365a545937383735.pem")
b := New("local-grpc", "local-http", filemgr.NewManager(), nil) b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
li, err := b.buildMetricsListener(&config.Config{ li, err := b.buildMetricsListener(&config.Config{
Options: &config.Options{ Options: &config.Options{
MetricsAddr: "127.0.0.1:9902", MetricsAddr: "127.0.0.1:9902",
@ -65,7 +65,7 @@ func Test_buildMetricsHTTPConnectionManagerFilter(t *testing.T) {
"prefix": "/" "prefix": "/"
}, },
"route": { "route": {
"cluster": "pomerium-control-plane-http" "cluster": "pomerium-control-plane-metrics"
} }
}] }]
}] }]
@ -108,7 +108,7 @@ func Test_buildMetricsHTTPConnectionManagerFilter(t *testing.T) {
} }
func Test_buildMainHTTPConnectionManagerFilter(t *testing.T) { func Test_buildMainHTTPConnectionManagerFilter(t *testing.T) {
b := New("local-grpc", "local-http", nil, nil) b := New("local-grpc", "local-http", "local-metrics", nil, nil)
options := config.NewDefaultOptions() options := config.NewDefaultOptions()
options.SkipXffAppend = true options.SkipXffAppend = true
@ -523,7 +523,7 @@ func Test_buildMainHTTPConnectionManagerFilter(t *testing.T) {
} }
func Test_buildDownstreamTLSContext(t *testing.T) { func Test_buildDownstreamTLSContext(t *testing.T) {
b := New("local-grpc", "local-http", filemgr.NewManager(), nil) b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
cacheDir, _ := os.UserCacheDir() cacheDir, _ := os.UserCacheDir()
certFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-crt-354e49305a5a39414a545530374e58454e48334148524c4e324258463837364355564c4e4532464b54355139495547514a38.pem") certFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-crt-354e49305a5a39414a545530374e58454e48334148524c4e324258463837364355564c4e4532464b54355139495547514a38.pem")
@ -805,7 +805,7 @@ func Test_hostMatchesDomain(t *testing.T) {
} }
func Test_buildRouteConfiguration(t *testing.T) { func Test_buildRouteConfiguration(t *testing.T) {
b := New("local-grpc", "local-http", nil, nil) b := New("local-grpc", "local-http", "local-metrics", nil, nil)
virtualHosts := make([]*envoy_config_route_v3.VirtualHost, 10) virtualHosts := make([]*envoy_config_route_v3.VirtualHost, 10)
routeConfig, err := b.buildRouteConfiguration("test-route-configuration", virtualHosts) routeConfig, err := b.buildRouteConfiguration("test-route-configuration", virtualHosts)
require.NoError(t, err) require.NoError(t, err)
@ -815,7 +815,7 @@ func Test_buildRouteConfiguration(t *testing.T) {
} }
func Test_requireProxyProtocol(t *testing.T) { func Test_requireProxyProtocol(t *testing.T) {
b := New("local-grpc", "local-http", nil, nil) b := New("local-grpc", "local-http", "local-metrics", nil, nil)
t.Run("required", func(t *testing.T) { t.Run("required", func(t *testing.T) {
li, err := b.buildMainListener(context.Background(), &config.Config{Options: &config.Options{ li, err := b.buildMainListener(context.Background(), &config.Config{Options: &config.Options{
UseProxyProtocol: true, UseProxyProtocol: true,

View file

@ -57,51 +57,20 @@ func (b *Builder) buildPomeriumHTTPRoutes(options *config.Options, domain string
return nil, err return nil, err
} }
if !isFrontingAuthenticate { if !isFrontingAuthenticate {
// enable ext_authz routes = append(routes,
r, err := b.buildControlPlanePathRoute("/.pomerium/jwt", true) // enable ext_authz
if err != nil { b.buildControlPlanePathRoute("/.pomerium/jwt", true),
return nil, err // disable ext_authz and passthrough to proxy handlers
} b.buildControlPlanePathRoute("/ping", false),
routes = append(routes, r) b.buildControlPlanePathRoute("/healthz", false),
b.buildControlPlanePathRoute("/.pomerium", false),
// disable ext_authz and passthrough to proxy handlers b.buildControlPlanePrefixRoute("/.pomerium/", false),
r, err = b.buildControlPlanePathRoute("/ping", false) b.buildControlPlanePathRoute("/.well-known/pomerium", false),
if err != nil { b.buildControlPlanePrefixRoute("/.well-known/pomerium/", false),
return nil, err )
}
routes = append(routes, r)
r, err = b.buildControlPlanePathRoute("/healthz", false)
if err != nil {
return nil, err
}
routes = append(routes, r)
r, err = b.buildControlPlanePathRoute("/.pomerium", false)
if err != nil {
return nil, err
}
routes = append(routes, r)
r, err = b.buildControlPlanePrefixRoute("/.pomerium/", false)
if err != nil {
return nil, err
}
routes = append(routes, r)
r, err = b.buildControlPlanePathRoute("/.well-known/pomerium", false)
if err != nil {
return nil, err
}
routes = append(routes, r)
r, err = b.buildControlPlanePrefixRoute("/.well-known/pomerium/", false)
if err != nil {
return nil, err
}
routes = append(routes, r)
// per #837, only add robots.txt if there are no unauthenticated routes // per #837, only add robots.txt if there are no unauthenticated routes
if !hasPublicPolicyMatchingURL(options, url.URL{Scheme: "https", Host: domain, Path: "/robots.txt"}) { if !hasPublicPolicyMatchingURL(options, url.URL{Scheme: "https", Host: domain, Path: "/robots.txt"}) {
r, err := b.buildControlPlanePathRoute("/robots.txt", false) routes = append(routes, b.buildControlPlanePathRoute("/robots.txt", false))
if err != nil {
return nil, err
}
routes = append(routes, r)
} }
} }
// if we're handling authentication, add the oauth2 callback url // if we're handling authentication, add the oauth2 callback url
@ -110,11 +79,10 @@ func (b *Builder) buildPomeriumHTTPRoutes(options *config.Options, domain string
return nil, err return nil, err
} }
if config.IsAuthenticate(options.Services) && hostMatchesDomain(authenticateURL, domain) { if config.IsAuthenticate(options.Services) && hostMatchesDomain(authenticateURL, domain) {
r, err := b.buildControlPlanePrefixRoute("/", false) routes = append(routes,
if err != nil { b.buildControlPlanePathRoute(options.AuthenticateCallbackPath, false),
return nil, err b.buildControlPlanePathRoute("/", false),
} )
routes = append(routes, r)
} }
// if we're the proxy and this is the forward-auth url // if we're the proxy and this is the forward-auth url
forwardAuthURL, err := options.GetForwardAuthURL() forwardAuthURL, err := options.GetForwardAuthURL()
@ -196,7 +164,7 @@ func (b *Builder) buildControlPlanePathAndQueryRoute(path string, queryparams []
}, nil }, nil
} }
func (b *Builder) buildControlPlanePathRoute(path string, protected bool) (*envoy_config_route_v3.Route, error) { func (b *Builder) buildControlPlanePathRoute(path string, protected bool) *envoy_config_route_v3.Route {
r := &envoy_config_route_v3.Route{ r := &envoy_config_route_v3.Route{
Name: "pomerium-path-" + path, Name: "pomerium-path-" + path,
Match: &envoy_config_route_v3.RouteMatch{ Match: &envoy_config_route_v3.RouteMatch{
@ -215,10 +183,10 @@ func (b *Builder) buildControlPlanePathRoute(path string, protected bool) (*envo
"envoy.filters.http.ext_authz": disableExtAuthz, "envoy.filters.http.ext_authz": disableExtAuthz,
} }
} }
return r, nil return r
} }
func (b *Builder) buildControlPlanePrefixRoute(prefix string, protected bool) (*envoy_config_route_v3.Route, error) { func (b *Builder) buildControlPlanePrefixRoute(prefix string, protected bool) *envoy_config_route_v3.Route {
r := &envoy_config_route_v3.Route{ r := &envoy_config_route_v3.Route{
Name: "pomerium-prefix-" + prefix, Name: "pomerium-prefix-" + prefix,
Match: &envoy_config_route_v3.RouteMatch{ Match: &envoy_config_route_v3.RouteMatch{
@ -237,7 +205,7 @@ func (b *Builder) buildControlPlanePrefixRoute(prefix string, protected bool) (*
"envoy.filters.http.ext_authz": disableExtAuthz, "envoy.filters.http.ext_authz": disableExtAuthz,
} }
} }
return r, nil return r
} }
// getClusterID returns a cluster ID // getClusterID returns a cluster ID

View file

@ -96,7 +96,8 @@ func Test_buildPomeriumHTTPRoutes(t *testing.T) {
`+routeString("path", "/.well-known/pomerium", false)+`, `+routeString("path", "/.well-known/pomerium", false)+`,
`+routeString("prefix", "/.well-known/pomerium/", false)+`, `+routeString("prefix", "/.well-known/pomerium/", false)+`,
`+routeString("path", "/robots.txt", false)+`, `+routeString("path", "/robots.txt", false)+`,
`+routeString("prefix", "/", false)+` `+routeString("path", "/oauth2/callback", false)+`,
`+routeString("path", "/", false)+`
]`, routes) ]`, routes)
}) })
t.Run("proxy fronting authenticate", func(t *testing.T) { t.Run("proxy fronting authenticate", func(t *testing.T) {
@ -167,8 +168,7 @@ func Test_buildPomeriumHTTPRoutes(t *testing.T) {
func Test_buildControlPlanePathRoute(t *testing.T) { func Test_buildControlPlanePathRoute(t *testing.T) {
b := &Builder{filemgr: filemgr.NewManager()} b := &Builder{filemgr: filemgr.NewManager()}
route, err := b.buildControlPlanePathRoute("/hello/world", false) route := b.buildControlPlanePathRoute("/hello/world", false)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, ` testutil.AssertProtoJSONEqual(t, `
{ {
"name": "pomerium-path-/hello/world", "name": "pomerium-path-/hello/world",
@ -190,8 +190,7 @@ func Test_buildControlPlanePathRoute(t *testing.T) {
func Test_buildControlPlanePrefixRoute(t *testing.T) { func Test_buildControlPlanePrefixRoute(t *testing.T) {
b := &Builder{filemgr: filemgr.NewManager()} b := &Builder{filemgr: filemgr.NewManager()}
route, err := b.buildControlPlanePrefixRoute("/hello/world/", false) route := b.buildControlPlanePrefixRoute("/hello/world/", false)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, ` testutil.AssertProtoJSONEqual(t, `
{ {
"name": "pomerium-prefix-/hello/world/", "name": "pomerium-prefix-/hello/world/",

View file

@ -3,6 +3,7 @@ package config
import ( import (
"context" "context"
"net/http" "net/http"
"net/url"
"os" "os"
"sync" "sync"
@ -16,12 +17,13 @@ import (
// A MetricsManager manages metrics for a given configuration. // A MetricsManager manages metrics for a given configuration.
type MetricsManager struct { type MetricsManager struct {
mu sync.RWMutex mu sync.RWMutex
installationID string installationID string
serviceName string serviceName string
addr string addr string
basicAuth string basicAuth string
handler http.Handler envoyAdminAddress string
handler http.Handler
} }
// NewMetricsManager creates a new MetricsManager. // NewMetricsManager creates a new MetricsManager.
@ -46,8 +48,8 @@ func (mgr *MetricsManager) OnConfigChange(ctx context.Context, cfg *Config) {
mgr.mu.Lock() mgr.mu.Lock()
defer mgr.mu.Unlock() defer mgr.mu.Unlock()
mgr.updateInfo(cfg) mgr.updateInfo(ctx, cfg)
mgr.updateServer(cfg) mgr.updateServer(ctx, cfg)
} }
func (mgr *MetricsManager) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (mgr *MetricsManager) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@ -61,7 +63,7 @@ func (mgr *MetricsManager) ServeHTTP(w http.ResponseWriter, r *http.Request) {
mgr.handler.ServeHTTP(w, r) mgr.handler.ServeHTTP(w, r)
} }
func (mgr *MetricsManager) updateInfo(cfg *Config) { func (mgr *MetricsManager) updateInfo(ctx context.Context, cfg *Config) {
serviceName := telemetry.ServiceName(cfg.Options.Services) serviceName := telemetry.ServiceName(cfg.Options.Services)
if serviceName == mgr.serviceName { if serviceName == mgr.serviceName {
return return
@ -69,7 +71,7 @@ func (mgr *MetricsManager) updateInfo(cfg *Config) {
hostname, err := os.Hostname() hostname, err := os.Hostname()
if err != nil { if err != nil {
log.Error(context.TODO()).Err(err).Msg("telemetry/metrics: failed to get OS hostname") log.Error(ctx).Err(err).Msg("telemetry/metrics: failed to get OS hostname")
hostname = "__unknown__" hostname = "__unknown__"
} }
@ -77,26 +79,34 @@ func (mgr *MetricsManager) updateInfo(cfg *Config) {
mgr.serviceName = serviceName mgr.serviceName = serviceName
} }
func (mgr *MetricsManager) updateServer(cfg *Config) { func (mgr *MetricsManager) updateServer(ctx context.Context, cfg *Config) {
if cfg.Options.MetricsAddr == mgr.addr && if cfg.Options.MetricsAddr == mgr.addr &&
cfg.Options.MetricsBasicAuth == mgr.basicAuth && cfg.Options.MetricsBasicAuth == mgr.basicAuth &&
cfg.Options.InstallationID == mgr.installationID { cfg.Options.InstallationID == mgr.installationID &&
cfg.Options.EnvoyAdminAddress == mgr.envoyAdminAddress {
return return
} }
mgr.addr = cfg.Options.MetricsAddr mgr.addr = cfg.Options.MetricsAddr
mgr.basicAuth = cfg.Options.MetricsBasicAuth mgr.basicAuth = cfg.Options.MetricsBasicAuth
mgr.installationID = cfg.Options.InstallationID mgr.installationID = cfg.Options.InstallationID
mgr.envoyAdminAddress = cfg.Options.EnvoyAdminAddress
mgr.handler = nil mgr.handler = nil
if mgr.addr == "" { if mgr.addr == "" {
log.Info(context.TODO()).Msg("metrics: http server disabled") log.Info(ctx).Msg("metrics: http server disabled")
return return
} }
handler, err := metrics.PrometheusHandler(EnvoyAdminURL, mgr.installationID) envoyURL, err := url.Parse("http://" + cfg.Options.EnvoyAdminAddress)
if err != nil { if err != nil {
log.Error(context.TODO()).Err(err).Msg("metrics: failed to create prometheus handler") log.Error(ctx).Err(err).Msg("metrics: invalid envoy admin address, disabling")
return
}
handler, err := metrics.PrometheusHandler(envoyURL, mgr.installationID)
if err != nil {
log.Error(ctx).Err(err).Msg("metrics: failed to create prometheus handler")
return return
} }

View file

@ -49,9 +49,6 @@ const (
// gRPC server, or is used for healthchecks (authorize only service) // gRPC server, or is used for healthchecks (authorize only service)
const DefaultAlternativeAddr = ":5443" const DefaultAlternativeAddr = ":5443"
// EnvoyAdminURL indicates where the envoy control plane is listening
var EnvoyAdminURL = &url.URL{Host: "127.0.0.1:9901", Scheme: "http"}
// The randomSharedKey is used if no shared key is supplied in all-in-one mode. // The randomSharedKey is used if no shared key is supplied in all-in-one mode.
var randomSharedKey = cryptutil.NewBase64Key() var randomSharedKey = cryptutil.NewBase64Key()

View file

@ -86,6 +86,8 @@ func Run(ctx context.Context, configFile string) error {
Str("grpc-port", src.GetConfig().GRPCPort). Str("grpc-port", src.GetConfig().GRPCPort).
Str("http-port", src.GetConfig().HTTPPort). Str("http-port", src.GetConfig().HTTPPort).
Str("outbound-port", src.GetConfig().OutboundPort). Str("outbound-port", src.GetConfig().OutboundPort).
Str("metrics-port", src.GetConfig().MetricsPort).
Str("debug-port", src.GetConfig().DebugPort).
Msg("server started") Msg("server started")
// create envoy server // create envoy server

View file

@ -49,12 +49,12 @@ func (srv *Server) addHTTPMiddleware() {
root.HandleFunc("/ping", httputil.HealthCheck) root.HandleFunc("/ping", httputil.HealthCheck)
// pprof // pprof
root.Path("/debug/pprof/cmdline").HandlerFunc(pprof.Cmdline) srv.DebugRouter.Path("/debug/pprof/cmdline").HandlerFunc(pprof.Cmdline)
root.Path("/debug/pprof/profile").HandlerFunc(pprof.Profile) srv.DebugRouter.Path("/debug/pprof/profile").HandlerFunc(pprof.Profile)
root.Path("/debug/pprof/symbol").HandlerFunc(pprof.Symbol) srv.DebugRouter.Path("/debug/pprof/symbol").HandlerFunc(pprof.Symbol)
root.Path("/debug/pprof/trace").HandlerFunc(pprof.Trace) srv.DebugRouter.Path("/debug/pprof/trace").HandlerFunc(pprof.Trace)
root.PathPrefix("/debug/pprof/").HandlerFunc(pprof.Index) srv.DebugRouter.PathPrefix("/debug/pprof/").HandlerFunc(pprof.Index)
// metrics // metrics
root.Handle("/metrics", srv.metricsMgr) srv.MetricsRouter.Handle("/metrics", srv.metricsMgr)
} }

View file

@ -50,11 +50,15 @@ func (avo *atomicVersionedConfig) Store(cfg versionedConfig) {
// A Server is the control-plane gRPC and HTTP servers. // A Server is the control-plane gRPC and HTTP servers.
type Server struct { type Server struct {
GRPCListener net.Listener GRPCListener net.Listener
GRPCServer *grpc.Server GRPCServer *grpc.Server
HTTPListener net.Listener HTTPListener net.Listener
HTTPRouter *mux.Router HTTPRouter *mux.Router
Builder *envoyconfig.Builder MetricsListener net.Listener
MetricsRouter *mux.Router
DebugListener net.Listener
DebugRouter *mux.Router
Builder *envoyconfig.Builder
currentConfig atomicVersionedConfig currentConfig atomicVersionedConfig
name string name string
@ -106,7 +110,25 @@ func NewServer(cfg *config.Config, metricsMgr *config.MetricsManager) (*Server,
_ = srv.GRPCListener.Close() _ = srv.GRPCListener.Close()
return nil, err return nil, err
} }
srv.MetricsListener, err = net.Listen("tcp4", net.JoinHostPort("127.0.0.1", cfg.MetricsPort))
if err != nil {
_ = srv.GRPCListener.Close()
_ = srv.HTTPListener.Close()
return nil, err
}
srv.DebugListener, err = net.Listen("tcp4", net.JoinHostPort("127.0.0.1", cfg.DebugPort))
if err != nil {
_ = srv.GRPCListener.Close()
_ = srv.HTTPListener.Close()
_ = srv.DebugListener.Close()
return nil, err
}
srv.HTTPRouter = mux.NewRouter() srv.HTTPRouter = mux.NewRouter()
srv.DebugRouter = mux.NewRouter()
srv.MetricsRouter = mux.NewRouter()
srv.addHTTPMiddleware() srv.addHTTPMiddleware()
srv.filemgr = filemgr.NewManager() srv.filemgr = filemgr.NewManager()
@ -115,6 +137,7 @@ func NewServer(cfg *config.Config, metricsMgr *config.MetricsManager) (*Server,
srv.Builder = envoyconfig.New( srv.Builder = envoyconfig.New(
srv.GRPCListener.Addr().String(), srv.GRPCListener.Addr().String(),
srv.HTTPListener.Addr().String(), srv.HTTPListener.Addr().String(),
srv.MetricsListener.Addr().String(),
srv.filemgr, srv.filemgr,
srv.reproxy, srv.reproxy,
) )
@ -175,28 +198,41 @@ func (srv *Server) Run(ctx context.Context) error {
return nil return nil
}) })
hsrv := (&http.Server{ for _, entry := range []struct {
BaseContext: func(li net.Listener) context.Context { Name string
return ctx Listener net.Listener
}, Handler *mux.Router
Handler: srv.HTTPRouter, }{
}) {"http", srv.HTTPListener, srv.HTTPRouter},
{"debug", srv.DebugListener, srv.DebugRouter},
{"metrics", srv.MetricsListener, srv.MetricsRouter},
} {
entry := entry
hsrv := (&http.Server{
BaseContext: func(li net.Listener) context.Context {
return ctx
},
Handler: entry.Handler,
})
// start the HTTP server // start the HTTP server
eg.Go(func() error { eg.Go(func() error {
log.Info(ctx).Str("addr", srv.HTTPListener.Addr().String()).Msg("starting control-plane HTTP server") log.Info(ctx).
return hsrv.Serve(srv.HTTPListener) Str("addr", entry.Listener.Addr().String()).
}) Msgf("starting control-plane %s server", entry.Name)
return hsrv.Serve(entry.Listener)
})
// gracefully stop the HTTP server on context cancellation // gracefully stop the HTTP server on context cancellation
eg.Go(func() error { eg.Go(func() error {
<-ctx.Done() <-ctx.Done()
ctx, cleanup := context.WithTimeout(ctx, time.Second*5) ctx, cleanup := context.WithTimeout(ctx, time.Second*5)
defer cleanup() defer cleanup()
return hsrv.Shutdown(ctx) return hsrv.Shutdown(ctx)
}) })
}
return eg.Wait() return eg.Wait()
} }

View file

@ -66,6 +66,10 @@ func (r *Reporter) OnConfigChange(ctx context.Context, cfg *config.Config) {
} }
func getReportedServices(cfg *config.Config) ([]*pb.Service, error) { func getReportedServices(cfg *config.Config) ([]*pb.Service, error) {
if cfg.Options.MetricsAddr == "" {
return nil, nil
}
mu, err := metricsURL(*cfg.Options) mu, err := metricsURL(*cfg.Options)
if err != nil { if err != nil {
return nil, err return nil, err