cache: support databroker option changes (#1294)

cache/cache.go

@@ -15,6 +15,7 @@ import (
 	"github.com/pomerium/pomerium/internal/directory"
 	"github.com/pomerium/pomerium/internal/identity"
 	"github.com/pomerium/pomerium/internal/identity/manager"
+	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/internal/telemetry"
 	"github.com/pomerium/pomerium/internal/urlutil"
 	"github.com/pomerium/pomerium/pkg/cryptutil"
@@ -35,18 +36,7 @@ type Cache struct {
 }
 
 // New creates a new cache service.
-func New(opts config.Options) (*Cache, error) {
-	if err := validate(opts); err != nil {
-		return nil, fmt.Errorf("cache: bad option: %w", err)
-	}
-
-	authenticator, err := identity.NewAuthenticator(opts.GetOauthOptions())
-	if err != nil {
-		return nil, fmt.Errorf("cache: failed to create authenticator: %w", err)
-	}
-
-	directoryProvider := directory.GetProvider(&opts)
-
+func New(cfg *config.Config) (*Cache, error) {
 	localListener, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
 		return nil, err
@@ -56,7 +46,7 @@ func New(opts config.Options) (*Cache, error) {
 	// if we no longer register with that grpc Server
 	localGRPCServer := grpc.NewServer()
 
-	clientStatsHandler := telemetry.NewGRPCClientStatsHandler(opts.Services)
+	clientStatsHandler := telemetry.NewGRPCClientStatsHandler(cfg.Options.Services)
 	clientDialOptions := clientStatsHandler.DialOptions(grpc.WithInsecure())
 
 	localGRPCConnection, err := grpc.DialContext(
@@ -68,30 +58,33 @@ func New(opts config.Options) (*Cache, error) {
 		return nil, err
 	}
 
-	dataBrokerServer, err := NewDataBrokerServer(localGRPCServer, opts)
-	if err != nil {
-		return nil, err
-	}
-	dataBrokerClient := databroker.NewDataBrokerServiceClient(localGRPCConnection)
-
-	manager := manager.New(
-		authenticator,
-		directoryProvider,
-		dataBrokerClient,
-		manager.WithGroupRefreshInterval(opts.RefreshDirectoryInterval),
-		manager.WithGroupRefreshTimeout(opts.RefreshDirectoryTimeout),
-	)
-
-	return &Cache{
-		dataBrokerServer: dataBrokerServer,
-		manager:          manager,
+	dataBrokerServer := NewDataBrokerServer(localGRPCServer, cfg)
 
+	c := &Cache{
+		dataBrokerServer:             dataBrokerServer,
 		localListener:                localListener,
 		localGRPCServer:              localGRPCServer,
 		localGRPCConnection:          localGRPCConnection,
-		deprecatedCacheClusterDomain: opts.GetDataBrokerURL().Hostname(),
-		dataBrokerStorageType:        opts.DataBrokerStorageType,
-	}, nil
+		deprecatedCacheClusterDomain: cfg.Options.GetDataBrokerURL().Hostname(),
+		dataBrokerStorageType:        cfg.Options.DataBrokerStorageType,
+	}
+
+	err = c.update(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+// OnConfigChange is called whenever configuration is changed.
+func (c *Cache) OnConfigChange(cfg *config.Config) {
+	err := c.update(cfg)
+	if err != nil {
+		log.Error().Err(err).Msg("cache: error updating configuration")
+	}
+
+	c.dataBrokerServer.OnConfigChange(cfg)
 }
 
 // Register registers all the gRPC services with the given server.
@@ -121,9 +114,45 @@ func (c *Cache) Run(ctx context.Context) error {
 	return t.Wait()
 }
 
+func (c *Cache) update(cfg *config.Config) error {
+	if err := validate(cfg.Options); err != nil {
+		return fmt.Errorf("cache: bad option: %w", err)
+	}
+
+	authenticator, err := identity.NewAuthenticator(cfg.Options.GetOauthOptions())
+	if err != nil {
+		return fmt.Errorf("cache: failed to create authenticator: %w", err)
+	}
+
+	directoryProvider := directory.GetProvider(directory.Options{
+		ServiceAccount: cfg.Options.ServiceAccount,
+		Provider:       cfg.Options.Provider,
+		ProviderURL:    cfg.Options.ProviderURL,
+		QPS:            cfg.Options.QPS,
+	})
+
+	dataBrokerClient := databroker.NewDataBrokerServiceClient(c.localGRPCConnection)
+
+	options := []manager.Option{
+		manager.WithAuthenticator(authenticator),
+		manager.WithDirectoryProvider(directoryProvider),
+		manager.WithDataBrokerClient(dataBrokerClient),
+		manager.WithGroupRefreshInterval(cfg.Options.RefreshDirectoryInterval),
+		manager.WithGroupRefreshTimeout(cfg.Options.RefreshDirectoryTimeout),
+	}
+
+	if c.manager == nil {
+		c.manager = manager.New(options...)
+	} else {
+		c.manager.UpdateConfig(options...)
+	}
+
+	return nil
+}
+
 // validate checks that proper configuration settings are set to create
 // a cache instance
-func validate(o config.Options) error {
+func validate(o *config.Options) error {
 	if _, err := cryptutil.NewAEADCipherFromBase64(o.SharedKey); err != nil {
 		return fmt.Errorf("invalid 'SHARED_SECRET': %w", err)
 	}

cache/cache_test.go

@@ -30,7 +30,7 @@ func TestNew(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			tt.opts.Provider = "google"
-			_, err := New(tt.opts)
+			_, err := New(&config.Config{Options: &tt.opts})
 			if (err != nil) != tt.wantErr {
 				t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
 				return

cache/databroker.go

@@ -1,55 +1,38 @@
 package cache
 
 import (
-	"crypto/tls"
-	"crypto/x509"
-	"encoding/base64"
-	"fmt"
-	"io/ioutil"
-
 	"google.golang.org/grpc"
 
 	"github.com/pomerium/pomerium/config"
-	internal_databroker "github.com/pomerium/pomerium/internal/databroker"
-	"github.com/pomerium/pomerium/internal/log"
-	"github.com/pomerium/pomerium/pkg/cryptutil"
-	"github.com/pomerium/pomerium/pkg/grpc/databroker"
+	"github.com/pomerium/pomerium/internal/databroker"
+	databrokerpb "github.com/pomerium/pomerium/pkg/grpc/databroker"
 )
 
 // A DataBrokerServer implements the data broker service interface.
 type DataBrokerServer struct {
-	databroker.DataBrokerServiceServer
+	*databroker.Server
 }
 
 // NewDataBrokerServer creates a new databroker service server.
-func NewDataBrokerServer(grpcServer *grpc.Server, opts config.Options) (*DataBrokerServer, error) {
-	key, err := base64.StdEncoding.DecodeString(opts.SharedKey)
-	if err != nil || len(key) != cryptutil.DefaultKeySize {
-		return nil, fmt.Errorf("shared key is required and must be %d bytes long", cryptutil.DefaultKeySize)
-	}
-
-	caCertPool := x509.NewCertPool()
-	if caCert, err := ioutil.ReadFile(opts.DataBrokerStorageCAFile); err == nil {
-		caCertPool.AppendCertsFromPEM(caCert)
-	} else {
-		log.Warn().Err(err).Msg("failed to read databroker CA file")
-	}
-	tlsConfig := &tls.Config{
-		RootCAs: caCertPool,
-		// nolint: gosec
-		InsecureSkipVerify: opts.DataBrokerStorageCertSkipVerify,
-	}
-	if opts.DataBrokerCertificate != nil {
-		tlsConfig.Certificates = []tls.Certificate{*opts.DataBrokerCertificate}
-	}
-
-	internalSrv := internal_databroker.New(
-		internal_databroker.WithSecret(key),
-		internal_databroker.WithStorageType(opts.DataBrokerStorageType),
-		internal_databroker.WithStorageConnectionString(opts.DataBrokerStorageConnectionString),
-		internal_databroker.WithStorageTLSConfig(tlsConfig),
-	)
-	srv := &DataBrokerServer{DataBrokerServiceServer: internalSrv}
-	databroker.RegisterDataBrokerServiceServer(grpcServer, srv)
-	return srv, nil
+func NewDataBrokerServer(grpcServer *grpc.Server, cfg *config.Config) *DataBrokerServer {
+	srv := &DataBrokerServer{}
+	srv.Server = databroker.New(srv.getOptions(cfg)...)
+	databrokerpb.RegisterDataBrokerServiceServer(grpcServer, srv)
+	return srv
+}
+
+// OnConfigChange updates the underlying databroker server whenever configuration is changed.
+func (srv *DataBrokerServer) OnConfigChange(cfg *config.Config) {
+	srv.UpdateConfig(srv.getOptions(cfg)...)
+}
+
+func (srv *DataBrokerServer) getOptions(cfg *config.Config) []databroker.ServerOption {
+	return []databroker.ServerOption{
+		databroker.WithSharedKey(cfg.Options.SharedKey),
+		databroker.WithStorageType(cfg.Options.DataBrokerStorageType),
+		databroker.WithStorageConnectionString(cfg.Options.DataBrokerStorageConnectionString),
+		databroker.WithStorageCAFile(cfg.Options.DataBrokerStorageCAFile),
+		databroker.WithStorageCertificate(cfg.Options.DataBrokerCertificate),
+		databroker.WithStorageCertSkipVerify(cfg.Options.DataBrokerStorageCertSkipVerify),
+	}
 }

cache/databroker_test.go

@@ -27,7 +27,7 @@ func init() {
 	lis = bufconn.Listen(bufSize)
 	s := grpc.NewServer()
 	internalSrv := internal_databroker.New()
-	srv := &DataBrokerServer{DataBrokerServiceServer: internalSrv}
+	srv := &DataBrokerServer{Server: internalSrv}
 	databroker.RegisterDataBrokerServiceServer(s, srv)
 
 	go func() {

cache/memberlist_test.go

@@ -15,10 +15,12 @@ import (
 )
 
 func TestCache_runMemberList(t *testing.T) {
-	c, err := New(config.Options{
-		SharedKey:     cryptutil.NewBase64Key(),
-		DataBrokerURL: &url.URL{Scheme: "http", Host: "member1"},
-		Provider:      "google",
+	c, err := New(&config.Config{
+		Options: &config.Options{
+			SharedKey:     cryptutil.NewBase64Key(),
+			DataBrokerURL: &url.URL{Scheme: "http", Host: "member1"},
+			Provider:      "google",
+		},
 	})
 	require.NoError(t, err)