databroker: refactor databroker to sync all changes (#1879)

* refactor backend, implement encrypted store

* refactor in-memory store

* wip

* wip

* wip

* add syncer test

* fix redis expiry

* fix linting issues

* fix test by skipping non-config records

* fix backoff import

* fix init issues

* fix query

* wait for initial sync before starting directory sync

* add type to SyncLatest

* add more log messages, fix deadlock in in-memory store, always return server version from SyncLatest

* update sync types and tests

* add redis tests

* skip macos in github actions

* add comments to proto

* split getBackend into separate methods

* handle errors in initVersion

* return different error for not found vs other errors in get

* use exponential backoff for redis transaction retry

* rename raw to result

* use context instead of close channel

* store type urls as constants in databroker

* use timestampb instead of ptypes

* fix group merging not waiting

* change locked names

* update GetAll to return latest record version

* add method to grpcutil to get the type url for a protobuf type

internal/databroker/config_source.go

@@ -3,12 +3,7 @@ package databroker
 import (
 	"context"
 	"encoding/base64"
-	"errors"
 	"sync"
-	"time"
-
-	"github.com/cenkalti/backoff/v4"
-	"github.com/golang/protobuf/ptypes"
 
 	"github.com/pomerium/pomerium/config"
 	"github.com/pomerium/pomerium/internal/hashutil"
@@ -17,15 +12,9 @@ import (
 	"github.com/pomerium/pomerium/pkg/grpc"
 	configpb "github.com/pomerium/pomerium/pkg/grpc/config"
 	"github.com/pomerium/pomerium/pkg/grpc/databroker"
+	"github.com/pomerium/pomerium/pkg/grpcutil"
 )
 
-var configTypeURL string
-
-func init() {
-	any, _ := ptypes.MarshalAny(new(configpb.Config))
-	configTypeURL = any.GetTypeUrl()
-}
-
 // ConfigSource provides a new Config source that decorates an underlying config with
 // configuration derived from the data broker.
 type ConfigSource struct {
@@ -35,8 +24,6 @@ type ConfigSource struct {
 	dbConfigs        map[string]*configpb.Config
 	updaterHash      uint64
 	cancel           func()
-	serverVersion    string
-	recordVersion    string
 
 	config.ChangeDispatcher
 }
@@ -188,78 +175,51 @@ func (src *ConfigSource) runUpdater(cfg *config.Config) {
 	ctx := context.Background()
 	ctx, src.cancel = context.WithCancel(ctx)
 
-	go tryForever(ctx, func(onSuccess func()) error {
-		src.mu.Lock()
-		serverVersion, recordVersion := src.serverVersion, src.recordVersion
-		src.mu.Unlock()
-
-		stream, err := client.Sync(ctx, &databroker.SyncRequest{
-			Type:          configTypeURL,
-			ServerVersion: serverVersion,
-			RecordVersion: recordVersion,
-		})
-		if err != nil {
-			return err
-		}
-
-		for {
-			res, err := stream.Recv()
-			if err != nil {
-				return err
-			}
-			onSuccess()
-
-			if len(res.GetRecords()) > 0 {
-				src.onSync(res.GetRecords())
-				for _, record := range res.GetRecords() {
-					recordVersion = record.GetVersion()
-				}
-			}
-
-			src.mu.Lock()
-			src.serverVersion, src.recordVersion = res.GetServerVersion(), recordVersion
-			src.mu.Unlock()
-		}
-	})
+	syncer := databroker.NewSyncer(&syncerHandler{
+		client: client,
+		src:    src,
+	}, databroker.WithTypeURL(grpcutil.GetTypeURL(new(configpb.Config))))
+	go func() { _ = syncer.Run(ctx) }()
 }
 
-func (src *ConfigSource) onSync(records []*databroker.Record) {
-	src.mu.Lock()
+type syncerHandler struct {
+	src    *ConfigSource
+	client databroker.DataBrokerServiceClient
+}
+
+func (s *syncerHandler) GetDataBrokerServiceClient() databroker.DataBrokerServiceClient {
+	return s.client
+}
+
+func (s *syncerHandler) ClearRecords(ctx context.Context) {
+	s.src.mu.Lock()
+	s.src.dbConfigs = map[string]*configpb.Config{}
+	s.src.mu.Unlock()
+}
+
+func (s *syncerHandler) UpdateRecords(ctx context.Context, records []*databroker.Record) {
+	if len(records) == 0 {
+		return
+	}
+
+	s.src.mu.Lock()
 	for _, record := range records {
 		if record.GetDeletedAt() != nil {
-			delete(src.dbConfigs, record.GetId())
+			delete(s.src.dbConfigs, record.GetId())
 			continue
 		}
 
 		var cfgpb configpb.Config
-		err := ptypes.UnmarshalAny(record.GetData(), &cfgpb)
+		err := record.GetData().UnmarshalTo(&cfgpb)
 		if err != nil {
 			log.Warn().Err(err).Msg("databroker: error decoding config")
-			delete(src.dbConfigs, record.GetId())
+			delete(s.src.dbConfigs, record.GetId())
 			continue
 		}
 
-		src.dbConfigs[record.GetId()] = &cfgpb
+		s.src.dbConfigs[record.GetId()] = &cfgpb
 	}
-	src.mu.Unlock()
+	s.src.mu.Unlock()
 
-	src.rebuild(false)
-}
-
-func tryForever(ctx context.Context, callback func(onSuccess func()) error) {
-	bo := backoff.NewExponentialBackOff()
-	bo.MaxElapsedTime = 0
-	for {
-		err := callback(bo.Reset)
-		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
-			return
-		} else if err != nil {
-			log.Warn().Err(err).Msg("sync error")
-		}
-		select {
-		case <-ctx.Done():
-			return
-		case <-time.After(bo.NextBackOff()):
-		}
-	}
+	s.src.rebuild(false)
 }