databroker: refactor databroker to sync all changes (#1879)

* refactor backend, implement encrypted store

* refactor in-memory store

* wip

* wip

* wip

* add syncer test

* fix redis expiry

* fix linting issues

* fix test by skipping non-config records

* fix backoff import

* fix init issues

* fix query

* wait for initial sync before starting directory sync

* add type to SyncLatest

* add more log messages, fix deadlock in in-memory store, always return server version from SyncLatest

* update sync types and tests

* add redis tests

* skip macos in github actions

* add comments to proto

* split getBackend into separate methods

* handle errors in initVersion

* return different error for not found vs other errors in get

* use exponential backoff for redis transaction retry

* rename raw to result

* use context instead of close channel

* store type urls as constants in databroker

* use timestampb instead of ptypes

* fix group merging not waiting

* change locked names

* update GetAll to return latest record version

* add method to grpcutil to get the type url for a protobuf type

authorize/authorize.go

@@ -24,19 +24,16 @@ type Authorize struct {
 	currentOptions *config.AtomicOptions
 	templates      *template.Template
 
-	dataBrokerInitialSync map[string]chan struct{}
+	dataBrokerInitialSync chan struct{}
 }
 
 // New validates and creates a new Authorize service from a set of config options.
 func New(cfg *config.Config) (*Authorize, error) {
 	a := Authorize{
-		currentOptions: config.NewAtomicOptions(),
-		store:          evaluator.NewStore(),
-		templates:      template.Must(frontend.NewTemplates()),
-		dataBrokerInitialSync: map[string]chan struct{}{
-			"type.googleapis.com/directory.Group": make(chan struct{}, 1),
-			"type.googleapis.com/directory.User":  make(chan struct{}, 1),
-		},
+		currentOptions:        config.NewAtomicOptions(),
+		store:                 evaluator.NewStore(),
+		templates:             template.Must(frontend.NewTemplates()),
+		dataBrokerInitialSync: make(chan struct{}),
 	}
 
 	state, err := newAuthorizeStateFromConfig(cfg, a.store)
@@ -48,6 +45,22 @@ func New(cfg *config.Config) (*Authorize, error) {
 	return &a, nil
 }
 
+// Run runs the authorize service.
+func (a *Authorize) Run(ctx context.Context) error {
+	return newDataBrokerSyncer(a).Run(ctx)
+}
+
+// WaitForInitialSync blocks until the initial sync is complete.
+func (a *Authorize) WaitForInitialSync(ctx context.Context) error {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-a.dataBrokerInitialSync:
+	}
+	log.Info().Msg("initial sync from databroker complete")
+	return nil
+}
+
 func validateOptions(o *config.Options) error {
 	if _, err := cryptutil.NewAEADCipherFromBase64(o.SharedKey); err != nil {
 		return fmt.Errorf("authorize: bad 'SHARED_SECRET': %w", err)