databroker: refactor databroker to sync all changes (#1879)

* refactor backend, implement encrypted store

* refactor in-memory store

* wip

* wip

* wip

* add syncer test

* fix redis expiry

* fix linting issues

* fix test by skipping non-config records

* fix backoff import

* fix init issues

* fix query

* wait for initial sync before starting directory sync

* add type to SyncLatest

* add more log messages, fix deadlock in in-memory store, always return server version from SyncLatest

* update sync types and tests

* add redis tests

* skip macos in github actions

* add comments to proto

* split getBackend into separate methods

* handle errors in initVersion

* return different error for not found vs other errors in get

* use exponential backoff for redis transaction retry

* rename raw to result

* use context instead of close channel

* store type urls as constants in databroker

* use timestampb instead of ptypes

* fix group merging not waiting

* change locked names

* update GetAll to return latest record version

* add method to grpcutil to get the type url for a protobuf type
This commit is contained in:
Caleb Doxsey 2021-02-18 15:24:33 -07:00 committed by GitHub
parent b1871b0f2e
commit 5d60cff21e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
66 changed files with 2762 additions and 2871 deletions

View file

@ -32,13 +32,15 @@ func Get(ctx context.Context, client databroker.DataBrokerServiceClient, userID
return &u, nil
}
// Set sets a user in the databroker.
func Set(ctx context.Context, client databroker.DataBrokerServiceClient, u *User) (*databroker.Record, error) {
// Put sets a user in the databroker.
func Put(ctx context.Context, client databroker.DataBrokerServiceClient, u *User) (*databroker.Record, error) {
any, _ := anypb.New(u)
res, err := client.Set(ctx, &databroker.SetRequest{
Type: any.GetTypeUrl(),
Id: u.Id,
Data: any,
res, err := client.Put(ctx, &databroker.PutRequest{
Record: &databroker.Record{
Type: any.GetTypeUrl(),
Id: u.Id,
Data: any,
},
})
if err != nil {
return nil, err
@ -46,13 +48,15 @@ func Set(ctx context.Context, client databroker.DataBrokerServiceClient, u *User
return res.GetRecord(), nil
}
// SetServiceAccount sets a service account in the databroker.
func SetServiceAccount(ctx context.Context, client databroker.DataBrokerServiceClient, sa *ServiceAccount) (*databroker.Record, error) {
// PutServiceAccount sets a service account in the databroker.
func PutServiceAccount(ctx context.Context, client databroker.DataBrokerServiceClient, sa *ServiceAccount) (*databroker.Record, error) {
any, _ := anypb.New(sa)
res, err := client.Set(ctx, &databroker.SetRequest{
Type: any.GetTypeUrl(),
Id: sa.GetId(),
Data: any,
res, err := client.Put(ctx, &databroker.PutRequest{
Record: &databroker.Record{
Type: any.GetTypeUrl(),
Id: sa.GetId(),
Data: any,
},
})
if err != nil {
return nil, err