databroker: refactor databroker to sync all changes (#1879)

* refactor backend, implement encrypted store

* refactor in-memory store

* wip

* wip

* wip

* add syncer test

* fix redis expiry

* fix linting issues

* fix test by skipping non-config records

* fix backoff import

* fix init issues

* fix query

* wait for initial sync before starting directory sync

* add type to SyncLatest

* add more log messages, fix deadlock in in-memory store, always return server version from SyncLatest

* update sync types and tests

* add redis tests

* skip macos in github actions

* add comments to proto

* split getBackend into separate methods

* handle errors in initVersion

* return different error for not found vs other errors in get

* use exponential backoff for redis transaction retry

* rename raw to result

* use context instead of close channel

* store type urls as constants in databroker

* use timestampb instead of ptypes

* fix group merging not waiting

* change locked names

* update GetAll to return latest record version

* add method to grpcutil to get the type url for a protobuf type
This commit is contained in:
Caleb Doxsey 2021-02-18 15:24:33 -07:00 committed by GitHub
parent b1871b0f2e
commit 5d60cff21e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
66 changed files with 2762 additions and 2871 deletions

View file

@ -23,7 +23,6 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/types/known/emptypb"
"gopkg.in/square/go-jose.v2/jwt"
"github.com/pomerium/pomerium/config"
@ -166,7 +165,7 @@ func TestAuthenticate_SignIn(t *testing.T) {
return &databroker.GetResponse{
Record: &databroker.Record{
Version: "0001",
Version: 1,
Type: data.GetTypeUrl(),
Id: "SESSION_ID",
Data: data,
@ -246,9 +245,6 @@ func TestAuthenticate_SignOut(t *testing.T) {
encryptedEncoder: mock.Encoder{},
sharedEncoder: mock.Encoder{},
dataBrokerClient: mockDataBrokerServiceClient{
delete: func(ctx context.Context, in *databroker.DeleteRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) {
return nil, nil
},
get: func(ctx context.Context, in *databroker.GetRequest, opts ...grpc.CallOption) (*databroker.GetResponse, error) {
data, err := ptypes.MarshalAny(&session.Session{
Id: "SESSION_ID",
@ -259,13 +255,16 @@ func TestAuthenticate_SignOut(t *testing.T) {
return &databroker.GetResponse{
Record: &databroker.Record{
Version: "0001",
Version: 1,
Type: data.GetTypeUrl(),
Id: "SESSION_ID",
Data: data,
},
}, nil
},
put: func(ctx context.Context, in *databroker.PutRequest, opts ...grpc.CallOption) (*databroker.PutResponse, error) {
return nil, nil
},
},
directoryClient: new(mockDirectoryServiceClient),
}),
@ -368,8 +367,8 @@ func TestAuthenticate_OAuthCallback(t *testing.T) {
get: func(ctx context.Context, in *databroker.GetRequest, opts ...grpc.CallOption) (*databroker.GetResponse, error) {
return nil, fmt.Errorf("not implemented")
},
set: func(ctx context.Context, in *databroker.SetRequest, opts ...grpc.CallOption) (*databroker.SetResponse, error) {
return &databroker.SetResponse{Record: &databroker.Record{Data: in.Data}}, nil
put: func(ctx context.Context, in *databroker.PutRequest, opts ...grpc.CallOption) (*databroker.PutResponse, error) {
return nil, nil
},
},
directoryClient: new(mockDirectoryServiceClient),
@ -514,7 +513,7 @@ func TestAuthenticate_SessionValidatorMiddleware(t *testing.T) {
return &databroker.GetResponse{
Record: &databroker.Record{
Version: "0001",
Version: 1,
Type: data.GetTypeUrl(),
Id: "SESSION_ID",
Data: data,
@ -633,7 +632,7 @@ func TestAuthenticate_userInfo(t *testing.T) {
return &databroker.GetResponse{
Record: &databroker.Record{
Version: "0001",
Version: 1,
Type: data.GetTypeUrl(),
Id: "SESSION_ID",
Data: data,
@ -672,21 +671,16 @@ func TestAuthenticate_userInfo(t *testing.T) {
type mockDataBrokerServiceClient struct {
databroker.DataBrokerServiceClient
delete func(ctx context.Context, in *databroker.DeleteRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
get func(ctx context.Context, in *databroker.GetRequest, opts ...grpc.CallOption) (*databroker.GetResponse, error)
set func(ctx context.Context, in *databroker.SetRequest, opts ...grpc.CallOption) (*databroker.SetResponse, error)
}
func (m mockDataBrokerServiceClient) Delete(ctx context.Context, in *databroker.DeleteRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) {
return m.delete(ctx, in, opts...)
get func(ctx context.Context, in *databroker.GetRequest, opts ...grpc.CallOption) (*databroker.GetResponse, error)
put func(ctx context.Context, in *databroker.PutRequest, opts ...grpc.CallOption) (*databroker.PutResponse, error)
}
func (m mockDataBrokerServiceClient) Get(ctx context.Context, in *databroker.GetRequest, opts ...grpc.CallOption) (*databroker.GetResponse, error) {
return m.get(ctx, in, opts...)
}
func (m mockDataBrokerServiceClient) Set(ctx context.Context, in *databroker.SetRequest, opts ...grpc.CallOption) (*databroker.SetResponse, error) {
return m.set(ctx, in, opts...)
func (m mockDataBrokerServiceClient) Put(ctx context.Context, in *databroker.PutRequest, opts ...grpc.CallOption) (*databroker.PutResponse, error) {
return m.put(ctx, in, opts...)
}
type mockDirectoryServiceClient struct {
@ -729,7 +723,7 @@ func TestAuthenticate_SignOut_CSRF(t *testing.T) {
return &databroker.GetResponse{
Record: &databroker.Record{
Version: "0001",
Version: 1,
Type: data.GetTypeUrl(),
Id: "SESSION_ID",
Data: data,