mirror of
https://github.com/pomerium/pomerium.git
synced 2025-08-03 16:59:22 +02:00
databroker: refactor databroker to sync all changes (#1879)
* refactor backend, implement encrypted store * refactor in-memory store * wip * wip * wip * add syncer test * fix redis expiry * fix linting issues * fix test by skipping non-config records * fix backoff import * fix init issues * fix query * wait for initial sync before starting directory sync * add type to SyncLatest * add more log messages, fix deadlock in in-memory store, always return server version from SyncLatest * update sync types and tests * add redis tests * skip macos in github actions * add comments to proto * split getBackend into separate methods * handle errors in initVersion * return different error for not found vs other errors in get * use exponential backoff for redis transaction retry * rename raw to result * use context instead of close channel * store type urls as constants in databroker * use timestampb instead of ptypes * fix group merging not waiting * change locked names * update GetAll to return latest record version * add method to grpcutil to get the type url for a protobuf type
This commit is contained in:
Caleb Doxsey
GitHub
parent
b1871b0f2e
commit
5d60cff21e
66 changed files with 2762 additions and 2871 deletions
|
|
@ -1,3 +1,4 @@
|
|||
// Package databroker contains the databroker service.
|
||||
package databroker
|
||||
|
||||
import (
|
||||
|
|
@ -5,8 +6,6 @@ import (
|
|||
"encoding/base64"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/golang/protobuf/ptypes/empty"
|
||||
|
||||
"github.com/pomerium/pomerium/config"
|
||||
"github.com/pomerium/pomerium/internal/databroker"
|
||||
databrokerpb "github.com/pomerium/pomerium/pkg/grpc/databroker"
|
||||
|
|
@ -52,13 +51,6 @@ func (srv *dataBrokerServer) setKey(cfg *config.Config) {
|
|||
srv.sharedKey.Store(bs)
|
||||
}
|
||||
|
||||
func (srv *dataBrokerServer) Delete(ctx context.Context, req *databrokerpb.DeleteRequest) (*empty.Empty, error) {
|
||||
if err := grpcutil.RequireSignedJWT(ctx, srv.sharedKey.Load().([]byte)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return srv.server.Delete(ctx, req)
|
||||
}
|
||||
|
||||
func (srv *dataBrokerServer) Get(ctx context.Context, req *databrokerpb.GetRequest) (*databrokerpb.GetResponse, error) {
|
||||
if err := grpcutil.RequireSignedJWT(ctx, srv.sharedKey.Load().([]byte)); err != nil {
|
||||
return nil, err
|
||||
|
|
@ -66,13 +58,6 @@ func (srv *dataBrokerServer) Get(ctx context.Context, req *databrokerpb.GetReque
|
|||
return srv.server.Get(ctx, req)
|
||||
}
|
||||
|
||||
func (srv *dataBrokerServer) GetAll(ctx context.Context, req *databrokerpb.GetAllRequest) (*databrokerpb.GetAllResponse, error) {
|
||||
if err := grpcutil.RequireSignedJWT(ctx, srv.sharedKey.Load().([]byte)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return srv.server.GetAll(ctx, req)
|
||||
}
|
||||
|
||||
func (srv *dataBrokerServer) Query(ctx context.Context, req *databrokerpb.QueryRequest) (*databrokerpb.QueryResponse, error) {
|
||||
if err := grpcutil.RequireSignedJWT(ctx, srv.sharedKey.Load().([]byte)); err != nil {
|
||||
return nil, err
|
||||
|
|
@ -80,11 +65,11 @@ func (srv *dataBrokerServer) Query(ctx context.Context, req *databrokerpb.QueryR
|
|||
return srv.server.Query(ctx, req)
|
||||
}
|
||||
|
||||
func (srv *dataBrokerServer) Set(ctx context.Context, req *databrokerpb.SetRequest) (*databrokerpb.SetResponse, error) {
|
||||
func (srv *dataBrokerServer) Put(ctx context.Context, req *databrokerpb.PutRequest) (*databrokerpb.PutResponse, error) {
|
||||
if err := grpcutil.RequireSignedJWT(ctx, srv.sharedKey.Load().([]byte)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return srv.server.Set(ctx, req)
|
||||
return srv.server.Put(ctx, req)
|
||||
}
|
||||
|
||||
func (srv *dataBrokerServer) Sync(req *databrokerpb.SyncRequest, stream databrokerpb.DataBrokerService_SyncServer) error {
|
||||
|
|
@ -94,16 +79,9 @@ func (srv *dataBrokerServer) Sync(req *databrokerpb.SyncRequest, stream databrok
|
|||
return srv.server.Sync(req, stream)
|
||||
}
|
||||
|
||||
func (srv *dataBrokerServer) GetTypes(ctx context.Context, req *empty.Empty) (*databrokerpb.GetTypesResponse, error) {
|
||||
if err := grpcutil.RequireSignedJWT(ctx, srv.sharedKey.Load().([]byte)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return srv.server.GetTypes(ctx, req)
|
||||
}
|
||||
|
||||
func (srv *dataBrokerServer) SyncTypes(req *empty.Empty, stream databrokerpb.DataBrokerService_SyncTypesServer) error {
|
||||
func (srv *dataBrokerServer) SyncLatest(req *databrokerpb.SyncLatestRequest, stream databrokerpb.DataBrokerService_SyncLatestServer) error {
|
||||
if err := grpcutil.RequireSignedJWT(stream.Context(), srv.sharedKey.Load().([]byte)); err != nil {
|
||||
return err
|
||||
}
|
||||
return srv.server.SyncTypes(req, stream)
|
||||
return srv.server.SyncLatest(req, stream)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,9 +7,10 @@ import (
|
|||
"testing"
|
||||
|
||||
"github.com/golang/protobuf/ptypes"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
"google.golang.org/grpc/test/bufconn"
|
||||
|
||||
internal_databroker "github.com/pomerium/pomerium/internal/databroker"
|
||||
|
|
@ -51,44 +52,44 @@ func TestServerSync(t *testing.T) {
|
|||
any, _ := ptypes.MarshalAny(new(user.User))
|
||||
numRecords := 200
|
||||
|
||||
var serverVersion uint64
|
||||
|
||||
for i := 0; i < numRecords; i++ {
|
||||
c.Set(ctx, &databroker.SetRequest{Type: any.TypeUrl, Id: strconv.Itoa(i), Data: any})
|
||||
res, err := c.Put(ctx, &databroker.PutRequest{
|
||||
Record: &databroker.Record{
|
||||
Type: any.TypeUrl,
|
||||
Id: strconv.Itoa(i),
|
||||
Data: any,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
serverVersion = res.GetServerVersion()
|
||||
}
|
||||
|
||||
t.Run("Sync ok", func(t *testing.T) {
|
||||
client, _ := c.Sync(ctx, &databroker.SyncRequest{Type: any.GetTypeUrl()})
|
||||
client, _ := c.Sync(ctx, &databroker.SyncRequest{
|
||||
ServerVersion: serverVersion,
|
||||
})
|
||||
count := 0
|
||||
for {
|
||||
res, err := client.Recv()
|
||||
_, err := client.Recv()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
count += len(res.Records)
|
||||
count++
|
||||
if count == numRecords {
|
||||
break
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("Error occurred while syncing", func(t *testing.T) {
|
||||
ctx, cancelFunc := context.WithCancel(ctx)
|
||||
defer cancelFunc()
|
||||
|
||||
client, _ := c.Sync(ctx, &databroker.SyncRequest{Type: any.GetTypeUrl()})
|
||||
count := 0
|
||||
numRecordsWanted := 100
|
||||
cancelFuncCalled := false
|
||||
for {
|
||||
res, err := client.Recv()
|
||||
if err != nil {
|
||||
assert.True(t, cancelFuncCalled)
|
||||
break
|
||||
}
|
||||
count += len(res.Records)
|
||||
if count == numRecordsWanted {
|
||||
cancelFunc()
|
||||
cancelFuncCalled = true
|
||||
}
|
||||
}
|
||||
t.Run("Aborted", func(t *testing.T) {
|
||||
client, err := c.Sync(ctx, &databroker.SyncRequest{
|
||||
ServerVersion: 0,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
_, err = client.Recv()
|
||||
require.Error(t, err)
|
||||
require.Equal(t, codes.Aborted, status.Code(err))
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -104,19 +105,25 @@ func BenchmarkSync(b *testing.B) {
|
|||
numRecords := 10000
|
||||
|
||||
for i := 0; i < numRecords; i++ {
|
||||
c.Set(ctx, &databroker.SetRequest{Type: any.TypeUrl, Id: strconv.Itoa(i), Data: any})
|
||||
_, _ = c.Put(ctx, &databroker.PutRequest{
|
||||
Record: &databroker.Record{
|
||||
Type: any.TypeUrl,
|
||||
Id: strconv.Itoa(i),
|
||||
Data: any,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
client, _ := c.Sync(ctx, &databroker.SyncRequest{Type: any.GetTypeUrl()})
|
||||
client, _ := c.Sync(ctx, &databroker.SyncRequest{})
|
||||
count := 0
|
||||
for {
|
||||
res, err := client.Recv()
|
||||
_, err := client.Recv()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
count += len(res.Records)
|
||||
count++
|
||||
if count == numRecords {
|
||||
break
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,10 +31,12 @@ func (c *DataBroker) RefreshUser(ctx context.Context, req *directory.RefreshUser
|
|||
return nil, err
|
||||
}
|
||||
|
||||
_, err = c.dataBrokerServer.Set(ctx, &databroker.SetRequest{
|
||||
Type: any.GetTypeUrl(),
|
||||
Id: u.GetId(),
|
||||
Data: any,
|
||||
_, err = c.dataBrokerServer.Put(ctx, &databroker.PutRequest{
|
||||
Record: &databroker.Record{
|
||||
Type: any.GetTypeUrl(),
|
||||
Id: u.GetId(),
|
||||
Data: any,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue