3pp-mirror/pomerium
1
0
Fork 0
mirror of https://github.com/pomerium/pomerium.git synced 2025-05-31 09:57:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

databroker: add support for putting multiple records (#3291)

Browse source 
* databroker: add support for putting multiple records

* add OptimumPutRequestsFromRecords function

* replace GetAll with SyncLatest

* fix stream when there are no records
This commit is contained in:
Caleb Doxsey 2022-04-26 22:41:38 +00:00 • committed by GitHub
parent 343fa43ed4
commit f73c5c615f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 790 additions and 660 deletions
Download patch file Download diff file Expand all files Collapse all files

50
pkg/grpc/databroker/databroker.go
View file

@ -43,8 +43,12 @@ func Get(ctx context.Context, client DataBrokerServiceClient, object recordObjec
}
// Put puts a record into the databroker.
func Put(ctx context.Context, client DataBrokerServiceClient, object recordObject) (*PutResponse, error) {
return client.Put(ctx, &PutRequest{Record: NewRecord(object)})
func Put(ctx context.Context, client DataBrokerServiceClient, objects ...recordObject) (*PutResponse, error) {
records := make([]*Record, len(objects))
for i, object := range objects {
records[i] = NewRecord(object)
}
return client.Put(ctx, &PutRequest{Records: records})
}
// ApplyOffsetAndLimit applies the offset and limit to the list of records.
@ -95,3 +99,45 @@ loop:
return records, recordVersion, serverVersion, nil
}
// GetRecord gets the first record, or nil if there are none.
func (x *PutRequest) GetRecord() *Record {
records := x.GetRecords()
if len(records) == 0 {
return nil
}
return records[0]
}
// GetRecord gets the first record, or nil if there are none.
func (x *PutResponse) GetRecord() *Record {
records := x.GetRecords()
if len(records) == 0 {
return nil
}
return records[0]
}
// default is 4MB, but we'll do 1MB
const maxMessageSize = 1024 * 1024 * 1
// OptimumPutRequestsFromRecords creates one or more PutRequests from a slice of records.
// If the size of the request exceeds the max message size it will be split in half
// recursively until the requests are less than or equal to the max message size.
func OptimumPutRequestsFromRecords(records []*Record) []*PutRequest {
if len(records) <= 1 {
return []*PutRequest{{Records: records}}
}
req := &PutRequest{
Records: records,
}
if proto.Size(req) <= maxMessageSize {
return []*PutRequest{req}
}
return append(
OptimumPutRequestsFromRecords(records[:len(records)/2]),
OptimumPutRequestsFromRecords(records[len(records)/2:])...,
)
}