3pp-mirror/pomerium
1
0
Fork 0
mirror of https://github.com/pomerium/pomerium.git synced 2025-06-03 11:22:45 +02:00
Code Issues Projects Releases Packages Wiki Activity

databroker: add support for syncing by type (#3412)

Browse source 
* databroker: add support for syncing by type

* add type url, fix query
This commit is contained in:
Caleb Doxsey 2022-06-13 09:52:13 -06:00 • committed by GitHub
parent 2487e9af20
commit 45a29ea879
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 163 additions and 125 deletions
Download patch file Download diff file Expand all files Collapse all files

2
internal/databroker/server.go
View file

@ -308,7 +308,7 @@ func (srv *Server) Sync(req *databroker.SyncRequest, stream databroker.DataBroke
return err return err
} }
recordStream, err := backend.Sync(ctx, req.GetServerVersion(), req.GetRecordVersion()) recordStream, err := backend.Sync(ctx, req.GetType(), req.GetServerVersion(), req.GetRecordVersion())
if err != nil { if err != nil {
return err return err
} }

165
pkg/grpc/databroker/databroker.pb.go
View file

@ -685,6 +685,7 @@ type SyncRequest struct {
ServerVersion uint64 `protobuf:"varint,1,opt,name=server_version,json=serverVersion,proto3" json:"server_version,omitempty"` ServerVersion uint64 `protobuf:"varint,1,opt,name=server_version,json=serverVersion,proto3" json:"server_version,omitempty"`
RecordVersion uint64 `protobuf:"varint,2,opt,name=record_version,json=recordVersion,proto3" json:"record_version,omitempty"` RecordVersion uint64 `protobuf:"varint,2,opt,name=record_version,json=recordVersion,proto3" json:"record_version,omitempty"`
Type string `protobuf:"bytes,3,opt,name=type,proto3" json:"type,omitempty"`
} }
func (x *SyncRequest) Reset() { func (x *SyncRequest) Reset() {
@ -733,6 +734,13 @@ func (x *SyncRequest) GetRecordVersion() uint64 {
return 0 return 0
} }
func (x *SyncRequest) GetType() string {
if x != nil {
return x.Type
}
return ""
}
type SyncResponse struct { type SyncResponse struct {
state protoimpl.MessageState state protoimpl.MessageState
sizeCache protoimpl.SizeCache sizeCache protoimpl.SizeCache
@ -1219,90 +1227,91 @@ var file_databroker_proto_rawDesc = []byte{
0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2d, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2d, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18,
0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b,
0x65, 0x72, 0x2e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x65, 0x72, 0x2e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69,
0x6f, 0x6e, 0x73, 0x22, 0x5b, 0x0a, 0x0b, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x6f, 0x6e, 0x73, 0x22, 0x6f, 0x0a, 0x0b, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x76, 0x65, 0x72,
0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, 0x73, 0x65, 0x72, 0x76, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, 0x73, 0x65, 0x72, 0x76,
0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x63, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x63,
0x6f, 0x72, 0x64, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x6f, 0x72, 0x64, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28,
0x04, 0x52, 0x0d, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x04, 0x52, 0x0d, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e,
0x22, 0x3a, 0x0a, 0x0c, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04,
0x12, 0x2a, 0x0a, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x74, 0x79, 0x70, 0x65, 0x22, 0x3a, 0x0a, 0x0c, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70,
0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x01,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65,
0x72, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64,
0x22, 0x27, 0x0a, 0x11, 0x53, 0x79, 0x6e, 0x63, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x82, 0x01, 0x0a, 0x12, 0x53, 0x79,
0x6e, 0x63, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
0x12, 0x2c, 0x0a, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b,
0x32, 0x12, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x32, 0x12, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x52, 0x65,
0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x22, 0x27, 0x0a, 0x11, 0x63, 0x6f, 0x72, 0x64, 0x48, 0x00, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x12, 0x32,
0x53, 0x79, 0x6e, 0x63, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x0a, 0x08, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b,
0x74, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x32, 0x14, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x56, 0x65,
0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x82, 0x01, 0x0a, 0x12, 0x53, 0x79, 0x6e, 0x63, 0x4c, 0x61, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x08, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f,
0x74, 0x65, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2c, 0x0a, 0x06, 0x6e, 0x73, 0x42, 0x0a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x60,
0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x64, 0x0a, 0x13, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65,
0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20,
0x48, 0x00, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x12, 0x32, 0x0a, 0x08, 0x76, 0x65, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x35, 0x0a, 0x08, 0x64, 0x75, 0x72,
0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f,
0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75,
0x6e, 0x73, 0x48, 0x00, 0x52, 0x08, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x0a, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e,
0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x60, 0x0a, 0x13, 0x41, 0x63, 0x22, 0x26, 0x0a, 0x14, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65,
0x71, 0x75, 0x69, 0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01,
0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x39, 0x0a, 0x13, 0x52, 0x65, 0x6c, 0x65,
0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x35, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x61, 0x73, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e,
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x61, 0x6d, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x26, 0x0a, 0x14, 0x02, 0x69, 0x64, 0x22, 0x6e, 0x0a, 0x11, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x4c, 0x65, 0x61, 0x73,
0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65,
0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x0e, 0x0a, 0x02,
0x52, 0x02, 0x69, 0x64, 0x22, 0x39, 0x0a, 0x13, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x4c, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x35, 0x0a, 0x08,
0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19,
0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74,
0x6e, 0x0a, 0x11, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x69, 0x6f, 0x6e, 0x32, 0xfb, 0x04, 0x0a, 0x11, 0x44, 0x61, 0x74, 0x61, 0x42, 0x72, 0x6f, 0x6b,
0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x51, 0x0a, 0x0c, 0x41, 0x63, 0x71,
0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x75, 0x69, 0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x1f, 0x2e, 0x64, 0x61, 0x74, 0x61,
0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x35, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4c, 0x65,
0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x64, 0x61, 0x74,
0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4c,
0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x32, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x03,
0xfb, 0x04, 0x0a, 0x11, 0x44, 0x61, 0x74, 0x61, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x53, 0x65, 0x47, 0x65, 0x74, 0x12, 0x16, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72,
0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x51, 0x0a, 0x0c, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x64, 0x61,
0x4c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x1f, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70,
0x65, 0x72, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x16, 0x2e, 0x64, 0x61,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x50, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75,
0x6b, 0x65, 0x72, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x2e, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x05,
0x16, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x18, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x65, 0x72, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x19, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x51, 0x75, 0x65,
0x12, 0x36, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x16, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x0c, 0x52, 0x65,
0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x50, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x1f, 0x2e, 0x64, 0x61, 0x74,
0x17, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x50, 0x75, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x4c,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x05, 0x51, 0x75, 0x65, 0x72, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f,
0x79, 0x12, 0x18, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x51, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d,
0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x64, 0x61, 0x70, 0x74, 0x79, 0x12, 0x43, 0x0a, 0x0a, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x4c, 0x65, 0x61, 0x73,
0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x65, 0x12, 0x1d, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x52,
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x0c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x6e, 0x65, 0x77, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x1f, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62,
0x6b, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x4b, 0x0a, 0x0a, 0x53, 0x65, 0x74, 0x4f,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1d, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f,
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x6b, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65,
0x43, 0x0a, 0x0a, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x1d, 0x2e, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b,
0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73,
0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x04, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x17, 0x2e,
0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x52,
0x6d, 0x70, 0x74, 0x79, 0x12, 0x4b, 0x0a, 0x0a, 0x53, 0x65, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f,
0x6e, 0x73, 0x12, 0x1d, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x6b, 0x65, 0x72, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
0x53, 0x65, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x30, 0x01, 0x12, 0x4d, 0x0a, 0x0a, 0x53, 0x79, 0x6e, 0x63, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74,
0x74, 0x1a, 0x1e, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x53, 0x12, 0x1d, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x53, 0x79,
0x65, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x6e, 0x63, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
0x65, 0x12, 0x3b, 0x0a, 0x04, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x17, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x1e, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x53, 0x79, 0x6e,
0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x63, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30,
0x73, 0x74, 0x1a, 0x18, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x01, 0x42, 0x32, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f,
0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x4d, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2f, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75,
0x0a, 0x0a, 0x53, 0x79, 0x6e, 0x63, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x2e, 0x64, 0x6d, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x62,
0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x4c, 0x61, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x74, 0x65, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x64, 0x61,
0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x4c, 0x61, 0x74,
0x65, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x42, 0x32, 0x5a,
0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x6f, 0x6d, 0x65,
0x72, 0x69, 0x75, 0x6d, 0x2f, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2f, 0x70, 0x6b,
0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65,
0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
} }
var ( var (

1
pkg/grpc/databroker/databroker.proto
View file

@ -65,6 +65,7 @@ message SetOptionsResponse { Options options = 1; }
message SyncRequest { message SyncRequest {
uint64 server_version = 1; uint64 server_version = 1;
uint64 record_version = 2; uint64 record_version = 2;
string type = 3;
} }
message SyncResponse { Record record = 1; } message SyncResponse { Record record = 1; }

12
pkg/grpc/databroker/fast_forward.go
View file

@ -2,7 +2,6 @@ package databroker
import ( import (
"context" "context"
"time"
"github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/log"
) )
@ -33,18 +32,7 @@ func newFastForwardHandler(ctx context.Context, handler SyncerHandler) SyncerHan
} }
func (ff *fastForwardHandler) update(ctx context.Context, c *ffCmd) { func (ff *fastForwardHandler) update(ctx context.Context, c *ffCmd) {
versions := make([]uint64, len(c.records))
for i, r := range c.records {
versions[i] = r.Version
}
now := time.Now()
ff.handler.UpdateRecords(ctx, c.serverVersion, c.records) ff.handler.UpdateRecords(ctx, c.serverVersion, c.records)
log.Info(ctx).
Dur("elapsed", time.Since(now)).
Uint64("server_version", c.serverVersion).
Uints64("versions", versions).
Msg("UpdateRecords")
} }
func (ff *fastForwardHandler) runSelect(ctx context.Context) { func (ff *fastForwardHandler) runSelect(ctx context.Context) {

1
pkg/grpc/databroker/syncer.go
View file

@ -151,6 +151,7 @@ func (syncer *Syncer) sync(ctx context.Context) error {
stream, err := syncer.handler.GetDataBrokerServiceClient().Sync(ctx, &SyncRequest{ stream, err := syncer.handler.GetDataBrokerServiceClient().Sync(ctx, &SyncRequest{
ServerVersion: syncer.serverVersion, ServerVersion: syncer.serverVersion,
RecordVersion: syncer.recordVersion, RecordVersion: syncer.recordVersion,
Type: syncer.cfg.typeURL,
}) })
if err != nil { if err != nil {
log.Error(ctx).Err(err).Msg("error during sync") log.Error(ctx).Err(err).Msg("error during sync")

4
pkg/storage/encrypted.go
View file

@ -119,8 +119,8 @@ func (e *encryptedBackend) SetOptions(ctx context.Context, recordType string, op
return e.underlying.SetOptions(ctx, recordType, options) return e.underlying.SetOptions(ctx, recordType, options)
} }
func (e *encryptedBackend) Sync(ctx context.Context, serverVersion, recordVersion uint64) (RecordStream, error) { func (e *encryptedBackend) Sync(ctx context.Context, recordType string, serverVersion, recordVersion uint64) (RecordStream, error) {
stream, err := e.underlying.Sync(ctx, serverVersion, recordVersion) stream, err := e.underlying.Sync(ctx, recordType, serverVersion, recordVersion)
if err != nil { if err != nil {
return nil, err return nil, err
} }

16
pkg/storage/inmemory/backend.go
View file

@ -243,7 +243,7 @@ func (backend *Backend) SetOptions(_ context.Context, recordType string, options
} }
// Sync returns a record stream for any changes after recordVersion. // Sync returns a record stream for any changes after recordVersion.
func (backend *Backend) Sync(ctx context.Context, serverVersion, recordVersion uint64) (storage.RecordStream, error) { func (backend *Backend) Sync(ctx context.Context, recordType string, serverVersion, recordVersion uint64) (storage.RecordStream, error) {
backend.mu.RLock() backend.mu.RLock()
currentServerVersion := backend.serverVersion currentServerVersion := backend.serverVersion
backend.mu.RUnlock() backend.mu.RUnlock()
@ -251,7 +251,7 @@ func (backend *Backend) Sync(ctx context.Context, serverVersion, recordVersion u
if serverVersion != currentServerVersion { if serverVersion != currentServerVersion {
return nil, storage.ErrInvalidServerVersion return nil, storage.ErrInvalidServerVersion
} }
return newSyncRecordStream(ctx, backend, recordVersion), nil return newSyncRecordStream(ctx, backend, recordType, recordVersion), nil
} }
// SyncLatest returns a record stream for all the records. // SyncLatest returns a record stream for all the records.
@ -304,7 +304,7 @@ func (backend *Backend) enforceCapacity(recordType string) {
} }
} }
func (backend *Backend) getSince(version uint64) []*databroker.Record { func (backend *Backend) getSince(recordType string, version uint64) []*databroker.Record {
backend.mu.RLock() backend.mu.RLock()
defer backend.mu.RUnlock() defer backend.mu.RUnlock()
@ -322,6 +322,16 @@ func (backend *Backend) getSince(version uint64) []*databroker.Record {
} }
return true return true
}) })
if recordType != "" {
var filtered []*databroker.Record
for _, record := range records {
if record.GetType() == recordType {
filtered = append(filtered, record)
}
}
records = filtered
}
return records return records
} }

12
pkg/storage/inmemory/backend_test.go
View file

@ -87,7 +87,7 @@ func TestExpiry(t *testing.T) {
assert.NoError(t, err) assert.NoError(t, err)
assert.Equal(t, backend.serverVersion, sv) assert.Equal(t, backend.serverVersion, sv)
} }
stream, err := backend.Sync(ctx, backend.serverVersion, 0) stream, err := backend.Sync(ctx, "", backend.serverVersion, 0)
require.NoError(t, err) require.NoError(t, err)
var records []*databroker.Record var records []*databroker.Record
for stream.Next(false) { for stream.Next(false) {
@ -98,7 +98,7 @@ func TestExpiry(t *testing.T) {
backend.removeChangesBefore(time.Now().Add(time.Second)) backend.removeChangesBefore(time.Now().Add(time.Second))
stream, err = backend.Sync(ctx, backend.serverVersion, 0) stream, err = backend.Sync(ctx, "", backend.serverVersion, 0)
require.NoError(t, err) require.NoError(t, err)
records = nil records = nil
for stream.Next(false) { for stream.Next(false) {
@ -135,7 +135,7 @@ func TestStream(t *testing.T) {
backend := New() backend := New()
defer func() { _ = backend.Close() }() defer func() { _ = backend.Close() }()
stream, err := backend.Sync(ctx, backend.serverVersion, 0) stream, err := backend.Sync(ctx, "TYPE", backend.serverVersion, 0)
require.NoError(t, err) require.NoError(t, err)
defer func() { _ = stream.Close() }() defer func() { _ = stream.Close() }()
@ -167,7 +167,7 @@ func TestStreamClose(t *testing.T) {
ctx := context.Background() ctx := context.Background()
t.Run("by backend", func(t *testing.T) { t.Run("by backend", func(t *testing.T) {
backend := New() backend := New()
stream, err := backend.Sync(ctx, backend.serverVersion, 0) stream, err := backend.Sync(ctx, "", backend.serverVersion, 0)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, backend.Close()) require.NoError(t, backend.Close())
assert.False(t, stream.Next(true)) assert.False(t, stream.Next(true))
@ -175,7 +175,7 @@ func TestStreamClose(t *testing.T) {
}) })
t.Run("by stream", func(t *testing.T) { t.Run("by stream", func(t *testing.T) {
backend := New() backend := New()
stream, err := backend.Sync(ctx, backend.serverVersion, 0) stream, err := backend.Sync(ctx, "", backend.serverVersion, 0)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, stream.Close()) require.NoError(t, stream.Close())
assert.False(t, stream.Next(true)) assert.False(t, stream.Next(true))
@ -184,7 +184,7 @@ func TestStreamClose(t *testing.T) {
t.Run("by context", func(t *testing.T) { t.Run("by context", func(t *testing.T) {
ctx, cancel := context.WithCancel(ctx) ctx, cancel := context.WithCancel(ctx)
backend := New() backend := New()
stream, err := backend.Sync(ctx, backend.serverVersion, 0) stream, err := backend.Sync(ctx, "", backend.serverVersion, 0)
require.NoError(t, err) require.NoError(t, err)
cancel() cancel()
assert.False(t, stream.Next(true)) assert.False(t, stream.Next(true))

3
pkg/storage/inmemory/stream.go
View file

@ -54,6 +54,7 @@ func newSyncLatestRecordStream(
func newSyncRecordStream( func newSyncRecordStream(
ctx context.Context, ctx context.Context,
backend *Backend, backend *Backend,
recordType string,
recordVersion uint64, recordVersion uint64,
) storage.RecordStream { ) storage.RecordStream {
changed := backend.onChange.Bind() changed := backend.onChange.Bind()
@ -67,7 +68,7 @@ func newSyncRecordStream(
} }
for { for {
ready = backend.getSince(recordVersion) ready = backend.getSince(recordType, recordVersion)
if len(ready) > 0 { if len(ready) > 0 {
// records are sorted by version, // records are sorted by version,

3
pkg/storage/postgres/backend.go
View file

@ -210,6 +210,7 @@ func (backend *Backend) SetOptions(
// Sync syncs the records. // Sync syncs the records.
func (backend *Backend) Sync( func (backend *Backend) Sync(
ctx context.Context, ctx context.Context,
recordType string,
serverVersion, recordVersion uint64, serverVersion, recordVersion uint64,
) (storage.RecordStream, error) { ) (storage.RecordStream, error) {
// the original ctx will be used for the stream, this ctx used for pre-stream calls // the original ctx will be used for the stream, this ctx used for pre-stream calls
@ -224,7 +225,7 @@ func (backend *Backend) Sync(
return nil, storage.ErrInvalidServerVersion return nil, storage.ErrInvalidServerVersion
} }
return newChangedRecordStream(ctx, backend, recordVersion), nil return newChangedRecordStream(ctx, backend, recordType, recordVersion), nil
} }
// SyncLatest syncs the latest version of each record. // SyncLatest syncs the latest version of each record.

4
pkg/storage/postgres/backend_test.go
View file

@ -111,11 +111,11 @@ func TestBackend(t *testing.T) {
}) })
t.Run("changed", func(t *testing.T) { t.Run("changed", func(t *testing.T) {
serverVersion, recordVersion, stream, err := backend.SyncLatest(ctx, "", nil) serverVersion, recordVersion, stream, err := backend.SyncLatest(ctx, "sync-test", nil)
require.NoError(t, err) require.NoError(t, err)
assert.NoError(t, stream.Close()) assert.NoError(t, stream.Close())
stream, err = backend.Sync(ctx, serverVersion, recordVersion) stream, err = backend.Sync(ctx, "", serverVersion, recordVersion)
require.NoError(t, err) require.NoError(t, err)
defer stream.Close() defer stream.Close()

23
pkg/storage/postgres/postgres.go
View file

@ -4,6 +4,7 @@ package postgres
import ( import (
"context" "context"
"errors" "errors"
"fmt"
"time" "time"
"github.com/jackc/pgconn" "github.com/jackc/pgconn"
@ -78,27 +79,37 @@ func getLatestRecordVersion(ctx context.Context, q querier) (recordVersion uint6
return recordVersion, err return recordVersion, err
} }
func getNextChangedRecord(ctx context.Context, q querier, afterRecordVersion uint64) (*databroker.Record, error) { func getNextChangedRecord(ctx context.Context, q querier, recordType string, afterRecordVersion uint64) (*databroker.Record, error) {
var recordType, recordID string var recordID string
var version uint64 var version uint64
var data pgtype.JSONB var data pgtype.JSONB
var modifiedAt pgtype.Timestamptz var modifiedAt pgtype.Timestamptz
var deletedAt pgtype.Timestamptz var deletedAt pgtype.Timestamptz
err := q.QueryRow(ctx, ` query := `
SELECT type, id, version, data, modified_at, deleted_at SELECT type, id, version, data, modified_at, deleted_at
FROM ` + schemaName + `.` + recordChangesTableName + ` FROM ` + schemaName + `.` + recordChangesTableName + `
WHERE version > $1 WHERE version > $1
`, afterRecordVersion).Scan(&recordType, &recordID, &version, &data, &modifiedAt, &deletedAt) `
args := []any{afterRecordVersion}
if recordType != "" {
query += ` AND type = $2`
args = append(args, recordType)
}
query += `
ORDER BY version ASC
LIMIT 1
`
err := q.QueryRow(ctx, query, args...).Scan(&recordType, &recordID, &version, &data, &modifiedAt, &deletedAt)
if isNotFound(err) { if isNotFound(err) {
return nil, storage.ErrNotFound return nil, storage.ErrNotFound
} else if err != nil { } else if err != nil {
return nil, err return nil, fmt.Errorf("error querying next changed record: %w", err)
} }
var any anypb.Any var any anypb.Any
err = protojson.Unmarshal(data.Bytes, &any) err = protojson.Unmarshal(data.Bytes, &any)
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("error unmarshaling changed record data: %w", err)
} }
return &databroker.Record{ return &databroker.Record{

4
pkg/storage/postgres/stream.go
View file

@ -82,6 +82,7 @@ const watchPollInterval = 30 * time.Second
type changedRecordStream struct { type changedRecordStream struct {
backend *Backend backend *Backend
recordType string
recordVersion uint64 recordVersion uint64
ctx context.Context ctx context.Context
@ -95,10 +96,12 @@ type changedRecordStream struct {
func newChangedRecordStream( func newChangedRecordStream(
ctx context.Context, ctx context.Context,
backend *Backend, backend *Backend,
recordType string,
recordVersion uint64, recordVersion uint64,
) storage.RecordStream { ) storage.RecordStream {
stream := &changedRecordStream{ stream := &changedRecordStream{
backend: backend, backend: backend,
recordType: recordType,
recordVersion: recordVersion, recordVersion: recordVersion,
ticker: time.NewTicker(watchPollInterval), ticker: time.NewTicker(watchPollInterval),
changed: backend.onChange.Bind(), changed: backend.onChange.Bind(),
@ -129,6 +132,7 @@ func (stream *changedRecordStream) Next(block bool) bool {
stream.record, stream.err = getNextChangedRecord( stream.record, stream.err = getNextChangedRecord(
stream.ctx, stream.ctx,
pool, pool,
stream.recordType,
stream.recordVersion, stream.recordVersion,
) )
if isNotFound(stream.err) { if isNotFound(stream.err) {

8
pkg/storage/redis/redis.go
View file

@ -248,8 +248,12 @@ func (backend *Backend) SetOptions(ctx context.Context, recordType string, optio
} }
// Sync returns a record stream of any records changed after the specified recordVersion. // Sync returns a record stream of any records changed after the specified recordVersion.
func (backend *Backend) Sync(ctx context.Context, serverVersion, recordVersion uint64) (storage.RecordStream, error) { func (backend *Backend) Sync(
return newSyncRecordStream(ctx, backend, serverVersion, recordVersion), nil ctx context.Context,
recordType string,
serverVersion, recordVersion uint64,
) (storage.RecordStream, error) {
return newSyncRecordStream(ctx, backend, recordType, serverVersion, recordVersion), nil
} }
// SyncLatest returns a record stream of all the records. Some records may be returned twice if the are updated while the // SyncLatest returns a record stream of all the records. Some records may be returned twice if the are updated while the

4
pkg/storage/redis/redis_test.go
View file

@ -190,7 +190,7 @@ func TestExpiry(t *testing.T) {
}}) }})
assert.NoError(t, err) assert.NoError(t, err)
} }
stream, err := backend.Sync(ctx, serverVersion, 0) stream, err := backend.Sync(ctx, "TYPE", serverVersion, 0)
require.NoError(t, err) require.NoError(t, err)
var records []*databroker.Record var records []*databroker.Record
for stream.Next(false) { for stream.Next(false) {
@ -201,7 +201,7 @@ func TestExpiry(t *testing.T) {
backend.removeChangesBefore(ctx, time.Now().Add(time.Second)) backend.removeChangesBefore(ctx, time.Now().Add(time.Second))
stream, err = backend.Sync(ctx, serverVersion, 0) stream, err = backend.Sync(ctx, "TYPE", serverVersion, 0)
require.NoError(t, err) require.NoError(t, err)
records = nil records = nil
for stream.Next(false) { for stream.Next(false) {

18
pkg/storage/redis/stream.go
View file

@ -17,6 +17,7 @@ import (
func newSyncRecordStream( func newSyncRecordStream(
ctx context.Context, ctx context.Context,
backend *Backend, backend *Backend,
recordType string,
serverVersion uint64, serverVersion uint64,
recordVersion uint64, recordVersion uint64,
) storage.RecordStream { ) storage.RecordStream {
@ -36,7 +37,7 @@ func newSyncRecordStream(
return nil, storage.ErrInvalidServerVersion return nil, storage.ErrInvalidServerVersion
} }
record, err := nextChangedRecord(ctx, backend, &recordVersion) record, err := nextChangedRecord(ctx, backend, recordType, &recordVersion)
if err == nil { if err == nil {
return record, nil return record, nil
} else if !errors.Is(err, storage.ErrStreamDone) { } else if !errors.Is(err, storage.ErrStreamDone) {
@ -135,7 +136,7 @@ func nextScannedRecords(ctx context.Context, backend *Backend, cursor *uint64) (
return records, nil return records, nil
} }
func nextChangedRecord(ctx context.Context, backend *Backend, recordVersion *uint64) (*databroker.Record, error) { func nextChangedRecord(ctx context.Context, backend *Backend, recordType string, recordVersion *uint64) (*databroker.Record, error) {
for { for {
cmd := backend.client.ZRangeByScore(ctx, changesSetKey, &redis.ZRangeBy{ cmd := backend.client.ZRangeByScore(ctx, changesSetKey, &redis.ZRangeBy{
Min: fmt.Sprintf("(%d", *recordVersion), Min: fmt.Sprintf("(%d", *recordVersion),
@ -155,10 +156,17 @@ func nextChangedRecord(ctx context.Context, backend *Backend, recordVersion *uin
result := results[0] result := results[0]
var record databroker.Record var record databroker.Record
err = proto.Unmarshal([]byte(result), &record) err = proto.Unmarshal([]byte(result), &record)
if err != nil {
log.Warn(ctx).Err(err).Msg("redis: invalid record detected")
*recordVersion++ *recordVersion++
if err == nil { continue
}
*recordVersion = record.GetVersion()
if recordType != "" && record.GetType() != recordType {
continue
}
return &record, nil return &record, nil
} }
log.Warn(ctx).Err(err).Msg("redis: invalid record detected")
}
} }

2
pkg/storage/storage.go
View file

@ -38,7 +38,7 @@ type Backend interface {
// SetOptions sets the options for a type. // SetOptions sets the options for a type.
SetOptions(ctx context.Context, recordType string, options *databroker.Options) error SetOptions(ctx context.Context, recordType string, options *databroker.Options) error
// Sync syncs record changes after the specified version. // Sync syncs record changes after the specified version.
Sync(ctx context.Context, serverVersion, recordVersion uint64) (RecordStream, error) Sync(ctx context.Context, recordType string, serverVersion, recordVersion uint64) (RecordStream, error)
// SyncLatest syncs all the records. // SyncLatest syncs all the records.
SyncLatest(ctx context.Context, recordType string, filter FilterExpression) (serverVersion, recordVersion uint64, stream RecordStream, err error) SyncLatest(ctx context.Context, recordType string, filter FilterExpression) (serverVersion, recordVersion uint64, stream RecordStream, err error)
} }