mirror of
https://github.com/pomerium/pomerium.git
synced 2025-05-30 17:37:25 +02:00
databroker: add support for syncing by type (#3412)
* databroker: add support for syncing by type * add type url, fix query
This commit is contained in:
Caleb Doxsey
GitHub
parent
2487e9af20
commit
45a29ea879
17 changed files with 163 additions and 125 deletions
|
|
@ -308,7 +308,7 @@ func (srv *Server) Sync(req *databroker.SyncRequest, stream databroker.DataBroke
|
|||
return err
|
||||
}
|
||||
|
||||
recordStream, err := backend.Sync(ctx, req.GetServerVersion(), req.GetRecordVersion())
|
||||
recordStream, err := backend.Sync(ctx, req.GetType(), req.GetServerVersion(), req.GetRecordVersion())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -685,6 +685,7 @@ type SyncRequest struct {
|
|||
|
||||
ServerVersion uint64 `protobuf:"varint,1,opt,name=server_version,json=serverVersion,proto3" json:"server_version,omitempty"`
|
||||
RecordVersion uint64 `protobuf:"varint,2,opt,name=record_version,json=recordVersion,proto3" json:"record_version,omitempty"`
|
||||
Type string `protobuf:"bytes,3,opt,name=type,proto3" json:"type,omitempty"`
|
||||
}
|
||||
|
||||
func (x *SyncRequest) Reset() {
|
||||
|
|
@ -733,6 +734,13 @@ func (x *SyncRequest) GetRecordVersion() uint64 {
|
|||
return 0
|
||||
}
|
||||
|
||||
func (x *SyncRequest) GetType() string {
|
||||
if x != nil {
|
||||
return x.Type
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type SyncResponse struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
|
|
@ -1219,90 +1227,91 @@ var file_databroker_proto_rawDesc = []byte{
|
|||
0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2d, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18,
|
||||
0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b,
|
||||
0x65, 0x72, 0x2e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69,
|
||||
0x6f, 0x6e, 0x73, 0x22, 0x5b, 0x0a, 0x0b, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65,
|
||||
0x6f, 0x6e, 0x73, 0x22, 0x6f, 0x0a, 0x0b, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65,
|
||||
0x73, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x76, 0x65, 0x72,
|
||||
0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, 0x73, 0x65, 0x72, 0x76,
|
||||
0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x63,
|
||||
0x6f, 0x72, 0x64, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28,
|
||||
0x04, 0x52, 0x0d, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e,
|
||||
0x22, 0x3a, 0x0a, 0x0c, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
|
||||
0x12, 0x2a, 0x0a, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b,
|
||||
0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04,
|
||||
0x74, 0x79, 0x70, 0x65, 0x22, 0x3a, 0x0a, 0x0c, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70,
|
||||
0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x01,
|
||||
0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65,
|
||||
0x72, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64,
|
||||
0x22, 0x27, 0x0a, 0x11, 0x53, 0x79, 0x6e, 0x63, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x52, 0x65,
|
||||
0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20,
|
||||
0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x82, 0x01, 0x0a, 0x12, 0x53, 0x79,
|
||||
0x6e, 0x63, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
|
||||
0x12, 0x2c, 0x0a, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b,
|
||||
0x32, 0x12, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x52, 0x65,
|
||||
0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x22, 0x27, 0x0a, 0x11,
|
||||
0x53, 0x79, 0x6e, 0x63, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
|
||||
0x74, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
|
||||
0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x82, 0x01, 0x0a, 0x12, 0x53, 0x79, 0x6e, 0x63, 0x4c, 0x61,
|
||||
0x74, 0x65, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2c, 0x0a, 0x06,
|
||||
0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x64,
|
||||
0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64,
|
||||
0x48, 0x00, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x12, 0x32, 0x0a, 0x08, 0x76, 0x65,
|
||||
0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64,
|
||||
0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f,
|
||||
0x6e, 0x73, 0x48, 0x00, 0x52, 0x08, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x0a,
|
||||
0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x60, 0x0a, 0x13, 0x41, 0x63,
|
||||
0x71, 0x75, 0x69, 0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
|
||||
0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
|
||||
0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x35, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f,
|
||||
0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
|
||||
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69,
|
||||
0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x26, 0x0a, 0x14,
|
||||
0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70,
|
||||
0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
|
||||
0x52, 0x02, 0x69, 0x64, 0x22, 0x39, 0x0a, 0x13, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x4c,
|
||||
0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e,
|
||||
0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12,
|
||||
0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22,
|
||||
0x6e, 0x0a, 0x11, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71,
|
||||
0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01,
|
||||
0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02,
|
||||
0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x35, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61,
|
||||
0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f,
|
||||
0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72,
|
||||
0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x32,
|
||||
0xfb, 0x04, 0x0a, 0x11, 0x44, 0x61, 0x74, 0x61, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x53, 0x65,
|
||||
0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x51, 0x0a, 0x0c, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65,
|
||||
0x4c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x1f, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b,
|
||||
0x65, 0x72, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52,
|
||||
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f,
|
||||
0x6b, 0x65, 0x72, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65,
|
||||
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12,
|
||||
0x16, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74,
|
||||
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72,
|
||||
0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
|
||||
0x12, 0x36, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x16, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72,
|
||||
0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x50, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
|
||||
0x17, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x50, 0x75, 0x74,
|
||||
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x05, 0x51, 0x75, 0x65, 0x72,
|
||||
0x79, 0x12, 0x18, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x51,
|
||||
0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x64, 0x61,
|
||||
0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65,
|
||||
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x0c, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73,
|
||||
0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x1f, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f,
|
||||
0x6b, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65,
|
||||
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
|
||||
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12,
|
||||
0x43, 0x0a, 0x0a, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x1d, 0x2e,
|
||||
0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x6e, 0x65, 0x77,
|
||||
0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67,
|
||||
0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45,
|
||||
0x6d, 0x70, 0x74, 0x79, 0x12, 0x4b, 0x0a, 0x0a, 0x53, 0x65, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f,
|
||||
0x6e, 0x73, 0x12, 0x1d, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e,
|
||||
0x53, 0x65, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
|
||||
0x74, 0x1a, 0x1e, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x53,
|
||||
0x65, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
|
||||
0x65, 0x12, 0x3b, 0x0a, 0x04, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x17, 0x2e, 0x64, 0x61, 0x74, 0x61,
|
||||
0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65,
|
||||
0x73, 0x74, 0x1a, 0x18, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e,
|
||||
0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x4d,
|
||||
0x0a, 0x0a, 0x53, 0x79, 0x6e, 0x63, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x2e, 0x64,
|
||||
0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x4c, 0x61,
|
||||
0x74, 0x65, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x64, 0x61,
|
||||
0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x4c, 0x61, 0x74,
|
||||
0x65, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x42, 0x32, 0x5a,
|
||||
0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x6f, 0x6d, 0x65,
|
||||
0x72, 0x69, 0x75, 0x6d, 0x2f, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2f, 0x70, 0x6b,
|
||||
0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65,
|
||||
0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
0x63, 0x6f, 0x72, 0x64, 0x48, 0x00, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x12, 0x32,
|
||||
0x0a, 0x08, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b,
|
||||
0x32, 0x14, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x56, 0x65,
|
||||
0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x08, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f,
|
||||
0x6e, 0x73, 0x42, 0x0a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x60,
|
||||
0x0a, 0x13, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65,
|
||||
0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20,
|
||||
0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x35, 0x0a, 0x08, 0x64, 0x75, 0x72,
|
||||
0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f,
|
||||
0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75,
|
||||
0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e,
|
||||
0x22, 0x26, 0x0a, 0x14, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65,
|
||||
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01,
|
||||
0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x39, 0x0a, 0x13, 0x52, 0x65, 0x6c, 0x65,
|
||||
0x61, 0x73, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
|
||||
0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e,
|
||||
0x61, 0x6d, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
|
||||
0x02, 0x69, 0x64, 0x22, 0x6e, 0x0a, 0x11, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x4c, 0x65, 0x61, 0x73,
|
||||
0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65,
|
||||
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x0e, 0x0a, 0x02,
|
||||
0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x35, 0x0a, 0x08,
|
||||
0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19,
|
||||
0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
|
||||
0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74,
|
||||
0x69, 0x6f, 0x6e, 0x32, 0xfb, 0x04, 0x0a, 0x11, 0x44, 0x61, 0x74, 0x61, 0x42, 0x72, 0x6f, 0x6b,
|
||||
0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x51, 0x0a, 0x0c, 0x41, 0x63, 0x71,
|
||||
0x75, 0x69, 0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x1f, 0x2e, 0x64, 0x61, 0x74, 0x61,
|
||||
0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4c, 0x65,
|
||||
0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x64, 0x61, 0x74,
|
||||
0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4c,
|
||||
0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x03,
|
||||
0x47, 0x65, 0x74, 0x12, 0x16, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72,
|
||||
0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x64, 0x61,
|
||||
0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70,
|
||||
0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x16, 0x2e, 0x64, 0x61,
|
||||
0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x50, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75,
|
||||
0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72,
|
||||
0x2e, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x05,
|
||||
0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x18, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b,
|
||||
0x65, 0x72, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
|
||||
0x19, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x51, 0x75, 0x65,
|
||||
0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x0c, 0x52, 0x65,
|
||||
0x6c, 0x65, 0x61, 0x73, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x1f, 0x2e, 0x64, 0x61, 0x74,
|
||||
0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x4c,
|
||||
0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f,
|
||||
0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d,
|
||||
0x70, 0x74, 0x79, 0x12, 0x43, 0x0a, 0x0a, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x4c, 0x65, 0x61, 0x73,
|
||||
0x65, 0x12, 0x1d, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x52,
|
||||
0x65, 0x6e, 0x65, 0x77, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
|
||||
0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62,
|
||||
0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x4b, 0x0a, 0x0a, 0x53, 0x65, 0x74, 0x4f,
|
||||
0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1d, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f,
|
||||
0x6b, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65,
|
||||
0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b,
|
||||
0x65, 0x72, 0x2e, 0x53, 0x65, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73,
|
||||
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x04, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x17, 0x2e,
|
||||
0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x52,
|
||||
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f,
|
||||
0x6b, 0x65, 0x72, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
|
||||
0x30, 0x01, 0x12, 0x4d, 0x0a, 0x0a, 0x53, 0x79, 0x6e, 0x63, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74,
|
||||
0x12, 0x1d, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x53, 0x79,
|
||||
0x6e, 0x63, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
|
||||
0x1e, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x53, 0x79, 0x6e,
|
||||
0x63, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30,
|
||||
0x01, 0x42, 0x32, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f,
|
||||
0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75, 0x6d, 0x2f, 0x70, 0x6f, 0x6d, 0x65, 0x72, 0x69, 0x75,
|
||||
0x6d, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x62,
|
||||
0x72, 0x6f, 0x6b, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -65,6 +65,7 @@ message SetOptionsResponse { Options options = 1; }
|
|||
message SyncRequest {
|
||||
uint64 server_version = 1;
|
||||
uint64 record_version = 2;
|
||||
string type = 3;
|
||||
}
|
||||
message SyncResponse { Record record = 1; }
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ package databroker
|
|||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/pomerium/pomerium/internal/log"
|
||||
)
|
||||
|
|
@ -33,18 +32,7 @@ func newFastForwardHandler(ctx context.Context, handler SyncerHandler) SyncerHan
|
|||
}
|
||||
|
||||
func (ff *fastForwardHandler) update(ctx context.Context, c *ffCmd) {
|
||||
versions := make([]uint64, len(c.records))
|
||||
for i, r := range c.records {
|
||||
versions[i] = r.Version
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
ff.handler.UpdateRecords(ctx, c.serverVersion, c.records)
|
||||
log.Info(ctx).
|
||||
Dur("elapsed", time.Since(now)).
|
||||
Uint64("server_version", c.serverVersion).
|
||||
Uints64("versions", versions).
|
||||
Msg("UpdateRecords")
|
||||
}
|
||||
|
||||
func (ff *fastForwardHandler) runSelect(ctx context.Context) {
|
||||
|
|
|
|||
|
|
@ -151,6 +151,7 @@ func (syncer *Syncer) sync(ctx context.Context) error {
|
|||
stream, err := syncer.handler.GetDataBrokerServiceClient().Sync(ctx, &SyncRequest{
|
||||
ServerVersion: syncer.serverVersion,
|
||||
RecordVersion: syncer.recordVersion,
|
||||
Type: syncer.cfg.typeURL,
|
||||
})
|
||||
if err != nil {
|
||||
log.Error(ctx).Err(err).Msg("error during sync")
|
||||
|
|
|
|||
|
|
@ -119,8 +119,8 @@ func (e *encryptedBackend) SetOptions(ctx context.Context, recordType string, op
|
|||
return e.underlying.SetOptions(ctx, recordType, options)
|
||||
}
|
||||
|
||||
func (e *encryptedBackend) Sync(ctx context.Context, serverVersion, recordVersion uint64) (RecordStream, error) {
|
||||
stream, err := e.underlying.Sync(ctx, serverVersion, recordVersion)
|
||||
func (e *encryptedBackend) Sync(ctx context.Context, recordType string, serverVersion, recordVersion uint64) (RecordStream, error) {
|
||||
stream, err := e.underlying.Sync(ctx, recordType, serverVersion, recordVersion)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -243,7 +243,7 @@ func (backend *Backend) SetOptions(_ context.Context, recordType string, options
|
|||
}
|
||||
|
||||
// Sync returns a record stream for any changes after recordVersion.
|
||||
func (backend *Backend) Sync(ctx context.Context, serverVersion, recordVersion uint64) (storage.RecordStream, error) {
|
||||
func (backend *Backend) Sync(ctx context.Context, recordType string, serverVersion, recordVersion uint64) (storage.RecordStream, error) {
|
||||
backend.mu.RLock()
|
||||
currentServerVersion := backend.serverVersion
|
||||
backend.mu.RUnlock()
|
||||
|
|
@ -251,7 +251,7 @@ func (backend *Backend) Sync(ctx context.Context, serverVersion, recordVersion u
|
|||
if serverVersion != currentServerVersion {
|
||||
return nil, storage.ErrInvalidServerVersion
|
||||
}
|
||||
return newSyncRecordStream(ctx, backend, recordVersion), nil
|
||||
return newSyncRecordStream(ctx, backend, recordType, recordVersion), nil
|
||||
}
|
||||
|
||||
// SyncLatest returns a record stream for all the records.
|
||||
|
|
@ -304,7 +304,7 @@ func (backend *Backend) enforceCapacity(recordType string) {
|
|||
}
|
||||
}
|
||||
|
||||
func (backend *Backend) getSince(version uint64) []*databroker.Record {
|
||||
func (backend *Backend) getSince(recordType string, version uint64) []*databroker.Record {
|
||||
backend.mu.RLock()
|
||||
defer backend.mu.RUnlock()
|
||||
|
||||
|
|
@ -322,6 +322,16 @@ func (backend *Backend) getSince(version uint64) []*databroker.Record {
|
|||
}
|
||||
return true
|
||||
})
|
||||
|
||||
if recordType != "" {
|
||||
var filtered []*databroker.Record
|
||||
for _, record := range records {
|
||||
if record.GetType() == recordType {
|
||||
filtered = append(filtered, record)
|
||||
}
|
||||
}
|
||||
records = filtered
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ func TestExpiry(t *testing.T) {
|
|||
assert.NoError(t, err)
|
||||
assert.Equal(t, backend.serverVersion, sv)
|
||||
}
|
||||
stream, err := backend.Sync(ctx, backend.serverVersion, 0)
|
||||
stream, err := backend.Sync(ctx, "", backend.serverVersion, 0)
|
||||
require.NoError(t, err)
|
||||
var records []*databroker.Record
|
||||
for stream.Next(false) {
|
||||
|
|
@ -98,7 +98,7 @@ func TestExpiry(t *testing.T) {
|
|||
|
||||
backend.removeChangesBefore(time.Now().Add(time.Second))
|
||||
|
||||
stream, err = backend.Sync(ctx, backend.serverVersion, 0)
|
||||
stream, err = backend.Sync(ctx, "", backend.serverVersion, 0)
|
||||
require.NoError(t, err)
|
||||
records = nil
|
||||
for stream.Next(false) {
|
||||
|
|
@ -135,7 +135,7 @@ func TestStream(t *testing.T) {
|
|||
backend := New()
|
||||
defer func() { _ = backend.Close() }()
|
||||
|
||||
stream, err := backend.Sync(ctx, backend.serverVersion, 0)
|
||||
stream, err := backend.Sync(ctx, "TYPE", backend.serverVersion, 0)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = stream.Close() }()
|
||||
|
||||
|
|
@ -167,7 +167,7 @@ func TestStreamClose(t *testing.T) {
|
|||
ctx := context.Background()
|
||||
t.Run("by backend", func(t *testing.T) {
|
||||
backend := New()
|
||||
stream, err := backend.Sync(ctx, backend.serverVersion, 0)
|
||||
stream, err := backend.Sync(ctx, "", backend.serverVersion, 0)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, backend.Close())
|
||||
assert.False(t, stream.Next(true))
|
||||
|
|
@ -175,7 +175,7 @@ func TestStreamClose(t *testing.T) {
|
|||
})
|
||||
t.Run("by stream", func(t *testing.T) {
|
||||
backend := New()
|
||||
stream, err := backend.Sync(ctx, backend.serverVersion, 0)
|
||||
stream, err := backend.Sync(ctx, "", backend.serverVersion, 0)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, stream.Close())
|
||||
assert.False(t, stream.Next(true))
|
||||
|
|
@ -184,7 +184,7 @@ func TestStreamClose(t *testing.T) {
|
|||
t.Run("by context", func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
backend := New()
|
||||
stream, err := backend.Sync(ctx, backend.serverVersion, 0)
|
||||
stream, err := backend.Sync(ctx, "", backend.serverVersion, 0)
|
||||
require.NoError(t, err)
|
||||
cancel()
|
||||
assert.False(t, stream.Next(true))
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ func newSyncLatestRecordStream(
|
|||
func newSyncRecordStream(
|
||||
ctx context.Context,
|
||||
backend *Backend,
|
||||
recordType string,
|
||||
recordVersion uint64,
|
||||
) storage.RecordStream {
|
||||
changed := backend.onChange.Bind()
|
||||
|
|
@ -67,7 +68,7 @@ func newSyncRecordStream(
|
|||
}
|
||||
|
||||
for {
|
||||
ready = backend.getSince(recordVersion)
|
||||
ready = backend.getSince(recordType, recordVersion)
|
||||
|
||||
if len(ready) > 0 {
|
||||
// records are sorted by version,
|
||||
|
|
|
|||
|
|
@ -210,6 +210,7 @@ func (backend *Backend) SetOptions(
|
|||
// Sync syncs the records.
|
||||
func (backend *Backend) Sync(
|
||||
ctx context.Context,
|
||||
recordType string,
|
||||
serverVersion, recordVersion uint64,
|
||||
) (storage.RecordStream, error) {
|
||||
// the original ctx will be used for the stream, this ctx used for pre-stream calls
|
||||
|
|
@ -224,7 +225,7 @@ func (backend *Backend) Sync(
|
|||
return nil, storage.ErrInvalidServerVersion
|
||||
}
|
||||
|
||||
return newChangedRecordStream(ctx, backend, recordVersion), nil
|
||||
return newChangedRecordStream(ctx, backend, recordType, recordVersion), nil
|
||||
}
|
||||
|
||||
// SyncLatest syncs the latest version of each record.
|
||||
|
|
|
|||
|
|
@ -111,11 +111,11 @@ func TestBackend(t *testing.T) {
|
|||
})
|
||||
|
||||
t.Run("changed", func(t *testing.T) {
|
||||
serverVersion, recordVersion, stream, err := backend.SyncLatest(ctx, "", nil)
|
||||
serverVersion, recordVersion, stream, err := backend.SyncLatest(ctx, "sync-test", nil)
|
||||
require.NoError(t, err)
|
||||
assert.NoError(t, stream.Close())
|
||||
|
||||
stream, err = backend.Sync(ctx, serverVersion, recordVersion)
|
||||
stream, err = backend.Sync(ctx, "", serverVersion, recordVersion)
|
||||
require.NoError(t, err)
|
||||
defer stream.Close()
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ package postgres
|
|||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgconn"
|
||||
|
|
@ -78,27 +79,37 @@ func getLatestRecordVersion(ctx context.Context, q querier) (recordVersion uint6
|
|||
return recordVersion, err
|
||||
}
|
||||
|
||||
func getNextChangedRecord(ctx context.Context, q querier, afterRecordVersion uint64) (*databroker.Record, error) {
|
||||
var recordType, recordID string
|
||||
func getNextChangedRecord(ctx context.Context, q querier, recordType string, afterRecordVersion uint64) (*databroker.Record, error) {
|
||||
var recordID string
|
||||
var version uint64
|
||||
var data pgtype.JSONB
|
||||
var modifiedAt pgtype.Timestamptz
|
||||
var deletedAt pgtype.Timestamptz
|
||||
err := q.QueryRow(ctx, `
|
||||
query := `
|
||||
SELECT type, id, version, data, modified_at, deleted_at
|
||||
FROM `+schemaName+`.`+recordChangesTableName+`
|
||||
FROM ` + schemaName + `.` + recordChangesTableName + `
|
||||
WHERE version > $1
|
||||
`, afterRecordVersion).Scan(&recordType, &recordID, &version, &data, &modifiedAt, &deletedAt)
|
||||
`
|
||||
args := []any{afterRecordVersion}
|
||||
if recordType != "" {
|
||||
query += ` AND type = $2`
|
||||
args = append(args, recordType)
|
||||
}
|
||||
query += `
|
||||
ORDER BY version ASC
|
||||
LIMIT 1
|
||||
`
|
||||
err := q.QueryRow(ctx, query, args...).Scan(&recordType, &recordID, &version, &data, &modifiedAt, &deletedAt)
|
||||
if isNotFound(err) {
|
||||
return nil, storage.ErrNotFound
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("error querying next changed record: %w", err)
|
||||
}
|
||||
|
||||
var any anypb.Any
|
||||
err = protojson.Unmarshal(data.Bytes, &any)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("error unmarshaling changed record data: %w", err)
|
||||
}
|
||||
|
||||
return &databroker.Record{
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ const watchPollInterval = 30 * time.Second
|
|||
|
||||
type changedRecordStream struct {
|
||||
backend *Backend
|
||||
recordType string
|
||||
recordVersion uint64
|
||||
|
||||
ctx context.Context
|
||||
|
|
@ -95,10 +96,12 @@ type changedRecordStream struct {
|
|||
func newChangedRecordStream(
|
||||
ctx context.Context,
|
||||
backend *Backend,
|
||||
recordType string,
|
||||
recordVersion uint64,
|
||||
) storage.RecordStream {
|
||||
stream := &changedRecordStream{
|
||||
backend: backend,
|
||||
recordType: recordType,
|
||||
recordVersion: recordVersion,
|
||||
ticker: time.NewTicker(watchPollInterval),
|
||||
changed: backend.onChange.Bind(),
|
||||
|
|
@ -129,6 +132,7 @@ func (stream *changedRecordStream) Next(block bool) bool {
|
|||
stream.record, stream.err = getNextChangedRecord(
|
||||
stream.ctx,
|
||||
pool,
|
||||
stream.recordType,
|
||||
stream.recordVersion,
|
||||
)
|
||||
if isNotFound(stream.err) {
|
||||
|
|
|
|||
|
|
@ -248,8 +248,12 @@ func (backend *Backend) SetOptions(ctx context.Context, recordType string, optio
|
|||
}
|
||||
|
||||
// Sync returns a record stream of any records changed after the specified recordVersion.
|
||||
func (backend *Backend) Sync(ctx context.Context, serverVersion, recordVersion uint64) (storage.RecordStream, error) {
|
||||
return newSyncRecordStream(ctx, backend, serverVersion, recordVersion), nil
|
||||
func (backend *Backend) Sync(
|
||||
ctx context.Context,
|
||||
recordType string,
|
||||
serverVersion, recordVersion uint64,
|
||||
) (storage.RecordStream, error) {
|
||||
return newSyncRecordStream(ctx, backend, recordType, serverVersion, recordVersion), nil
|
||||
}
|
||||
|
||||
// SyncLatest returns a record stream of all the records. Some records may be returned twice if the are updated while the
|
||||
|
|
|
|||
|
|
@ -190,7 +190,7 @@ func TestExpiry(t *testing.T) {
|
|||
}})
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
stream, err := backend.Sync(ctx, serverVersion, 0)
|
||||
stream, err := backend.Sync(ctx, "TYPE", serverVersion, 0)
|
||||
require.NoError(t, err)
|
||||
var records []*databroker.Record
|
||||
for stream.Next(false) {
|
||||
|
|
@ -201,7 +201,7 @@ func TestExpiry(t *testing.T) {
|
|||
|
||||
backend.removeChangesBefore(ctx, time.Now().Add(time.Second))
|
||||
|
||||
stream, err = backend.Sync(ctx, serverVersion, 0)
|
||||
stream, err = backend.Sync(ctx, "TYPE", serverVersion, 0)
|
||||
require.NoError(t, err)
|
||||
records = nil
|
||||
for stream.Next(false) {
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import (
|
|||
func newSyncRecordStream(
|
||||
ctx context.Context,
|
||||
backend *Backend,
|
||||
recordType string,
|
||||
serverVersion uint64,
|
||||
recordVersion uint64,
|
||||
) storage.RecordStream {
|
||||
|
|
@ -36,7 +37,7 @@ func newSyncRecordStream(
|
|||
return nil, storage.ErrInvalidServerVersion
|
||||
}
|
||||
|
||||
record, err := nextChangedRecord(ctx, backend, &recordVersion)
|
||||
record, err := nextChangedRecord(ctx, backend, recordType, &recordVersion)
|
||||
if err == nil {
|
||||
return record, nil
|
||||
} else if !errors.Is(err, storage.ErrStreamDone) {
|
||||
|
|
@ -135,7 +136,7 @@ func nextScannedRecords(ctx context.Context, backend *Backend, cursor *uint64) (
|
|||
return records, nil
|
||||
}
|
||||
|
||||
func nextChangedRecord(ctx context.Context, backend *Backend, recordVersion *uint64) (*databroker.Record, error) {
|
||||
func nextChangedRecord(ctx context.Context, backend *Backend, recordType string, recordVersion *uint64) (*databroker.Record, error) {
|
||||
for {
|
||||
cmd := backend.client.ZRangeByScore(ctx, changesSetKey, &redis.ZRangeBy{
|
||||
Min: fmt.Sprintf("(%d", *recordVersion),
|
||||
|
|
@ -155,10 +156,17 @@ func nextChangedRecord(ctx context.Context, backend *Backend, recordVersion *uin
|
|||
result := results[0]
|
||||
var record databroker.Record
|
||||
err = proto.Unmarshal([]byte(result), &record)
|
||||
if err != nil {
|
||||
log.Warn(ctx).Err(err).Msg("redis: invalid record detected")
|
||||
*recordVersion++
|
||||
if err == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
*recordVersion = record.GetVersion()
|
||||
if recordType != "" && record.GetType() != recordType {
|
||||
continue
|
||||
}
|
||||
|
||||
return &record, nil
|
||||
}
|
||||
log.Warn(ctx).Err(err).Msg("redis: invalid record detected")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ type Backend interface {
|
|||
// SetOptions sets the options for a type.
|
||||
SetOptions(ctx context.Context, recordType string, options *databroker.Options) error
|
||||
// Sync syncs record changes after the specified version.
|
||||
Sync(ctx context.Context, serverVersion, recordVersion uint64) (RecordStream, error)
|
||||
Sync(ctx context.Context, recordType string, serverVersion, recordVersion uint64) (RecordStream, error)
|
||||
// SyncLatest syncs all the records.
|
||||
SyncLatest(ctx context.Context, recordType string, filter FilterExpression) (serverVersion, recordVersion uint64, stream RecordStream, err error)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue