mirror of
https://github.com/pomerium/pomerium.git
synced 2025-07-10 13:28:36 +02:00
databroker: add sync-cache (#5639)
## Summary Add a new `SyncCache`: ```go type SyncCache interface { // Clear deletes all the data for the given record type in the sync cache. Clear(recordType string) error // Records yields the databroker records stored in the cache. Records(recordType string) iter.Seq2[*Record, error] // Sync syncs the cache with the databroker. Sync(ctx context.Context, client DataBrokerServiceClient, recordType string) error } ``` The cache maintains databroker records in a local pebble database (which could be on-disk or in-memory). The way it's used is you first call `.Sync(ctx, client, recordType)` and then `.Records(recordType)`, which returns an iterator over all the records. Internally we store the databroker records in a pebble key-value database. Pebble was chosen because its fast and well-tested, but any ordered key-value store would work. The first time we call `SyncLatest` to retrieve all the records. Each subsequent time we call `Sync` with the current server and record versions to retrieve only the changes. This is significantly more efficient than calling `SyncLatest` every time. The primary use for this is in the enterprise-console as part of directory sync to improve performance with large datasets. ## Related issues - [ENG-2401](https://linear.app/pomerium/issue/ENG-2401/enterprise-console-improve-performance-of-directory-sync-using-cached) ## Checklist - [x] reference any related issues - [x] updated unit tests - [x] add appropriate label (`enhancement`, `bug`, `breaking`, `dependencies`, `ci`) - [x] ready for review --------- Co-authored-by: Denis Mishin <dmishin@pomerium.com>
This commit is contained in:
parent
c0a8b79ef1
commit
ff607fa018
5 changed files with 569 additions and 2 deletions
115
pkg/pebbleutil/pebbleutil.go
Normal file
115
pkg/pebbleutil/pebbleutil.go
Normal file
|
@ -0,0 +1,115 @@
|
|||
package pebbleutil
|
||||
|
||||
import (
|
||||
"context"
|
||||
"iter"
|
||||
"slices"
|
||||
|
||||
"github.com/cockroachdb/pebble/v2"
|
||||
"github.com/cockroachdb/pebble/v2/vfs"
|
||||
)
|
||||
|
||||
// Iterate iterates over a pebble reader.
|
||||
func Iterate[T any](src pebble.Reader, iterOptions *pebble.IterOptions, f func(it *pebble.Iterator) (T, error)) iter.Seq2[T, error] {
|
||||
var zero T
|
||||
return func(yield func(T, error) bool) {
|
||||
it, err := src.NewIter(iterOptions)
|
||||
if err != nil {
|
||||
yield(zero, err)
|
||||
return
|
||||
}
|
||||
|
||||
for it.First(); it.Valid(); it.Next() {
|
||||
value, err := f(it)
|
||||
if err != nil {
|
||||
_ = it.Close()
|
||||
yield(zero, err)
|
||||
return
|
||||
}
|
||||
|
||||
if !yield(value, nil) {
|
||||
_ = it.Close()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
err = it.Error()
|
||||
if err != nil {
|
||||
_ = it.Close()
|
||||
yield(zero, err)
|
||||
return
|
||||
}
|
||||
|
||||
err = it.Close()
|
||||
if err != nil {
|
||||
yield(zero, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// IterateKeys yields the keys in a pebble reader.
|
||||
func IterateKeys(src pebble.Reader, iterOptions *pebble.IterOptions) iter.Seq2[[]byte, error] {
|
||||
return Iterate(src, iterOptions, func(it *pebble.Iterator) ([]byte, error) {
|
||||
return slices.Clone(it.Key()), nil
|
||||
})
|
||||
}
|
||||
|
||||
// IterateValues yields the values in a pebble reader.
|
||||
func IterateValues(src pebble.Reader, iterOptions *pebble.IterOptions) iter.Seq2[[]byte, error] {
|
||||
return Iterate(src, iterOptions, func(it *pebble.Iterator) ([]byte, error) {
|
||||
value, err := it.ValueAndErr()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return slices.Clone(value), nil
|
||||
})
|
||||
}
|
||||
|
||||
// MustOpen opens a pebble database. It sets options useful for pomerium and panics if there is an error.
|
||||
func MustOpen(dirname string, options *pebble.Options) *pebble.DB {
|
||||
db, err := Open(dirname, options)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
// MustOpenMemory opens an in-memory pebble database. It panics if there is an error.
|
||||
func MustOpenMemory(options *pebble.Options) *pebble.DB {
|
||||
if options == nil {
|
||||
options = new(pebble.Options)
|
||||
}
|
||||
options.FS = vfs.NewMem()
|
||||
return MustOpen("", options)
|
||||
}
|
||||
|
||||
// Open opens a pebble database. It sets options useful for pomerium.
|
||||
func Open(dirname string, options *pebble.Options) (*pebble.DB, error) {
|
||||
if options == nil {
|
||||
options = new(pebble.Options)
|
||||
}
|
||||
options.LoggerAndTracer = pebbleLogger{}
|
||||
return pebble.Open(dirname, options)
|
||||
}
|
||||
|
||||
// PrefixToUpperBound returns an upper bound for the given prefix.
|
||||
func PrefixToUpperBound(prefix []byte) []byte {
|
||||
upperBound := make([]byte, len(prefix))
|
||||
copy(upperBound, prefix)
|
||||
for i := len(upperBound) - 1; i >= 0; i-- {
|
||||
upperBound[i] = upperBound[i] + 1
|
||||
if upperBound[i] != 0 {
|
||||
return upperBound[:i+1]
|
||||
}
|
||||
}
|
||||
return nil // no upper-bound
|
||||
}
|
||||
|
||||
type pebbleLogger struct{}
|
||||
|
||||
func (pebbleLogger) Infof(_ string, _ ...any) {}
|
||||
func (pebbleLogger) Errorf(_ string, _ ...any) {}
|
||||
func (pebbleLogger) Fatalf(_ string, _ ...any) {}
|
||||
func (pebbleLogger) Eventf(_ context.Context, _ string, _ ...any) {}
|
||||
func (pebbleLogger) IsTracingEnabled(_ context.Context) bool { return false }
|
Loading…
Add table
Add a link
Reference in a new issue