databroker: refactor databroker to sync all changes (#1879)

* refactor backend, implement encrypted store

* refactor in-memory store

* wip

* wip

* wip

* add syncer test

* fix redis expiry

* fix linting issues

* fix test by skipping non-config records

* fix backoff import

* fix init issues

* fix query

* wait for initial sync before starting directory sync

* add type to SyncLatest

* add more log messages, fix deadlock in in-memory store, always return server version from SyncLatest

* update sync types and tests

* add redis tests

* skip macos in github actions

* add comments to proto

* split getBackend into separate methods

* handle errors in initVersion

* return different error for not found vs other errors in get

* use exponential backoff for redis transaction retry

* rename raw to result

* use context instead of close channel

* store type urls as constants in databroker

* use timestampb instead of ptypes

* fix group merging not waiting

* change locked names

* update GetAll to return latest record version

* add method to grpcutil to get the type url for a protobuf type

pkg/cryptutil/helpers.go

@@ -3,6 +3,7 @@ package cryptutil
 import (
 	"crypto/rand"
 	"encoding/base64"
+	"encoding/binary"
 )
 
 // DefaultKeySize is the default key size in bytes.
@@ -29,6 +30,13 @@ func NewRandomStringN(c int) string {
 	return base64.StdEncoding.EncodeToString(randomBytes(c))
 }
 
+// NewRandomUInt64 returns a random uint64.
+//
+// Panics if source of randomness fails.
+func NewRandomUInt64() uint64 {
+	return binary.LittleEndian.Uint64(randomBytes(8))
+}
+
 // randomBytes generates C number of random bytes suitable for cryptographic
 // operations.
 //