databroker: refactor databroker to sync all changes (#1879)

* refactor backend, implement encrypted store

* refactor in-memory store

* wip

* wip

* wip

* add syncer test

* fix redis expiry

* fix linting issues

* fix test by skipping non-config records

* fix backoff import

* fix init issues

* fix query

* wait for initial sync before starting directory sync

* add type to SyncLatest

* add more log messages, fix deadlock in in-memory store, always return server version from SyncLatest

* update sync types and tests

* add redis tests

* skip macos in github actions

* add comments to proto

* split getBackend into separate methods

* handle errors in initVersion

* return different error for not found vs other errors in get

* use exponential backoff for redis transaction retry

* rename raw to result

* use context instead of close channel

* store type urls as constants in databroker

* use timestampb instead of ptypes

* fix group merging not waiting

* change locked names

* update GetAll to return latest record version

* add method to grpcutil to get the type url for a protobuf type

internal/testutil/testutil.go

@@ -3,6 +3,8 @@ package testutil
 
 import (
 	"encoding/json"
+	"os"
+	"path/filepath"
 	"reflect"
 	"testing"
 
@@ -32,3 +34,28 @@ func toProtoJSON(protoMsg interface{}) json.RawMessage {
 	bs, _ := protojson.Marshal(v2)
 	return bs
 }
+
+// ModRoot returns the directory containing the go.mod file.
+func ModRoot() string {
+	dir, err := os.Getwd()
+	if err != nil {
+		panic("error getting working directory")
+	}
+
+	for {
+		if fi, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil && !fi.IsDir() {
+			return dir
+		}
+		d := filepath.Dir(dir)
+		if d == dir {
+			break
+		}
+		dir = d
+	}
+	return ""
+}
+
+// TestDataRoot returns the testdata directory.
+func TestDataRoot() string {
+	return filepath.Join(ModRoot(), "internal", "testutil", "testdata")
+}