replace GetAllPages with InitialSync, improve merge performance (#1624)

* replace GetAllPages with InitialSync, improve merge performance

* fmt proto

* add test for base64 function

* add sync test

* go mod tidy

Co-authored-by: Bobby DeSimone <bobbydesimone@gmail.com>

internal/encoding/base64.go

@@ -0,0 +1,23 @@
+package encoding
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"strings"
+)
+
+// DecodeBase64OrJSON decodes a JSON string that can optionally be base64 encoded.
+func DecodeBase64OrJSON(in string, out interface{}) error {
+	in = strings.TrimSpace(in)
+
+	// the data can be base64 encoded
+	if !json.Valid([]byte(in)) {
+		bs, err := base64.StdEncoding.DecodeString(in)
+		if err != nil {
+			return err
+		}
+		in = string(bs)
+	}
+
+	return json.Unmarshal([]byte(in), out)
+}

internal/encoding/base64_test.go

@@ -0,0 +1,20 @@
+package encoding
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDecodeBase64OrJSON(t *testing.T) {
+	var obj struct {
+		X string `json:"x"`
+	}
+	err := DecodeBase64OrJSON(`    {"x": "y"}    `, &obj)
+	assert.NoError(t, err)
+	assert.Equal(t, "y", obj.X)
+
+	err = DecodeBase64OrJSON(`    eyJ4IjoieiJ9Cg==    `, &obj)
+	assert.NoError(t, err)
+	assert.Equal(t, "z", obj.X)
+}