Add new configurable bootstrap writers (#2405) (#5114)

* Add new configurable bootstrap writers (#2405)

This PR adds the ability to configure different backends to use for
storing modifications to the zero bootstrap config. The two currently
implemented backends allow writing changes to a file or to a Kubernetes
secret. Backend selection is determined by the scheme in a URI passed to
the flag '--config-writeback-uri'.

In a Kubernetes environment, where the bootstrap config is mounted into
the pod from a secret, this option allows Pomerium to write changes back
to the secret, as writes to the mounted secret file on disk are not
persisted.

* Use env vars for bootstrap config filepath/writeback uri

* linter pass and code cleanup

* Add new config writer options mechanism

This moves the encryption cipher parameter out of the WriteConfig()
method in the ConfigWriter interface and into a new ConfigWriterOptions
struct. Options (e.g. cipher) can be applied to an existing ConfigWriter
to allow customizing implementation-specific behavior.

* Code cleanup/lint fixes

* Move vendored k8s code into separate package, and add license header and package comment

cmd/pomerium/main.go

@@ -12,6 +12,8 @@ import (
 	"github.com/pomerium/pomerium/config"
 	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/internal/version"
+	_ "github.com/pomerium/pomerium/internal/zero/bootstrap/writers/filesystem"
+	_ "github.com/pomerium/pomerium/internal/zero/bootstrap/writers/k8s"
 	zero_cmd "github.com/pomerium/pomerium/internal/zero/cmd"
 	"github.com/pomerium/pomerium/pkg/cmd/pomerium"
 	"github.com/pomerium/pomerium/pkg/envoy/files"

internal/zero/bootstrap/bootstrap.go

@@ -100,11 +100,11 @@ func (svc *Source) updateAndSave(ctx context.Context) error {
 
 	svc.UpdateBootstrap(ctx, *cfg)
 
-	if svc.fileCachePath == nil {
+	if svc.writer == nil {
 		return nil
 	}
 
-	err = SaveBootstrapConfigToFile(cfg, *svc.fileCachePath, svc.fileCipher)
+	err = SaveBootstrapConfig(ctx, svc.writer, cfg)
 	if err != nil {
 		log.Ctx(ctx).Error().Err(err).
 			Msg("failed to save bootstrap config to file, note it may prevent Pomerium from starting up in case of connectivity issues")

internal/zero/bootstrap/file.go

@@ -9,11 +9,13 @@ package bootstrap
  *
  */
 import (
+	"context"
 	"crypto/cipher"
 	"encoding/json"
 	"fmt"
 	"os"
 
+	"github.com/pomerium/pomerium/internal/zero/bootstrap/writers"
 	"github.com/pomerium/pomerium/pkg/cryptutil"
 	"github.com/pomerium/pomerium/pkg/health"
 	cluster_api "github.com/pomerium/pomerium/pkg/zero/cluster"
@@ -39,9 +41,9 @@ func LoadBootstrapConfigFromFile(fp string, cipher cipher.AEAD) (*cluster_api.Bo
 	return &dst, nil
 }
 
-// SaveBootstrapConfigToFile saves the bootstrap configuration to a file.
+// SaveBootstrapConfig saves the bootstrap configuration to a file.
-func SaveBootstrapConfigToFile(src *cluster_api.BootstrapConfig, fp string, cipher cipher.AEAD) error {
+func SaveBootstrapConfig(ctx context.Context, writer writers.ConfigWriter, src *cluster_api.BootstrapConfig) error {
-	err := saveBootstrapConfigToFile(src, fp, cipher)
+	err := writer.WriteConfig(ctx, src)
 	if err != nil {
 		health.ReportError(health.ZeroBootstrapConfigSave, err)
 	} else {
@@ -49,17 +51,3 @@ func SaveBootstrapConfigToFile(src *cluster_api.BootstrapConfig, fp string, ciph
 	}
 	return err
 }
-
-func saveBootstrapConfigToFile(src *cluster_api.BootstrapConfig, fp string, cipher cipher.AEAD) error {
-	plaintext, err := json.Marshal(src)
-	if err != nil {
-		return fmt.Errorf("marshal file config: %w", err)
-	}
-
-	ciphertext := cryptutil.Encrypt(cipher, plaintext, nil)
-	err = os.WriteFile(fp, ciphertext, 0o600)
-	if err != nil {
-		return fmt.Errorf("write bootstrap config: %w", err)
-	}
-	return nil
-}

internal/zero/bootstrap/file_test.go

@@ -1,33 +0,0 @@
-package bootstrap_test
-
-import (
-	"os"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/pomerium/pomerium/internal/zero/bootstrap"
-	"github.com/pomerium/pomerium/pkg/cryptutil"
-	cluster_api "github.com/pomerium/pomerium/pkg/zero/cluster"
-)
-
-func TestFile(t *testing.T) {
-	cipher, err := cryptutil.NewAEADCipher(cryptutil.NewKey())
-	require.NoError(t, err)
-
-	txt := "test"
-	src := cluster_api.BootstrapConfig{
-		DatabrokerStorageConnection: &txt,
-	}
-
-	fd, err := os.CreateTemp(t.TempDir(), "test.data")
-	require.NoError(t, err)
-	require.NoError(t, fd.Close())
-
-	require.NoError(t, bootstrap.SaveBootstrapConfigToFile(&src, fd.Name(), cipher))
-
-	dst, err := bootstrap.LoadBootstrapConfigFromFile(fd.Name(), cipher)
-	require.NoError(t, err)
-
-	require.Equal(t, src, *dst)
-}

internal/zero/bootstrap/new.go

@@ -15,6 +15,7 @@ import (
 	"github.com/pomerium/pomerium/internal/atomicutil"
 	"github.com/pomerium/pomerium/internal/deterministicecdsa"
 	sdk "github.com/pomerium/pomerium/internal/zero/api"
+	"github.com/pomerium/pomerium/internal/zero/bootstrap/writers"
 	"github.com/pomerium/pomerium/pkg/cryptutil"
 	"github.com/pomerium/pomerium/pkg/netutil"
 )
@@ -27,13 +28,14 @@ type Source struct {
 
 	fileCachePath *string
 	fileCipher    cipher.AEAD
+	writer        writers.ConfigWriter
 
 	checkForUpdate chan struct{}
 	updateInterval atomicutil.Value[time.Duration]
 }
 
 // New creates a new bootstrap config source
-func New(secret []byte, fileCachePath *string, api *sdk.API) (*Source, error) {
+func New(secret []byte, fileCachePath *string, writer writers.ConfigWriter, api *sdk.API) (*Source, error) {
 	cfg := new(config.Config)
 
 	err := setConfigDefaults(cfg)
@@ -53,12 +55,19 @@ func New(secret []byte, fileCachePath *string, api *sdk.API) (*Source, error) {
 		return nil, fmt.Errorf("init cypher: %w", err)
 	}
 
+	if writer != nil {
+		writer = writer.WithOptions(writers.ConfigWriterOptions{
+			Cipher: cipher,
+		})
+	}
+
 	svc := &Source{
 		api:            api,
 		source:         source{ready: make(chan struct{})},
 		fileCachePath:  fileCachePath,
 		fileCipher:     cipher,
 		checkForUpdate: make(chan struct{}, 1),
+		writer:         writer,
 	}
 	svc.cfg.Store(cfg)
 	svc.updateInterval.Store(DefaultCheckForUpdateIntervalWhenDisconnected)

internal/zero/bootstrap/new_test.go

@@ -11,7 +11,7 @@ import (
 func TestConfigDeterministic(t *testing.T) {
 	secret := []byte("secret")
 
-	src, err := bootstrap.New(secret, nil, nil)
+	src, err := bootstrap.New(secret, nil, nil, nil)
 	require.NoError(t, err)
 	cfg := src.GetConfig()
 	require.NotNil(t, cfg)
@@ -20,7 +20,7 @@ func TestConfigDeterministic(t *testing.T) {
 	require.NoError(t, cfg.Options.Validate())
 
 	// test that the config is deterministic
-	src2, err := bootstrap.New(secret, nil, nil)
+	src2, err := bootstrap.New(secret, nil, nil, nil)
 	require.NoError(t, err)
 
 	cfg2 := src2.GetConfig()

internal/zero/bootstrap/source_test.go

@@ -18,7 +18,7 @@ func TestConfigChanges(t *testing.T) {
 
 	secret := []byte("secret")
 
-	src, err := bootstrap.New(secret, nil, nil)
+	src, err := bootstrap.New(secret, nil, nil, nil)
 	require.NoError(t, err)
 
 	ptr := func(s string) *string { return &s }

internal/zero/bootstrap/writers/filesystem/file.go

@@ -0,0 +1,58 @@
+package filesystem
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"os"
+
+	"github.com/pomerium/pomerium/internal/zero/bootstrap/writers"
+	"github.com/pomerium/pomerium/pkg/cryptutil"
+	cluster_api "github.com/pomerium/pomerium/pkg/zero/cluster"
+)
+
+func init() {
+	writers.RegisterBuilder("file", newFileWriter)
+}
+
+func newFileWriter(uri *url.URL) (writers.ConfigWriter, error) {
+	if uri.Host != "" {
+		// prevent the common mistake of "file://path/to/file"
+		return nil, fmt.Errorf(`invalid file uri %q (did you mean "file:///%s%s"?)`, uri.String(), uri.Host, uri.Path)
+	}
+	return &fileWriter{
+		filePath: uri.Path,
+	}, nil
+}
+
+type fileWriter struct {
+	opts     writers.ConfigWriterOptions
+	filePath string
+}
+
+// WithOptions implements writers.ConfigWriter.
+func (w *fileWriter) WithOptions(opts writers.ConfigWriterOptions) writers.ConfigWriter {
+	clone := *w
+	clone.opts = opts
+	return &clone
+}
+
+// WriteConfig implements ConfigWriter.
+func (w *fileWriter) WriteConfig(_ context.Context, src *cluster_api.BootstrapConfig) error {
+	data, err := json.Marshal(src)
+	if err != nil {
+		return fmt.Errorf("marshal file config: %w", err)
+	}
+
+	if w.opts.Cipher != nil {
+		data = cryptutil.Encrypt(w.opts.Cipher, data, nil)
+	}
+	err = os.WriteFile(w.filePath, data, 0o600)
+	if err != nil {
+		return fmt.Errorf("write bootstrap config: %w", err)
+	}
+	return nil
+}
+
+var _ writers.ConfigWriter = (*fileWriter)(nil)

internal/zero/bootstrap/writers/filesystem/file_test.go

@@ -0,0 +1,67 @@
+package filesystem_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/pomerium/pomerium/internal/zero/bootstrap"
+	"github.com/pomerium/pomerium/internal/zero/bootstrap/writers"
+	"github.com/pomerium/pomerium/pkg/cryptutil"
+	cluster_api "github.com/pomerium/pomerium/pkg/zero/cluster"
+)
+
+func TestFileWriter(t *testing.T) {
+	cipher, err := cryptutil.NewAEADCipher(cryptutil.NewKey())
+	require.NoError(t, err)
+
+	txt := "test"
+	src := cluster_api.BootstrapConfig{
+		DatabrokerStorageConnection: &txt,
+	}
+
+	fd, err := os.CreateTemp(t.TempDir(), "test.data")
+	require.NoError(t, err)
+	require.NoError(t, fd.Close())
+
+	writer, err := writers.NewForURI(fmt.Sprintf("file://%s", fd.Name()))
+	require.NoError(t, err)
+
+	writer = writer.WithOptions(writers.ConfigWriterOptions{
+		Cipher: cipher,
+	})
+	require.NoError(t, bootstrap.SaveBootstrapConfig(context.Background(), writer, &src))
+
+	dst, err := bootstrap.LoadBootstrapConfigFromFile(fd.Name(), cipher)
+	require.NoError(t, err)
+
+	require.Equal(t, src, *dst)
+}
+
+func TestNewForURI(t *testing.T) {
+	for _, tc := range []struct {
+		uri string
+		err string
+	}{
+		{
+			uri: "file:///path/to/file",
+		},
+		{
+			uri: "file://path/to/file",
+			err: `invalid file uri "file://path/to/file" (did you mean "file:///path/to/file"?)`,
+		},
+	} {
+		w, err := writers.NewForURI(tc.uri)
+		if tc.err == "" {
+			assert.NoError(t, err)
+			assert.NotNil(t, w)
+		} else {
+			assert.EqualError(t, err, tc.err)
+			assert.Nil(t, w)
+		}
+	}
+}

internal/zero/bootstrap/writers/k8s/rest/config.go

@@ -0,0 +1,86 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// This package contains some of the in-cluster configuration logic from [config.go]
+// to avoid a dependency on k8s.io/client-go. Only code used in Pomerium is
+// included, and some usages of helper functions/types have been refactored out.
+//
+// [config.go]: https://github.com/kubernetes/client-go/blob/d11d5308d688d65723cb1bfcaeb7703b95debc5a/rest/config.go
+package rest
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"net"
+	"os"
+)
+
+var ErrNotInCluster = errors.New("unable to load in-cluster configuration, KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT must be defined")
+
+// Config holds the common attributes that can be passed to a Kubernetes client on
+// initialization.
+type Config struct {
+	// Host must be a host string, a host:port pair, or a URL to the base of the apiserver.
+	// If a URL is given then the (optional) Path of that URL represents a prefix that must
+	// be appended to all request URIs used to access the apiserver. This allows a frontend
+	// proxy to easily relocate all of the apiserver endpoints.
+	Host string
+	// TLSClientConfig contains settings to enable transport layer security
+	TLSClientConfig *tls.Config
+	// Server requires Bearer authentication. This client will not attempt to use
+	// refresh tokens for an OAuth2 flow.
+	BearerToken string
+}
+
+// InClusterConfig returns a config object which uses the service account
+// kubernetes gives to pods. It's intended for clients that expect to be
+// running inside a pod running on kubernetes. It will return ErrNotInCluster
+// if called from a process not running in a kubernetes environment.
+func InClusterConfig() (*Config, error) {
+	var (
+		tokenFile  = "/var/run/secrets/kubernetes.io/serviceaccount/token" //nolint:gosec
+		rootCAFile = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+	)
+	host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT")
+	if len(host) == 0 || len(port) == 0 {
+		return nil, ErrNotInCluster
+	}
+
+	token, err := os.ReadFile(tokenFile)
+	if err != nil {
+		return nil, err
+	}
+
+	tlsClientConfig := &tls.Config{
+		MinVersion: tls.VersionTLS12,
+	}
+
+	cacert, err := os.ReadFile(rootCAFile)
+	if err != nil {
+		return nil, err
+	}
+
+	pool := x509.NewCertPool()
+	pool.AppendCertsFromPEM(cacert)
+	tlsClientConfig.RootCAs = pool
+
+	return &Config{
+		Host:            "https://" + net.JoinHostPort(host, port),
+		TLSClientConfig: tlsClientConfig,
+		BearerToken:     string(token),
+	}, nil
+}

internal/zero/bootstrap/writers/k8s/secret.go

@@ -0,0 +1,161 @@
+package k8s
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/pomerium/pomerium/internal/log"
+	"github.com/pomerium/pomerium/internal/zero/bootstrap/writers"
+	"github.com/pomerium/pomerium/internal/zero/bootstrap/writers/k8s/rest"
+	"github.com/pomerium/pomerium/pkg/cryptutil"
+	cluster_api "github.com/pomerium/pomerium/pkg/zero/cluster"
+)
+
+func init() {
+	writers.RegisterBuilder("secret", newInClusterSecretWriter)
+}
+
+type secretWriter struct {
+	opts         writers.ConfigWriterOptions
+	client       *http.Client
+	apiserverURL *url.URL
+	namespace    string
+	name         string
+	key          string
+}
+
+// WithOptions implements writers.ConfigWriter.
+func (w *secretWriter) WithOptions(opts writers.ConfigWriterOptions) writers.ConfigWriter {
+	clone := *w
+	clone.opts = opts
+	return &clone
+}
+
+func newSecretWriterForConfig(uri *url.URL, config *rest.Config) (writers.ConfigWriter, error) {
+	parts := strings.SplitN(path.Join(uri.Host, uri.Path), "/", 3)
+	if len(parts) != 3 || parts[1] == "" || parts[2] == "" {
+		return nil, fmt.Errorf("invalid secret uri %q, expecting format \"secret://namespace/name/key\"", uri.String())
+	} else if parts[0] == "" {
+		return nil, fmt.Errorf(`invalid secret uri %q (did you mean "secret:/%s"?)`, uri.String(), uri.Path)
+	}
+	u, err := url.Parse(config.Host)
+	if err != nil {
+		return nil, err
+	}
+
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	transport.TLSClientConfig = config.TLSClientConfig
+
+	client := &http.Client{
+		Transport: &roundTripper{
+			bearerToken: config.BearerToken,
+			base:        transport,
+		},
+	}
+
+	return &secretWriter{
+		client:       client,
+		apiserverURL: u,
+		namespace:    parts[0],
+		name:         parts[1],
+		key:          parts[2],
+	}, nil
+}
+
+func newInClusterSecretWriter(uri *url.URL) (writers.ConfigWriter, error) {
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		return nil, err
+	}
+	return newSecretWriterForConfig(uri, config)
+}
+
+// WriteConfig implements ConfigWriter.
+func (w *secretWriter) WriteConfig(ctx context.Context, src *cluster_api.BootstrapConfig) error {
+	u := w.apiserverURL.ResolveReference(&url.URL{
+		Path: path.Join("/api/v1/namespaces", w.namespace, "secrets", w.name),
+		RawQuery: url.Values{
+			"fieldManager": {"pomerium-zero"},
+			"force":        {"true"},
+		}.Encode(),
+	})
+	data, err := json.Marshal(src)
+	if err != nil {
+		return err
+	}
+
+	if w.opts.Cipher != nil {
+		data = cryptutil.Encrypt(w.opts.Cipher, data, nil)
+	}
+	encodedData := base64.StdEncoding.EncodeToString(data)
+
+	patch, _ := yaml.Marshal(map[string]any{
+		"apiVersion": "v1",
+		"kind":       "Secret",
+		"metadata": map[string]any{
+			"name":      w.name,
+			"namespace": w.namespace,
+		},
+		"data": map[string]string{
+			w.key: encodedData,
+		},
+	})
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPatch, u.String(), bytes.NewReader(patch))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/apply-patch+yaml")
+
+	resp, err := w.client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	switch resp.StatusCode {
+	case http.StatusOK, http.StatusCreated:
+		return nil
+	case http.StatusForbidden:
+		if resp.Header.Get("Content-Type") == "application/json" {
+			// log the detailed status message if available
+			status, err := io.ReadAll(resp.Body)
+			if err != nil {
+				break
+			}
+			var buf bytes.Buffer
+			err = json.Compact(&buf, status)
+			if err != nil {
+				break
+			}
+			log.Ctx(ctx).Error().
+				RawJSON("response", buf.Bytes()).
+				Msgf("%s %s: %s", req.Method, req.URL, resp.Status)
+		}
+	}
+	return fmt.Errorf("unexpected status: %s", resp.Status)
+}
+
+var _ writers.ConfigWriter = (*secretWriter)(nil)
+
+type roundTripper struct {
+	base        http.RoundTripper
+	bearerToken string
+}
+
+func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	if req.Header.Get("Authorization") == "" {
+		req = req.Clone(req.Context())
+		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", rt.bearerToken))
+	}
+	return rt.base.RoundTrip(req)
+}

internal/zero/bootstrap/writers/k8s/secret_test.go

@@ -0,0 +1,154 @@
+package k8s
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+
+	"github.com/pomerium/pomerium/internal/zero/bootstrap"
+	"github.com/pomerium/pomerium/internal/zero/bootstrap/writers"
+	"github.com/pomerium/pomerium/internal/zero/bootstrap/writers/k8s/rest"
+	"github.com/pomerium/pomerium/pkg/cryptutil"
+	cluster_api "github.com/pomerium/pomerium/pkg/zero/cluster"
+)
+
+func TestSecretWriter(t *testing.T) {
+	requests := make(chan *http.Request, 1)
+	server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		req := r.Clone(context.Background())
+		contents, _ := io.ReadAll(r.Body)
+		req.Body = io.NopCloser(bytes.NewReader(contents))
+		requests <- req
+		w.WriteHeader(http.StatusOK)
+	}))
+
+	server.StartTLS()
+	defer server.Close()
+
+	pool := x509.NewCertPool()
+	pool.AddCert(server.Certificate())
+
+	restConfig := &rest.Config{
+		Host: server.URL,
+		TLSClientConfig: &tls.Config{
+			MinVersion: tls.VersionTLS12,
+			RootCAs:    pool,
+		},
+		BearerToken: "token",
+	}
+
+	// replace the default in-cluster builder with one that uses the test server
+	writers.RegisterBuilder("secret", func(uri *url.URL) (writers.ConfigWriter, error) {
+		return newSecretWriterForConfig(uri, restConfig)
+	})
+
+	t.Run("Writer", func(t *testing.T) {
+		writer, err := writers.NewForURI("secret://pomerium/bootstrap/bootstrap.dat")
+		require.NoError(t, err)
+		cipher, err := cryptutil.NewAEADCipher(cryptutil.NewKey())
+		require.NoError(t, err)
+
+		txt := "test"
+		src := cluster_api.BootstrapConfig{
+			DatabrokerStorageConnection: &txt,
+		}
+
+		writer = writer.WithOptions(writers.ConfigWriterOptions{
+			Cipher: cipher,
+		})
+
+		require.NoError(t, bootstrap.SaveBootstrapConfig(context.Background(), writer, &src))
+
+		r := <-requests
+		assert.Equal(t, "PATCH", r.Method)
+		assert.Equal(t, "application/apply-patch+yaml", r.Header.Get("Content-Type"))
+		assert.Equal(t, "/api/v1/namespaces/pomerium/secrets/bootstrap?fieldManager=pomerium-zero&force=true", r.RequestURI)
+
+		unstructured := make(map[string]any)
+		require.NoError(t, yaml.NewDecoder(r.Body).Decode(&unstructured))
+
+		// decrypt data["bootstrap.dat"] and replace it with the plaintext, so
+		// it can be compared (the ciphertext will be different each time)
+		encoded, err := base64.StdEncoding.DecodeString(unstructured["data"].(map[string]any)["bootstrap.dat"].(string))
+		require.NoError(t, err)
+		plaintext, err := cryptutil.Decrypt(cipher, encoded, nil)
+		require.NoError(t, err)
+		unstructured["data"].(map[string]any)["bootstrap.dat"] = string(plaintext)
+
+		require.Equal(t, map[string]any{
+			"apiVersion": "v1",
+			"kind":       "Secret",
+			"metadata": map[string]any{
+				"name":      "bootstrap",
+				"namespace": "pomerium",
+			},
+			"data": map[string]any{
+				"bootstrap.dat": `{"databrokerStorageConnection":"test","sharedSecret":null}`,
+			},
+		}, unstructured)
+	})
+
+	t.Run("NewForURI", func(t *testing.T) {
+		for _, tc := range []struct {
+			uris []string
+			errf string
+		}{
+			{
+				uris: []string{
+					"secret://namespace",
+					"secret://namespace/name",
+					"secret:///",
+					"secret:////",
+					"secret://namespace//",
+					"secret://namespace/name/",
+				},
+				errf: `invalid secret uri "%s", expecting format "secret://namespace/name/key"`,
+			},
+			{
+				uris: []string{"secret:///namespace/name/key"},
+				errf: `invalid secret uri "%s" (did you mean "secret://namespace/name/key"?)`,
+			},
+			{
+				uris: []string{"secret:///namespace/name/key/with/slashes"},
+				errf: `invalid secret uri "%s" (did you mean "secret://namespace/name/key/with/slashes"?)`,
+			},
+			{
+				uris: []string{
+					"secret://namespace/name/key",
+					"secret://namespace/name/key/with/slashes",
+					"secret://namespace/name/key.with.dots",
+					"secret://namespace/name/key_with_underscores",
+					"secret://namespace/name/key-with-dashes",
+					"secret://namespace-with-dashes/name-with-dashes/key-with-dashes",
+					"secret://namespace_with_underscores/name_with_underscores/key_with_underscores",
+					"secret://namespace.with.dots/name.with.dots/key.with.dots",
+					"secret://namespace-with-dashes/name/key/with/slashes",
+					"secret://namespace_with_underscores/name.with.dots/_key/with_/_slashes_and_underscores",
+				},
+			},
+		} {
+			for _, uri := range tc.uris {
+				w, err := writers.NewForURI(uri)
+				if tc.errf == "" {
+					assert.NoError(t, err)
+					assert.NotNil(t, w)
+				} else {
+					assert.EqualError(t, err, fmt.Sprintf(tc.errf, uri))
+					assert.Nil(t, w)
+				}
+			}
+		}
+	})
+}

internal/zero/bootstrap/writers/writers.go

@@ -0,0 +1,50 @@
+package writers
+
+import (
+	"context"
+	"crypto/cipher"
+	"fmt"
+	"net/url"
+	"sync"
+
+	cluster_api "github.com/pomerium/pomerium/pkg/zero/cluster"
+)
+
+type ConfigWriter interface {
+	WriteConfig(ctx context.Context, src *cluster_api.BootstrapConfig) error
+	WithOptions(opts ConfigWriterOptions) ConfigWriter
+}
+
+type ConfigWriterOptions struct {
+	// A cipher used to encrypt the configuration before writing it.
+	// If nil, the configuration will be written in plaintext.
+	Cipher cipher.AEAD
+}
+
+// A WriterBuilder creates and initializes a new ConfigWriter previously
+// obtained from LoadWriter.
+type WriterBuilder func(uri *url.URL) (ConfigWriter, error)
+
+var writers sync.Map
+
+func RegisterBuilder(scheme string, wb WriterBuilder) {
+	writers.Store(scheme, wb)
+}
+
+func LoadBuilder(scheme string) WriterBuilder {
+	if writer, ok := writers.Load(scheme); ok {
+		return writer.(WriterBuilder)
+	}
+	return nil
+}
+
+func NewForURI(uri string) (ConfigWriter, error) {
+	u, err := url.Parse(uri)
+	if err != nil {
+		return nil, fmt.Errorf("malformed uri: %w", err)
+	}
+	if wb := LoadBuilder(u.Scheme); wb != nil {
+		return wb(u)
+	}
+	return nil, fmt.Errorf("unknown scheme: %q", u.Scheme)
+}

internal/zero/bootstrap/writers/writers_test.go

@@ -0,0 +1,38 @@
+package writers_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/pomerium/pomerium/internal/zero/bootstrap/writers"
+)
+
+func TestNewForURI(t *testing.T) {
+	for _, tc := range []struct {
+		uri string
+		err string
+	}{
+		{
+			uri: "/foo",
+			err: "unknown scheme: \"\"",
+		},
+		{
+			uri: "foo://bar",
+			err: "unknown scheme: \"foo\"",
+		},
+		{
+			uri: "foo://\x7f",
+			err: "malformed uri: parse \"foo://\\x7f\": net/url: invalid control character in URL",
+		},
+	} {
+		w, err := writers.NewForURI(tc.uri)
+		if tc.err == "" {
+			assert.NoError(t, err)
+			assert.NotNil(t, w)
+		} else {
+			assert.EqualError(t, err, tc.err)
+			assert.Nil(t, w)
+		}
+	}
+}

internal/zero/cmd/command.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
-	"path/filepath"
 	"syscall"
 
 	"github.com/rs/zerolog"
@@ -41,6 +40,11 @@ func Run(ctx context.Context, configFile string) error {
 	} else {
 		log.Ctx(ctx).Info().Str("file", bootstrapConfigFileName).Msg("cluster bootstrap config path")
 		opts = append(opts, controller.WithBootstrapConfigFileName(bootstrapConfigFileName))
+
+		if uri := getBootstrapConfigWritebackURI(); uri != "" {
+			log.Ctx(ctx).Debug().Str("uri", uri).Msg("cluster bootstrap config writeback URI")
+			opts = append(opts, controller.WithBootstrapConfigWritebackURI(uri))
+		}
 	}
 
 	return controller.Run(withInterrupt(ctx), opts...)
@@ -81,17 +85,3 @@ func setupLogger() error {
 
 	return nil
 }
-
-func getBootstrapConfigFileName() (string, error) {
-	cacheDir, err := os.UserCacheDir()
-	if err != nil {
-		return "", err
-	}
-
-	dir := filepath.Join(cacheDir, "pomerium")
-	if err := os.MkdirAll(dir, 0o700); err != nil {
-		return "", fmt.Errorf("error creating cache directory: %w", err)
-	}
-
-	return filepath.Join(dir, "bootstrap.dat"), nil
-}

internal/zero/cmd/env.go

@@ -1,7 +1,9 @@
 package cmd
 
 import (
+	"fmt"
 	"os"
+	"path/filepath"
 
 	"github.com/spf13/viper"
 )
@@ -10,6 +12,12 @@ const (
 	// PomeriumZeroTokenEnv is the environment variable name for the API token.
 	//nolint: gosec
 	PomeriumZeroTokenEnv = "POMERIUM_ZERO_TOKEN"
+
+	// BootstrapConfigFileName can be set to override the default location of the bootstrap config file.
+	BootstrapConfigFileName = "BOOTSTRAP_CONFIG_FILE"
+	// BootstrapConfigWritebackURI controls how changes to the bootstrap config are persisted.
+	// See controller.WithBootstrapConfigWritebackURI for details.
+	BootstrapConfigWritebackURI = "BOOTSTRAP_CONFIG_WRITEBACK_URI"
 )
 
 func getToken(configFile string) string {
@@ -50,3 +58,23 @@ func getOTELAPIEndpoint() string {
 	}
 	return "https://telemetry.pomerium.app"
 }
+
+func getBootstrapConfigFileName() (string, error) {
+	if filename := os.Getenv(BootstrapConfigFileName); filename != "" {
+		return filename, nil
+	}
+	cacheDir, err := os.UserCacheDir()
+	if err != nil {
+		return "", err
+	}
+	dir := filepath.Join(cacheDir, "pomerium")
+	if err := os.MkdirAll(dir, 0o700); err != nil {
+		return "", fmt.Errorf("error creating cache directory: %w", err)
+	}
+
+	return filepath.Join(dir, "bootstrap.dat"), nil
+}
+
+func getBootstrapConfigWritebackURI() string {
+	return os.Getenv(BootstrapConfigWritebackURI)
+}

internal/zero/controller/config.go

@@ -11,8 +11,9 @@ type controllerConfig struct {
 	connectAPIEndpoint string
 	otelEndpoint       string
 
-	tmpDir                  string
+	tmpDir                      string
-	bootstrapConfigFileName *string
+	bootstrapConfigFileName     *string
+	bootstrapConfigWritebackURI *string
 
 	reconcilerLeaseDuration  time.Duration
 	databrokerRequestTimeout time.Duration
@@ -60,6 +61,41 @@ func WithBootstrapConfigFileName(name string) Option {
 	}
 }
 
+// WithBootstrapConfigWritebackURI sets the URI to use for persisting changes made to the
+// bootstrap config read from a filename specified by WithBootstrapConfigFileName.
+// Accepts a URI with a non-empty scheme and path.
+//
+// The following schemes are supported:
+//
+// # file
+//
+// Writes the config to a file on disk.
+//
+// Example: "file:///path/to/file" would write the config to "/path/to/file"
+// on disk.
+//
+// # secret
+//
+// Writes the config to a Kubernetes Secret. Uses the format
+// "secret://namespace/name/key".
+//
+// Example: "secret://pomerium/bootstrap/bootstrap.dat" would
+// write the config to a secret named "bootstrap" in the "pomerium" namespace,
+// under the key "bootstrap.dat", as if created with the following YAML:
+//
+//	apiVersion: v1
+//	kind: Secret
+//	metadata:
+//	  name: bootstrap
+//	  namespace: pomerium
+//	data:
+//	  bootstrap.dat: <base64 encoded config>
+func WithBootstrapConfigWritebackURI(uri string) Option {
+	return func(c *controllerConfig) {
+		c.bootstrapConfigWritebackURI = &uri
+	}
+}
+
 // WithDatabrokerLeaseDuration sets the lease duration for the
 func WithDatabrokerLeaseDuration(duration time.Duration) Option {
 	return func(c *controllerConfig) {

internal/zero/controller/controller.go

@@ -15,6 +15,7 @@ import (
 	"github.com/pomerium/pomerium/internal/zero/analytics"
 	sdk "github.com/pomerium/pomerium/internal/zero/api"
 	"github.com/pomerium/pomerium/internal/zero/bootstrap"
+	"github.com/pomerium/pomerium/internal/zero/bootstrap/writers"
 	"github.com/pomerium/pomerium/internal/zero/healthcheck"
 	"github.com/pomerium/pomerium/internal/zero/leaser"
 	"github.com/pomerium/pomerium/internal/zero/reconciler"
@@ -33,7 +34,24 @@ func Run(ctx context.Context, opts ...Option) error {
 		return fmt.Errorf("init api: %w", err)
 	}
 
-	src, err := bootstrap.New([]byte(c.cfg.apiToken), c.cfg.bootstrapConfigFileName, c.api)
+	var writer writers.ConfigWriter
+	if c.cfg.bootstrapConfigFileName != nil {
+		var err error
+		var uri string
+		if c.cfg.bootstrapConfigWritebackURI != nil {
+			// if there is an explicitly configured writeback URI, use it
+			uri = *c.cfg.bootstrapConfigWritebackURI
+		} else {
+			// otherwise, default to "file://<filename>"
+			uri = "file://" + *c.cfg.bootstrapConfigFileName
+		}
+		writer, err = writers.NewForURI(uri)
+		if err != nil {
+			return fmt.Errorf("error creating bootstrap config writer: %w", err)
+		}
+	}
+
+	src, err := bootstrap.New([]byte(c.cfg.apiToken), c.cfg.bootstrapConfigFileName, writer, c.api)
 	if err != nil {
 		return fmt.Errorf("error creating bootstrap config: %w", err)
 	}