wip

config/config.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"net/url"
 
+	"github.com/pomerium/pomerium/config/envoyconfig/filemgr"
 	"github.com/pomerium/pomerium/internal/fileutil"
 	"github.com/pomerium/pomerium/internal/hashutil"
 	"github.com/pomerium/pomerium/internal/httputil"
@@ -87,6 +88,28 @@ func (cfg *Config) Clone() *Config {
 	}
 }
 
+// AllCertificateAuthoritiesSource returns a filemgr source from the certificate authorities.
+func (cfg *Config) AllCertificateAuthoritiesSource() (filemgr.Source, error) {
+	var sources []filemgr.Source
+	if cfg.Options.CA != "" {
+		bs, err := base64.StdEncoding.DecodeString(cfg.Options.CA)
+		if err != nil {
+			return nil, err
+		}
+		sources = append(sources, filemgr.BytesSource("ca.pem", bs))
+	}
+
+	if cfg.Options.CAFile != "" {
+		sources = append(sources, filemgr.FileSource(cfg.Options.CAFile))
+	}
+
+	if cfg.DerivedCAPEM != nil {
+		sources = append(sources, filemgr.BytesSource("ca.pem", cfg.DerivedCAPEM))
+	}
+
+	return filemgr.MultiSource("ca.pem", []byte("\n"), sources...), nil
+}
+
 // AllCertificateAuthoritiesPEM returns all CAs as PEM bundle bytes
 func (cfg *Config) AllCertificateAuthoritiesPEM() ([]byte, error) {
 	var combined bytes.Buffer

config/envoyconfig/clusters.go

@@ -239,12 +239,7 @@ func (b *Builder) buildInternalTransportSocket(
 		MatchTypedSubjectAltNames: []*envoy_extensions_transport_sockets_tls_v3.SubjectAltNameMatcher{
 			b.buildSubjectAltNameMatcher(endpoint, cfg.Options.OverrideCertificateName),
 		},
-	}
+		TrustedCa: b.buildTrustedCA(ctx, cfg),
-	bs, err := getCombinedCertificateAuthority(ctx, cfg)
-	if err != nil {
-		log.Ctx(ctx).Error().Err(err).Msg("unable to enable certificate verification because no root CAs were found")
-	} else {
-		validationContext.TrustedCa = b.filemgr.BytesDataSource("ca.pem", bs)
 	}
 	tlsContext := &envoy_extensions_transport_sockets_tls_v3.UpstreamTlsContext{
 		CommonTlsContext: &envoy_extensions_transport_sockets_tls_v3.CommonTlsContext{
@@ -340,12 +335,7 @@ func (b *Builder) buildPolicyValidationContext(
 		}
 		validationContext.TrustedCa = b.filemgr.BytesDataSource("custom-ca.pem", bs)
 	} else {
-		bs, err := getCombinedCertificateAuthority(ctx, cfg)
+		validationContext.TrustedCa = b.buildTrustedCA(ctx, cfg)
-		if err != nil {
-			log.Ctx(ctx).Error().Err(err).Msg("unable to enable certificate verification because no root CAs were found")
-		} else {
-			validationContext.TrustedCa = b.filemgr.BytesDataSource("ca.pem", bs)
-		}
 	}
 
 	if policy.TLSSkipVerify {

config/envoyconfig/clusters_test.go

@@ -39,15 +39,13 @@ func Test_buildPolicyTransportSocket(t *testing.T) {
 	customCA := filepath.Join(cacheDir, "pomerium", "envoy", "files", "custom-ca-3133535332543131503345494c.pem")
 
 	b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
-	rootCABytes, _ := getCombinedCertificateAuthority(ctx, &config.Config{Options: &config.Options{}})
+	rootCA := b.buildTrustedCA(ctx, &config.Config{Options: &config.Options{}})
-	rootCA := b.filemgr.BytesDataSource("ca.pem", rootCABytes).GetFilename()
 
 	o1 := config.NewDefaultOptions()
 	o2 := config.NewDefaultOptions()
 	o2.CA = base64.StdEncoding.EncodeToString([]byte{0, 0, 0, 0})
 
-	combinedCABytes, _ := getCombinedCertificateAuthority(ctx, &config.Config{Options: &config.Options{CA: o2.CA}})
+	combinedCA := b.buildTrustedCA(ctx, &config.Config{Options: &config.Options{CA: o2.CA}})
-	combinedCA := b.filemgr.BytesDataSource("ca.pem", combinedCABytes).GetFilename()
 
 	t.Run("insecure", func(t *testing.T) {
 		ts, err := b.buildPolicyTransportSocket(ctx, &config.Config{Options: o1}, &config.Policy{
@@ -102,7 +100,7 @@ func Test_buildPolicyTransportSocket(t *testing.T) {
 								}
 							}],
 							"trustedCa": {
-								"filename": "`+rootCA+`"
+								"filename": "`+rootCA.GetFilename()+`"
 							}
 						}
 					},
@@ -158,7 +156,7 @@ func Test_buildPolicyTransportSocket(t *testing.T) {
 								}
 							}],
 							"trustedCa": {
-								"filename": "`+rootCA+`"
+								"filename": "`+rootCA.GetFilename()+`"
 							}
 						}
 					},
@@ -214,7 +212,7 @@ func Test_buildPolicyTransportSocket(t *testing.T) {
 								}
 							}],
 							"trustedCa": {
-								"filename": "`+rootCA+`"
+								"filename": "`+rootCA.GetFilename()+`"
 							}
 						}
 					},
@@ -270,7 +268,7 @@ func Test_buildPolicyTransportSocket(t *testing.T) {
 								}
 							}],
 							"trustedCa": {
-								"filename": "`+rootCA+`"
+								"filename": "`+rootCA.GetFilename()+`"
 							},
 							"trustChainVerification": "ACCEPT_UNTRUSTED"
 						}
@@ -382,7 +380,7 @@ func Test_buildPolicyTransportSocket(t *testing.T) {
 								}
 							}],
 							"trustedCa": {
-								"filename": "`+combinedCA+`"
+								"filename": "`+combinedCA.GetFilename()+`"
 							}
 						}
 					},
@@ -447,7 +445,7 @@ func Test_buildPolicyTransportSocket(t *testing.T) {
 								}
 							}],
 							"trustedCa": {
-								"filename": "`+rootCA+`"
+								"filename": "`+rootCA.GetFilename()+`"
 							}
 						}
 					},
@@ -504,7 +502,7 @@ func Test_buildPolicyTransportSocket(t *testing.T) {
 								}
 							}],
 							"trustedCa": {
-								"filename": "`+rootCA+`"
+								"filename": "`+rootCA.GetFilename()+`"
 							}
 						}
 					},
@@ -518,8 +516,7 @@ func Test_buildPolicyTransportSocket(t *testing.T) {
 func Test_buildCluster(t *testing.T) {
 	ctx := context.Background()
 	b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
-	rootCABytes, _ := getCombinedCertificateAuthority(ctx, &config.Config{Options: &config.Options{}})
+	rootCA := b.buildTrustedCA(ctx, &config.Config{Options: &config.Options{}})
-	rootCA := b.filemgr.BytesDataSource("ca.pem", rootCABytes).GetFilename()
 	o1 := config.NewDefaultOptions()
 	t.Run("insecure", func(t *testing.T) {
 		endpoints, err := b.buildPolicyEndpoints(ctx, &config.Config{Options: o1}, &config.Policy{
@@ -643,7 +640,7 @@ func Test_buildCluster(t *testing.T) {
 										}
 									}],
 									"trustedCa": {
-										"filename": "`+rootCA+`"
+										"filename": "`+rootCA.GetFilename()+`"
 									}
 								}
 							},
@@ -691,7 +688,7 @@ func Test_buildCluster(t *testing.T) {
 									}
 								}],
 								"trustedCa": {
-									"filename": "`+rootCA+`"
+									"filename": "`+rootCA.GetFilename()+`"
 								}
 							}
 						},

config/envoyconfig/envoyconfig.go

@@ -2,7 +2,6 @@
 package envoyconfig
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -26,7 +25,6 @@ import (
 	"google.golang.org/protobuf/types/known/wrapperspb"
 
 	"github.com/pomerium/pomerium/config"
-	"github.com/pomerium/pomerium/internal/fileutil"
 	"github.com/pomerium/pomerium/internal/httputil"
 	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/pkg/cryptutil"
@@ -191,27 +189,6 @@ func getRootCertificateAuthority(ctx context.Context) (string, error) {
 	return rootCABundle.value, nil
 }
 
-func getCombinedCertificateAuthority(ctx context.Context, cfg *config.Config) ([]byte, error) {
-	rootFile, err := getRootCertificateAuthority(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	var buf bytes.Buffer
-	if err := fileutil.CopyFileUpTo(&buf, rootFile, 5<<20); err != nil {
-		return nil, fmt.Errorf("error reading root certificates: %w", err)
-	}
-	buf.WriteRune('\n')
-
-	all, err := cfg.AllCertificateAuthoritiesPEM()
-	if err != nil {
-		return nil, fmt.Errorf("get all CA: %w", err)
-	}
-	buf.Write(all)
-
-	return buf.Bytes(), nil
-}
-
 func marshalAny(msg proto.Message) *anypb.Any {
 	data := new(anypb.Any)
 	_ = anypb.MarshalFrom(data, msg, proto.MarshalOptions{

config/envoyconfig/filemgr/filemgr.go

@@ -2,6 +2,7 @@
 package filemgr
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"sync"
@@ -33,6 +34,52 @@ func (mgr *Manager) init() {
 	})
 }
 
+// DataSource returns an envoy config data source from the given source.
+func (mgr *Manager) DataSource(source Source) (*envoy_config_core_v3.DataSource, error) {
+	mgr.init()
+	if mgr.initErr != nil {
+		return nil, fmt.Errorf("filemgr: error creating cache directory: %w", mgr.initErr)
+	}
+
+	n, err := source.Checksum()
+	if err != nil {
+		return nil, fmt.Errorf("filemgr: error computing checksum: %w", err)
+	}
+
+	fileName := GetFileNameWithChecksum(source.FileName(), n)
+	filePath := filepath.Join(mgr.cfg.cacheDir, fileName)
+
+	// write file if it doesn't exist
+	if _, err := os.Stat(filePath); os.IsNotExist(err) {
+		tmpFilePath := filePath + ".tmp"
+		f, err := os.Create(tmpFilePath)
+		if err != nil {
+			return nil, fmt.Errorf("filemgr: error creating temporary file: %w", err)
+		}
+
+		_, err = source.WriteTo(f)
+		if err != nil {
+			_ = f.Close()
+			return nil, fmt.Errorf("filemgr: error writing temporary file: %w", err)
+		}
+
+		err = f.Close()
+		if err != nil {
+			return nil, fmt.Errorf("filemgr: error closing temporary file: %w", err)
+		}
+
+		err = os.Rename(tmpFilePath, filePath)
+		if err != nil {
+			_ = os.Remove(tmpFilePath) // delete the temporary file
+			return nil, fmt.Errorf("filemgr: error renaming temporary file: %w", err)
+		}
+	} else if err != nil {
+		return nil, fmt.Errorf("filemgr: error reading cache file: %w", err)
+	}
+
+	return inlineFilename(filePath), nil
+}
+
 // BytesDataSource returns an envoy config data source based on bytes.
 func (mgr *Manager) BytesDataSource(fileName string, data []byte) *envoy_config_core_v3.DataSource {
 	mgr.init()

config/envoyconfig/filemgr/filemgr_test.go

@@ -7,6 +7,7 @@ import (
 
 	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func Test(t *testing.T) {
@@ -50,3 +51,46 @@ func Test(t *testing.T) {
 		mgr.ClearCache()
 	})
 }
+
+func Test_Source(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "test1.txt"), []byte("TEST-1"), 0o600))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "test3.txt"), []byte("TEST-3"), 0o600))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "test5.txt"), []byte("TEST-5"), 0o600))
+
+	src := MultiSource("combined.txt", []byte{'|'},
+		FileSource(filepath.Join(dir, "test1.txt")),
+		BytesSource("test2.txt", []byte("TEST-2")),
+		FileSource(filepath.Join(dir, "test3.txt")),
+		BytesSource("test4.txt", []byte("TEST-4")),
+		FileSource(filepath.Join(dir, "test5.txt")),
+	)
+	n, err := src.Checksum()
+	assert.NoError(t, err)
+
+	combinedFilePath := filepath.Join(dir, GetFileNameWithChecksum("combined.txt", n))
+
+	mgr := NewManager(WithCacheDir(dir))
+	ds, err := mgr.DataSource(src)
+	assert.NoError(t, err)
+	assert.Equal(t, &envoy_config_core_v3.DataSource{
+		Specifier: &envoy_config_core_v3.DataSource_Filename{
+			Filename: combinedFilePath,
+		},
+	}, ds)
+
+	ds, err = mgr.DataSource(src)
+	assert.NoError(t, err)
+	assert.Equal(t, &envoy_config_core_v3.DataSource{
+		Specifier: &envoy_config_core_v3.DataSource_Filename{
+			Filename: combinedFilePath,
+		},
+	}, ds)
+
+	bs, err := os.ReadFile(combinedFilePath)
+	assert.NoError(t, err)
+	assert.Equal(t, "TEST-1|TEST-2|TEST-3|TEST-4|TEST-5", string(bs))
+}

config/envoyconfig/filemgr/name.go

@@ -16,3 +16,11 @@ func GetFileNameWithBytesHash(base string, data []byte) string {
 	ext := filepath.Ext(base)
 	return fmt.Sprintf("%s-%x%s", base[:len(base)-len(ext)], he, ext)
 }
+
+// GetFileNameWithChecksum constructs a filename using a base filename and a checksum.
+// For example: GetFileNameWithBytesHash("example.txt", 1234) ==> "example-1234.txt"
+func GetFileNameWithChecksum(base string, checksum uint64) string {
+	he := base36.Encode(checksum)
+	ext := filepath.Ext(base)
+	return fmt.Sprintf("%s-%x%s", base[:len(base)-len(ext)], he, ext)
+}

config/envoyconfig/filemgr/source.go

@@ -0,0 +1,133 @@
+package filemgr
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/zeebo/xxh3"
+
+	"github.com/pomerium/pomerium/internal/fileutil"
+	"github.com/pomerium/pomerium/internal/hashutil"
+)
+
+// A Source is a data source that can write bytes to a destination and has an associated
+// file name and checksum.
+type Source interface {
+	FileName() string
+	Checksum() (uint64, error)
+	io.WriterTo
+}
+
+type bytesSource struct {
+	fileName string
+	data     []byte
+}
+
+// BytesSource creates a source from a slice of bytes.
+func BytesSource(fileName string, data []byte) Source {
+	return bytesSource{
+		fileName: fileName,
+		data:     data,
+	}
+}
+
+func (s bytesSource) FileName() string {
+	return s.fileName
+}
+
+func (s bytesSource) Checksum() (uint64, error) {
+	return xxh3.HashSeed(s.data, 7546535), nil
+}
+
+func (s bytesSource) WriteTo(dst io.Writer) (int64, error) {
+	n, err := dst.Write(s.data)
+	return int64(n), err
+}
+
+type fileSource struct {
+	filePath string
+}
+
+// FileSource creates a source from a file.
+func FileSource(filePath string) Source {
+	return fileSource{
+		filePath: filePath,
+	}
+}
+
+func (s fileSource) FileName() string {
+	return filepath.Base(s.filePath)
+}
+
+func (s fileSource) Checksum() (uint64, error) {
+	return fileutil.StatCheckSum(s.filePath)
+}
+
+func (s fileSource) WriteTo(dst io.Writer) (int64, error) {
+	f, err := os.Open(s.filePath)
+	if err != nil {
+		return 0, err
+	}
+
+	n, err := f.WriteTo(dst)
+	if err != nil {
+		_ = f.Close()
+		return n, err
+	}
+
+	return n, f.Close()
+}
+
+type multiSource struct {
+	fileName  string
+	separator []byte
+	sources   []Source
+}
+
+// MultiSource creates a source from multiple sources. Each source is concatenated together
+// with the separator between them. The Checksum is computed from each of the source
+// checksums.
+func MultiSource(fileName string, separator []byte, sources ...Source) Source {
+	return &multiSource{
+		fileName:  fileName,
+		separator: separator,
+		sources:   sources,
+	}
+}
+
+func (s *multiSource) FileName() string {
+	return s.fileName
+}
+
+func (s *multiSource) Checksum() (uint64, error) {
+	h := hashutil.NewDigestWithSeed(4616647)
+	_, _ = h.Write(s.separator)
+	for _, ss := range s.sources {
+		n, err := ss.Checksum()
+		if err != nil {
+			return 0, err
+		}
+		h.WriteUint64(n)
+	}
+	return h.Sum64(), nil
+}
+
+func (s *multiSource) WriteTo(dst io.Writer) (int64, error) {
+	var total int64
+	for i, ss := range s.sources {
+		if i > 0 {
+			n, err := dst.Write(s.separator)
+			if err != nil {
+				return 0, err
+			}
+			total += int64(n)
+		}
+		n, err := ss.WriteTo(dst)
+		if err != nil {
+			return 0, err
+		}
+		total += n
+	}
+	return total, nil
+}

config/envoyconfig/tls.go

@@ -20,6 +20,7 @@ import (
 	"google.golang.org/protobuf/types/known/wrapperspb"
 
 	"github.com/pomerium/pomerium/config"
+	"github.com/pomerium/pomerium/config/envoyconfig/filemgr"
 	"github.com/pomerium/pomerium/internal/log"
 )
 
@@ -351,3 +352,25 @@ func (b *Builder) buildDownstreamValidationContext(
 		ValidationContext: vc,
 	}
 }
+
+func (b *Builder) buildTrustedCA(ctx context.Context, cfg *config.Config) *envoy_config_core_v3.DataSource {
+	rootFile, err := getRootCertificateAuthority(ctx)
+	if err != nil {
+		log.Ctx(ctx).Error().Err(err).Msg("unable to enable certificate verification because no root CAs were found")
+		return nil
+	}
+
+	src, err := cfg.AllCertificateAuthoritiesSource()
+	if err != nil {
+		log.Ctx(ctx).Error().Err(err).Msg("unable to enable certificate verification, invalid config")
+		return nil
+	}
+
+	ds, err := b.filemgr.DataSource(filemgr.MultiSource("ca.pem", []byte{'\n'}, filemgr.FileSource(rootFile), src))
+	if err != nil {
+		log.Ctx(ctx).Error().Err(err).Msg("unable to enable certificate verification, error loading CAs")
+		return nil
+	}
+
+	return ds
+}

internal/fileutil/checksum.go

@@ -0,0 +1,39 @@
+package fileutil
+
+import (
+	"errors"
+	"io/fs"
+	"os"
+	"syscall"
+
+	"github.com/pomerium/pomerium/internal/hashutil"
+)
+
+// StatCheckSum returns a checksum of the file info. It is valid to run this
+// function against a file path that doesn't exist and an error will not be returned.
+// The file path, size, modification time, device id and inode id are used to compute
+// the hash. Any change to this data, even if the underlying file contents are the
+// same, will result in a new checksum, and vice-versa, if the underlying contents
+// change, but none of the other data does, the checksum will be the same.
+func StatCheckSum(filePath string) (uint64, error) {
+	d := hashutil.NewDigestWithSeed(7968108)
+	d.WriteStringWithLen(filePath)
+
+	for _, fn := range []func(string) (os.FileInfo, error){os.Stat, os.Lstat} {
+		fi, err := fn(filePath)
+		if errors.Is(err, fs.ErrNotExist) {
+			_, _ = d.Write([]byte{0})
+		} else if err != nil {
+			return 0, err
+		} else {
+			d.WriteInt64(fi.Size())
+			d.WriteInt64(fi.ModTime().Unix())
+			if s, ok := fi.Sys().(*syscall.Stat_t); ok {
+				d.WriteUint64(s.Dev)
+				d.WriteUint64(s.Ino)
+			}
+		}
+	}
+
+	return d.Sum64(), nil
+}

internal/hashutil/hashutil.go

@@ -40,6 +40,14 @@ func NewDigest() *Digest {
 	return &d
 }
 
+// NewDigestWithSeed creates a new digest using the given seed.
+func NewDigestWithSeed(seed uint64) *Digest {
+	var d Digest
+	d.Hasher = *xxh3.NewSeed(seed)
+	d.Reset()
+	return &d
+}
+
 // WriteStringWithLen writes the string's length, then its contents to the hash.
 func (d *Digest) WriteStringWithLen(s string) {
 	d.WriteInt32(int32(len(s)))