From 0e258a9ed4a1aa6fac1ffdbd9d64dac03c763b51 Mon Sep 17 00:00:00 2001 From: Caleb Doxsey Date: Mon, 10 Feb 2025 13:04:27 -0700 Subject: [PATCH] wip --- config/config.go | 23 ++++ config/envoyconfig/clusters.go | 14 +-- config/envoyconfig/clusters_test.go | 27 ++--- config/envoyconfig/envoyconfig.go | 23 ---- config/envoyconfig/filemgr/filemgr.go | 47 ++++++++ config/envoyconfig/filemgr/filemgr_test.go | 44 +++++++ config/envoyconfig/filemgr/name.go | 8 ++ config/envoyconfig/filemgr/source.go | 133 +++++++++++++++++++++ config/envoyconfig/tls.go | 23 ++++ internal/fileutil/checksum.go | 39 ++++++ internal/hashutil/hashutil.go | 8 ++ 11 files changed, 339 insertions(+), 50 deletions(-) create mode 100644 config/envoyconfig/filemgr/source.go create mode 100644 internal/fileutil/checksum.go diff --git a/config/config.go b/config/config.go index 14a51b2e4..fb7a07044 100644 --- a/config/config.go +++ b/config/config.go @@ -9,6 +9,7 @@ import ( "net/http" "net/url" + "github.com/pomerium/pomerium/config/envoyconfig/filemgr" "github.com/pomerium/pomerium/internal/fileutil" "github.com/pomerium/pomerium/internal/hashutil" "github.com/pomerium/pomerium/internal/httputil" @@ -87,6 +88,28 @@ func (cfg *Config) Clone() *Config { } } +// AllCertificateAuthoritiesSource returns a filemgr source from the certificate authorities. +func (cfg *Config) AllCertificateAuthoritiesSource() (filemgr.Source, error) { + var sources []filemgr.Source + if cfg.Options.CA != "" { + bs, err := base64.StdEncoding.DecodeString(cfg.Options.CA) + if err != nil { + return nil, err + } + sources = append(sources, filemgr.BytesSource("ca.pem", bs)) + } + + if cfg.Options.CAFile != "" { + sources = append(sources, filemgr.FileSource(cfg.Options.CAFile)) + } + + if cfg.DerivedCAPEM != nil { + sources = append(sources, filemgr.BytesSource("ca.pem", cfg.DerivedCAPEM)) + } + + return filemgr.MultiSource("ca.pem", []byte("\n"), sources...), nil +} + // AllCertificateAuthoritiesPEM returns all CAs as PEM bundle bytes func (cfg *Config) AllCertificateAuthoritiesPEM() ([]byte, error) { var combined bytes.Buffer diff --git a/config/envoyconfig/clusters.go b/config/envoyconfig/clusters.go index bfb083cc5..8a29e071c 100644 --- a/config/envoyconfig/clusters.go +++ b/config/envoyconfig/clusters.go @@ -239,12 +239,7 @@ func (b *Builder) buildInternalTransportSocket( MatchTypedSubjectAltNames: []*envoy_extensions_transport_sockets_tls_v3.SubjectAltNameMatcher{ b.buildSubjectAltNameMatcher(endpoint, cfg.Options.OverrideCertificateName), }, - } - bs, err := getCombinedCertificateAuthority(ctx, cfg) - if err != nil { - log.Ctx(ctx).Error().Err(err).Msg("unable to enable certificate verification because no root CAs were found") - } else { - validationContext.TrustedCa = b.filemgr.BytesDataSource("ca.pem", bs) + TrustedCa: b.buildTrustedCA(ctx, cfg), } tlsContext := &envoy_extensions_transport_sockets_tls_v3.UpstreamTlsContext{ CommonTlsContext: &envoy_extensions_transport_sockets_tls_v3.CommonTlsContext{ @@ -340,12 +335,7 @@ func (b *Builder) buildPolicyValidationContext( } validationContext.TrustedCa = b.filemgr.BytesDataSource("custom-ca.pem", bs) } else { - bs, err := getCombinedCertificateAuthority(ctx, cfg) - if err != nil { - log.Ctx(ctx).Error().Err(err).Msg("unable to enable certificate verification because no root CAs were found") - } else { - validationContext.TrustedCa = b.filemgr.BytesDataSource("ca.pem", bs) - } + validationContext.TrustedCa = b.buildTrustedCA(ctx, cfg) } if policy.TLSSkipVerify { diff --git a/config/envoyconfig/clusters_test.go b/config/envoyconfig/clusters_test.go index 91ed0828a..f9f3d929a 100644 --- a/config/envoyconfig/clusters_test.go +++ b/config/envoyconfig/clusters_test.go @@ -39,15 +39,13 @@ func Test_buildPolicyTransportSocket(t *testing.T) { customCA := filepath.Join(cacheDir, "pomerium", "envoy", "files", "custom-ca-3133535332543131503345494c.pem") b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil) - rootCABytes, _ := getCombinedCertificateAuthority(ctx, &config.Config{Options: &config.Options{}}) - rootCA := b.filemgr.BytesDataSource("ca.pem", rootCABytes).GetFilename() + rootCA := b.buildTrustedCA(ctx, &config.Config{Options: &config.Options{}}) o1 := config.NewDefaultOptions() o2 := config.NewDefaultOptions() o2.CA = base64.StdEncoding.EncodeToString([]byte{0, 0, 0, 0}) - combinedCABytes, _ := getCombinedCertificateAuthority(ctx, &config.Config{Options: &config.Options{CA: o2.CA}}) - combinedCA := b.filemgr.BytesDataSource("ca.pem", combinedCABytes).GetFilename() + combinedCA := b.buildTrustedCA(ctx, &config.Config{Options: &config.Options{CA: o2.CA}}) t.Run("insecure", func(t *testing.T) { ts, err := b.buildPolicyTransportSocket(ctx, &config.Config{Options: o1}, &config.Policy{ @@ -102,7 +100,7 @@ func Test_buildPolicyTransportSocket(t *testing.T) { } }], "trustedCa": { - "filename": "`+rootCA+`" + "filename": "`+rootCA.GetFilename()+`" } } }, @@ -158,7 +156,7 @@ func Test_buildPolicyTransportSocket(t *testing.T) { } }], "trustedCa": { - "filename": "`+rootCA+`" + "filename": "`+rootCA.GetFilename()+`" } } }, @@ -214,7 +212,7 @@ func Test_buildPolicyTransportSocket(t *testing.T) { } }], "trustedCa": { - "filename": "`+rootCA+`" + "filename": "`+rootCA.GetFilename()+`" } } }, @@ -270,7 +268,7 @@ func Test_buildPolicyTransportSocket(t *testing.T) { } }], "trustedCa": { - "filename": "`+rootCA+`" + "filename": "`+rootCA.GetFilename()+`" }, "trustChainVerification": "ACCEPT_UNTRUSTED" } @@ -382,7 +380,7 @@ func Test_buildPolicyTransportSocket(t *testing.T) { } }], "trustedCa": { - "filename": "`+combinedCA+`" + "filename": "`+combinedCA.GetFilename()+`" } } }, @@ -447,7 +445,7 @@ func Test_buildPolicyTransportSocket(t *testing.T) { } }], "trustedCa": { - "filename": "`+rootCA+`" + "filename": "`+rootCA.GetFilename()+`" } } }, @@ -504,7 +502,7 @@ func Test_buildPolicyTransportSocket(t *testing.T) { } }], "trustedCa": { - "filename": "`+rootCA+`" + "filename": "`+rootCA.GetFilename()+`" } } }, @@ -518,8 +516,7 @@ func Test_buildPolicyTransportSocket(t *testing.T) { func Test_buildCluster(t *testing.T) { ctx := context.Background() b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil) - rootCABytes, _ := getCombinedCertificateAuthority(ctx, &config.Config{Options: &config.Options{}}) - rootCA := b.filemgr.BytesDataSource("ca.pem", rootCABytes).GetFilename() + rootCA := b.buildTrustedCA(ctx, &config.Config{Options: &config.Options{}}) o1 := config.NewDefaultOptions() t.Run("insecure", func(t *testing.T) { endpoints, err := b.buildPolicyEndpoints(ctx, &config.Config{Options: o1}, &config.Policy{ @@ -643,7 +640,7 @@ func Test_buildCluster(t *testing.T) { } }], "trustedCa": { - "filename": "`+rootCA+`" + "filename": "`+rootCA.GetFilename()+`" } } }, @@ -691,7 +688,7 @@ func Test_buildCluster(t *testing.T) { } }], "trustedCa": { - "filename": "`+rootCA+`" + "filename": "`+rootCA.GetFilename()+`" } } }, diff --git a/config/envoyconfig/envoyconfig.go b/config/envoyconfig/envoyconfig.go index 571112d41..f2185d163 100644 --- a/config/envoyconfig/envoyconfig.go +++ b/config/envoyconfig/envoyconfig.go @@ -2,7 +2,6 @@ package envoyconfig import ( - "bytes" "context" "errors" "fmt" @@ -26,7 +25,6 @@ import ( "google.golang.org/protobuf/types/known/wrapperspb" "github.com/pomerium/pomerium/config" - "github.com/pomerium/pomerium/internal/fileutil" "github.com/pomerium/pomerium/internal/httputil" "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/pkg/cryptutil" @@ -191,27 +189,6 @@ func getRootCertificateAuthority(ctx context.Context) (string, error) { return rootCABundle.value, nil } -func getCombinedCertificateAuthority(ctx context.Context, cfg *config.Config) ([]byte, error) { - rootFile, err := getRootCertificateAuthority(ctx) - if err != nil { - return nil, err - } - - var buf bytes.Buffer - if err := fileutil.CopyFileUpTo(&buf, rootFile, 5<<20); err != nil { - return nil, fmt.Errorf("error reading root certificates: %w", err) - } - buf.WriteRune('\n') - - all, err := cfg.AllCertificateAuthoritiesPEM() - if err != nil { - return nil, fmt.Errorf("get all CA: %w", err) - } - buf.Write(all) - - return buf.Bytes(), nil -} - func marshalAny(msg proto.Message) *anypb.Any { data := new(anypb.Any) _ = anypb.MarshalFrom(data, msg, proto.MarshalOptions{ diff --git a/config/envoyconfig/filemgr/filemgr.go b/config/envoyconfig/filemgr/filemgr.go index cd1ccdbb1..73e6b4fd6 100644 --- a/config/envoyconfig/filemgr/filemgr.go +++ b/config/envoyconfig/filemgr/filemgr.go @@ -2,6 +2,7 @@ package filemgr import ( + "fmt" "os" "path/filepath" "sync" @@ -33,6 +34,52 @@ func (mgr *Manager) init() { }) } +// DataSource returns an envoy config data source from the given source. +func (mgr *Manager) DataSource(source Source) (*envoy_config_core_v3.DataSource, error) { + mgr.init() + if mgr.initErr != nil { + return nil, fmt.Errorf("filemgr: error creating cache directory: %w", mgr.initErr) + } + + n, err := source.Checksum() + if err != nil { + return nil, fmt.Errorf("filemgr: error computing checksum: %w", err) + } + + fileName := GetFileNameWithChecksum(source.FileName(), n) + filePath := filepath.Join(mgr.cfg.cacheDir, fileName) + + // write file if it doesn't exist + if _, err := os.Stat(filePath); os.IsNotExist(err) { + tmpFilePath := filePath + ".tmp" + f, err := os.Create(tmpFilePath) + if err != nil { + return nil, fmt.Errorf("filemgr: error creating temporary file: %w", err) + } + + _, err = source.WriteTo(f) + if err != nil { + _ = f.Close() + return nil, fmt.Errorf("filemgr: error writing temporary file: %w", err) + } + + err = f.Close() + if err != nil { + return nil, fmt.Errorf("filemgr: error closing temporary file: %w", err) + } + + err = os.Rename(tmpFilePath, filePath) + if err != nil { + _ = os.Remove(tmpFilePath) // delete the temporary file + return nil, fmt.Errorf("filemgr: error renaming temporary file: %w", err) + } + } else if err != nil { + return nil, fmt.Errorf("filemgr: error reading cache file: %w", err) + } + + return inlineFilename(filePath), nil +} + // BytesDataSource returns an envoy config data source based on bytes. func (mgr *Manager) BytesDataSource(fileName string, data []byte) *envoy_config_core_v3.DataSource { mgr.init() diff --git a/config/envoyconfig/filemgr/filemgr_test.go b/config/envoyconfig/filemgr/filemgr_test.go index 653031eaf..d073092b7 100644 --- a/config/envoyconfig/filemgr/filemgr_test.go +++ b/config/envoyconfig/filemgr/filemgr_test.go @@ -7,6 +7,7 @@ import ( envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test(t *testing.T) { @@ -50,3 +51,46 @@ func Test(t *testing.T) { mgr.ClearCache() }) } + +func Test_Source(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + require.NoError(t, os.WriteFile(filepath.Join(dir, "test1.txt"), []byte("TEST-1"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "test3.txt"), []byte("TEST-3"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "test5.txt"), []byte("TEST-5"), 0o600)) + + src := MultiSource("combined.txt", []byte{'|'}, + FileSource(filepath.Join(dir, "test1.txt")), + BytesSource("test2.txt", []byte("TEST-2")), + FileSource(filepath.Join(dir, "test3.txt")), + BytesSource("test4.txt", []byte("TEST-4")), + FileSource(filepath.Join(dir, "test5.txt")), + ) + n, err := src.Checksum() + assert.NoError(t, err) + + combinedFilePath := filepath.Join(dir, GetFileNameWithChecksum("combined.txt", n)) + + mgr := NewManager(WithCacheDir(dir)) + ds, err := mgr.DataSource(src) + assert.NoError(t, err) + assert.Equal(t, &envoy_config_core_v3.DataSource{ + Specifier: &envoy_config_core_v3.DataSource_Filename{ + Filename: combinedFilePath, + }, + }, ds) + + ds, err = mgr.DataSource(src) + assert.NoError(t, err) + assert.Equal(t, &envoy_config_core_v3.DataSource{ + Specifier: &envoy_config_core_v3.DataSource_Filename{ + Filename: combinedFilePath, + }, + }, ds) + + bs, err := os.ReadFile(combinedFilePath) + assert.NoError(t, err) + assert.Equal(t, "TEST-1|TEST-2|TEST-3|TEST-4|TEST-5", string(bs)) +} diff --git a/config/envoyconfig/filemgr/name.go b/config/envoyconfig/filemgr/name.go index 91fd55db6..05626844c 100644 --- a/config/envoyconfig/filemgr/name.go +++ b/config/envoyconfig/filemgr/name.go @@ -16,3 +16,11 @@ func GetFileNameWithBytesHash(base string, data []byte) string { ext := filepath.Ext(base) return fmt.Sprintf("%s-%x%s", base[:len(base)-len(ext)], he, ext) } + +// GetFileNameWithChecksum constructs a filename using a base filename and a checksum. +// For example: GetFileNameWithBytesHash("example.txt", 1234) ==> "example-1234.txt" +func GetFileNameWithChecksum(base string, checksum uint64) string { + he := base36.Encode(checksum) + ext := filepath.Ext(base) + return fmt.Sprintf("%s-%x%s", base[:len(base)-len(ext)], he, ext) +} diff --git a/config/envoyconfig/filemgr/source.go b/config/envoyconfig/filemgr/source.go new file mode 100644 index 000000000..481795243 --- /dev/null +++ b/config/envoyconfig/filemgr/source.go @@ -0,0 +1,133 @@ +package filemgr + +import ( + "io" + "os" + "path/filepath" + + "github.com/zeebo/xxh3" + + "github.com/pomerium/pomerium/internal/fileutil" + "github.com/pomerium/pomerium/internal/hashutil" +) + +// A Source is a data source that can write bytes to a destination and has an associated +// file name and checksum. +type Source interface { + FileName() string + Checksum() (uint64, error) + io.WriterTo +} + +type bytesSource struct { + fileName string + data []byte +} + +// BytesSource creates a source from a slice of bytes. +func BytesSource(fileName string, data []byte) Source { + return bytesSource{ + fileName: fileName, + data: data, + } +} + +func (s bytesSource) FileName() string { + return s.fileName +} + +func (s bytesSource) Checksum() (uint64, error) { + return xxh3.HashSeed(s.data, 7546535), nil +} + +func (s bytesSource) WriteTo(dst io.Writer) (int64, error) { + n, err := dst.Write(s.data) + return int64(n), err +} + +type fileSource struct { + filePath string +} + +// FileSource creates a source from a file. +func FileSource(filePath string) Source { + return fileSource{ + filePath: filePath, + } +} + +func (s fileSource) FileName() string { + return filepath.Base(s.filePath) +} + +func (s fileSource) Checksum() (uint64, error) { + return fileutil.StatCheckSum(s.filePath) +} + +func (s fileSource) WriteTo(dst io.Writer) (int64, error) { + f, err := os.Open(s.filePath) + if err != nil { + return 0, err + } + + n, err := f.WriteTo(dst) + if err != nil { + _ = f.Close() + return n, err + } + + return n, f.Close() +} + +type multiSource struct { + fileName string + separator []byte + sources []Source +} + +// MultiSource creates a source from multiple sources. Each source is concatenated together +// with the separator between them. The Checksum is computed from each of the source +// checksums. +func MultiSource(fileName string, separator []byte, sources ...Source) Source { + return &multiSource{ + fileName: fileName, + separator: separator, + sources: sources, + } +} + +func (s *multiSource) FileName() string { + return s.fileName +} + +func (s *multiSource) Checksum() (uint64, error) { + h := hashutil.NewDigestWithSeed(4616647) + _, _ = h.Write(s.separator) + for _, ss := range s.sources { + n, err := ss.Checksum() + if err != nil { + return 0, err + } + h.WriteUint64(n) + } + return h.Sum64(), nil +} + +func (s *multiSource) WriteTo(dst io.Writer) (int64, error) { + var total int64 + for i, ss := range s.sources { + if i > 0 { + n, err := dst.Write(s.separator) + if err != nil { + return 0, err + } + total += int64(n) + } + n, err := ss.WriteTo(dst) + if err != nil { + return 0, err + } + total += n + } + return total, nil +} diff --git a/config/envoyconfig/tls.go b/config/envoyconfig/tls.go index d35a13080..20df890f2 100644 --- a/config/envoyconfig/tls.go +++ b/config/envoyconfig/tls.go @@ -20,6 +20,7 @@ import ( "google.golang.org/protobuf/types/known/wrapperspb" "github.com/pomerium/pomerium/config" + "github.com/pomerium/pomerium/config/envoyconfig/filemgr" "github.com/pomerium/pomerium/internal/log" ) @@ -351,3 +352,25 @@ func (b *Builder) buildDownstreamValidationContext( ValidationContext: vc, } } + +func (b *Builder) buildTrustedCA(ctx context.Context, cfg *config.Config) *envoy_config_core_v3.DataSource { + rootFile, err := getRootCertificateAuthority(ctx) + if err != nil { + log.Ctx(ctx).Error().Err(err).Msg("unable to enable certificate verification because no root CAs were found") + return nil + } + + src, err := cfg.AllCertificateAuthoritiesSource() + if err != nil { + log.Ctx(ctx).Error().Err(err).Msg("unable to enable certificate verification, invalid config") + return nil + } + + ds, err := b.filemgr.DataSource(filemgr.MultiSource("ca.pem", []byte{'\n'}, filemgr.FileSource(rootFile), src)) + if err != nil { + log.Ctx(ctx).Error().Err(err).Msg("unable to enable certificate verification, error loading CAs") + return nil + } + + return ds +} diff --git a/internal/fileutil/checksum.go b/internal/fileutil/checksum.go new file mode 100644 index 000000000..a8701c439 --- /dev/null +++ b/internal/fileutil/checksum.go @@ -0,0 +1,39 @@ +package fileutil + +import ( + "errors" + "io/fs" + "os" + "syscall" + + "github.com/pomerium/pomerium/internal/hashutil" +) + +// StatCheckSum returns a checksum of the file info. It is valid to run this +// function against a file path that doesn't exist and an error will not be returned. +// The file path, size, modification time, device id and inode id are used to compute +// the hash. Any change to this data, even if the underlying file contents are the +// same, will result in a new checksum, and vice-versa, if the underlying contents +// change, but none of the other data does, the checksum will be the same. +func StatCheckSum(filePath string) (uint64, error) { + d := hashutil.NewDigestWithSeed(7968108) + d.WriteStringWithLen(filePath) + + for _, fn := range []func(string) (os.FileInfo, error){os.Stat, os.Lstat} { + fi, err := fn(filePath) + if errors.Is(err, fs.ErrNotExist) { + _, _ = d.Write([]byte{0}) + } else if err != nil { + return 0, err + } else { + d.WriteInt64(fi.Size()) + d.WriteInt64(fi.ModTime().Unix()) + if s, ok := fi.Sys().(*syscall.Stat_t); ok { + d.WriteUint64(s.Dev) + d.WriteUint64(s.Ino) + } + } + } + + return d.Sum64(), nil +} diff --git a/internal/hashutil/hashutil.go b/internal/hashutil/hashutil.go index 7f4587adc..e47b7e694 100644 --- a/internal/hashutil/hashutil.go +++ b/internal/hashutil/hashutil.go @@ -40,6 +40,14 @@ func NewDigest() *Digest { return &d } +// NewDigestWithSeed creates a new digest using the given seed. +func NewDigestWithSeed(seed uint64) *Digest { + var d Digest + d.Hasher = *xxh3.NewSeed(seed) + d.Reset() + return &d +} + // WriteStringWithLen writes the string's length, then its contents to the hash. func (d *Digest) WriteStringWithLen(s string) { d.WriteInt32(int32(len(s)))