3pp-mirror/pomerium
1
0
Fork 0
mirror of https://github.com/pomerium/pomerium.git synced 2025-04-29 18:36:30 +02:00
Code Issues Projects Releases Packages Wiki Activity

core/filemgr: use xxhash instead of sha512 for filenames (#4697)

Browse source
This commit is contained in:
Caleb Doxsey 2023-11-01 13:52:32 -06:00 • committed by GitHub
parent e0693e54f0
commit fd8cb18c44
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 64 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
config/envoyconfig/clusters_test.go
View file

@ -23,7 +23,7 @@ import (
func Test_buildPolicyTransportSocket(t *testing.T) {
ctx := context.Background()
cacheDir, _ := os.UserCacheDir()
customCA := filepath.Join(cacheDir, "pomerium", "envoy", "files", "custom-ca-32484c314b584447463735303142374c31414145374650305a525539554938594d524855353757313942494d473847535231.pem")
customCA := filepath.Join(cacheDir, "pomerium", "envoy", "files", "custom-ca-57394a4e5157303436544830.pem")
b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
rootCABytes, _ := getCombinedCertificateAuthority(&config.Config{Options: &config.Options{}})
@ -406,10 +406,10 @@ func Test_buildPolicyTransportSocket(t *testing.T) {
},
"tlsCertificates": [{
"certificateChain":{
"filename": "`+filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-crt-354e49305a5a39414a545530374e58454e48334148524c4e324258463837364355564c4e4532464b54355139495547514a38.pem")+`"
"filename": "`+filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-crt-32375a484d4f49594c4d374830.pem")+`"
},
"privateKey": {
"filename": "`+filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-key-3350415a38414e4e4a4655424e55393430474147324651433949384e485341334b5157364f424b4c5856365a545937383735.pem")+`"
"filename": "`+filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-key-33393156483053584631414836.pem")+`"
}
}],
"validationContext": {

24
config/envoyconfig/filemgr/filemgr.go
View file

@ -3,20 +3,21 @@ package filemgr
import (
"context"
"fmt"
"os"
"path/filepath"
"sync"
envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
"github.com/martinlindhe/base36"
"github.com/pomerium/pomerium/internal/log"
"github.com/pomerium/pomerium/pkg/cryptutil"
)
// A Manager manages files for envoy.
type Manager struct {
cfg *config
initOnce sync.Once
initErr error
}
// NewManager creates a new Manager.
@ -27,18 +28,23 @@ func NewManager(options ...Option) *Manager {
}
}
func (mgr *Manager) init() {
mgr.initOnce.Do(func() {
mgr.initErr = os.MkdirAll(mgr.cfg.cacheDir, 0o700)
})
}
// BytesDataSource returns an envoy config data source based on bytes.
func (mgr *Manager) BytesDataSource(fileName string, data []byte) *envoy_config_core_v3.DataSource {
h := base36.EncodeBytes(cryptutil.Hash("filemgr", data))
ext := filepath.Ext(fileName)
fileName = fmt.Sprintf("%s-%x%s", fileName[:len(fileName)-len(ext)], h, ext)
if err := os.MkdirAll(mgr.cfg.cacheDir, 0o700); err != nil {
log.Error(context.TODO()).Err(err).Msg("filemgr: error creating cache directory, falling back to inline bytes")
mgr.init()
if mgr.initErr != nil {
log.Error(context.Background()).Err(mgr.initErr).Msg("filemgr: error creating cache directory, falling back to inline bytes")
return inlineBytes(data)
}
fileName = GetFileNameWithBytesHash(fileName, data)
filePath := filepath.Join(mgr.cfg.cacheDir, fileName)
if _, err := os.Stat(filePath); os.IsNotExist(err) {
err = os.WriteFile(filePath, data, 0o600)
if err != nil {

6
config/envoyconfig/filemgr/filemgr_test.go
View file

@ -17,7 +17,7 @@ func Test(t *testing.T) {
ds := mgr.BytesDataSource("test.txt", []byte{1, 2, 3, 4, 5})
assert.Equal(t, &envoy_config_core_v3.DataSource{
Specifier: &envoy_config_core_v3.DataSource_Filename{
Filename: filepath.Join(dir, "test-353354494b53534a5538435652584d594a5759394d43484f38514b34594b4b524b34515339593249344e4238474a5436414b.txt"),
Filename: filepath.Join(dir, "test-32354837325a545944534a4537.txt"),
},
}, ds)
mgr.ClearCache()
@ -32,7 +32,7 @@ func Test(t *testing.T) {
ds := mgr.FileDataSource(tmpFilePath)
assert.Equal(t, &envoy_config_core_v3.DataSource{
Specifier: &envoy_config_core_v3.DataSource_Filename{
Filename: filepath.Join(dir, "test-34514f59593332445a5649504230484142544c515057383944383730554833564d32574836354654585954304e424f464336.txt"),
Filename: filepath.Join(dir, "test-474136555958463735414951.txt"),
},
}, ds)
@ -41,7 +41,7 @@ func Test(t *testing.T) {
ds = mgr.FileDataSource(tmpFilePath)
assert.Equal(t, &envoy_config_core_v3.DataSource{
Specifier: &envoy_config_core_v3.DataSource_Filename{
Filename: filepath.Join(dir, "test-32564e4457304430393559364b5747373138584f484f5a51334d365758584b47364b555a4c444849513241513457323259.txt"),
Filename: filepath.Join(dir, "test-3331324c4a35574d5439444d4c.txt"),
},
}, ds)

18
config/envoyconfig/filemgr/name.go Normal file
View file

@ -0,0 +1,18 @@
package filemgr
import (
"fmt"
"path/filepath"
"github.com/cespare/xxhash/v2"
"github.com/martinlindhe/base36"
)
// GetFileNameWithBytesHash constructs a filename using a base filename and a hash of
// the data. For example: GetFileNameWithBytesHash("example.txt", []byte{...}) ==> "example-abcd1234.txt"
func GetFileNameWithBytesHash(base string, data []byte) string {
h := xxhash.Sum64(data)
he := base36.Encode(h)
ext := filepath.Ext(base)
return fmt.Sprintf("%s-%x%s", base[:len(base)-len(ext)], he, ext)
}

19
config/envoyconfig/filemgr/name_test.go Normal file
View file

@ -0,0 +1,19 @@
package filemgr
import (
"crypto/rand"
"testing"
"github.com/stretchr/testify/require"
)
func BenchmarkGetFileNameWithBytesHash(b *testing.B) {
bs := make([]byte, 1024*128)
_, err := rand.Read(bs)
require.NoError(b, err)
b.ResetTimer()
for i := 0; i < b.N; i++ {
GetFileNameWithBytesHash("example.crt", bs)
}
}

15
config/envoyconfig/listeners_test.go
View file

@ -41,8 +41,8 @@ func testData(t *testing.T, name string, data interface{}) string {
func Test_buildMetricsHTTPConnectionManagerFilter(t *testing.T) {
cacheDir, _ := os.UserCacheDir()
certFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-crt-354e49305a5a39414a545530374e58454e48334148524c4e324258463837364355564c4e4532464b54355139495547514a38.pem")
keyFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-key-3350415a38414e4e4a4655424e55393430474147324651433949384e485341334b5157364f424b4c5856365a545937383735.pem")
certFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-crt-32375a484d4f49594c4d374830.pem")
keyFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-key-33393156483053584631414836.pem")
b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
li, err := b.buildMetricsListener(&config.Config{
@ -74,7 +74,7 @@ func Test_buildDownstreamTLSContext(t *testing.T) {
b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
cacheDir, _ := os.UserCacheDir()
clientCAFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "client-ca-3533485838304b593757424e3354425157494c4747433534384f474f3631364d5332554c3332485a483834334d50454c344a.pem")
clientCAFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "client-ca-313754424855313435355a5348.pem")
t.Run("no-validation", func(t *testing.T) {
downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), &config.Config{Options: &config.Options{}}, nil)
@ -207,8 +207,7 @@ func Test_buildDownstreamTLSContext(t *testing.T) {
}}
maxVerifyDepth = 10
downstreamTLSContext, err :=
b.buildDownstreamTLSContextMulti(context.Background(), config, nil)
downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), config, nil)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `{
"maxVerifyDepth": 10,
@ -220,8 +219,7 @@ func Test_buildDownstreamTLSContext(t *testing.T) {
}`, downstreamTLSContext.GetCommonTlsContext().GetValidationContext())
maxVerifyDepth = 0
downstreamTLSContext, err =
b.buildDownstreamTLSContextMulti(context.Background(), config, nil)
downstreamTLSContext, err = b.buildDownstreamTLSContextMulti(context.Background(), config, nil)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `{
"onlyVerifyLeafCertCrl": true,
@ -243,8 +241,7 @@ func Test_buildDownstreamTLSContext(t *testing.T) {
},
},
}}
downstreamTLSContext, err :=
b.buildDownstreamTLSContextMulti(context.Background(), config, nil)
downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), config, nil)
require.NoError(t, err)
testutil.AssertProtoJSONEqual(t, `{
"maxVerifyDepth": 1,