pomerium/internal/controlplane/filemgr/filemgr.go
Caleb Doxsey 5d60cff21e
databroker: refactor databroker to sync all changes (#1879)
* refactor backend, implement encrypted store

* refactor in-memory store

* wip

* wip

* wip

* add syncer test

* fix redis expiry

* fix linting issues

* fix test by skipping non-config records

* fix backoff import

* fix init issues

* fix query

* wait for initial sync before starting directory sync

* add type to SyncLatest

* add more log messages, fix deadlock in in-memory store, always return server version from SyncLatest

* update sync types and tests

* add redis tests

* skip macos in github actions

* add comments to proto

* split getBackend into separate methods

* handle errors in initVersion

* return different error for not found vs other errors in get

* use exponential backoff for redis transaction retry

* rename raw to result

* use context instead of close channel

* store type urls as constants in databroker

* use timestampb instead of ptypes

* fix group merging not waiting

* change locked names

* update GetAll to return latest record version

* add method to grpcutil to get the type url for a protobuf type
2021-02-18 15:24:33 -07:00

92 lines
2.6 KiB
Go

// Package filemgr defines a Manager for managing files for the controlplane.
package filemgr
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
"github.com/martinlindhe/base36"
"github.com/pomerium/pomerium/internal/log"
"github.com/pomerium/pomerium/pkg/cryptutil"
)
// A Manager manages files for envoy.
type Manager struct {
cfg *config
}
// NewManager creates a new Manager.
func NewManager(options ...Option) *Manager {
cfg := newConfig(options...)
return &Manager{
cfg: cfg,
}
}
// BytesDataSource returns an envoy config data source based on bytes.
func (mgr *Manager) BytesDataSource(fileName string, data []byte) *envoy_config_core_v3.DataSource {
h := base36.EncodeBytes(cryptutil.Hash("filemgr", data))
ext := filepath.Ext(fileName)
fileName = fmt.Sprintf("%s-%x%s", fileName[:len(fileName)-len(ext)], h, ext)
if err := os.MkdirAll(mgr.cfg.cacheDir, 0o700); err != nil {
log.Error().Err(err).Msg("filemgr: error creating cache directory, falling back to inline bytes")
return inlineBytes(data)
}
filePath := filepath.Join(mgr.cfg.cacheDir, fileName)
if _, err := os.Stat(filePath); os.IsNotExist(err) {
err = ioutil.WriteFile(filePath, data, 0o600)
if err != nil {
log.Error().Err(err).Msg("filemgr: error writing cache file, falling back to inline bytes")
return inlineBytes(data)
}
} else if err != nil {
log.Error().Err(err).Msg("filemgr: error reading cache file, falling back to inline bytes")
return inlineBytes(data)
}
return inlineFilename(filePath)
}
// ClearCache clears the file cache.
func (mgr *Manager) ClearCache() {
err := filepath.Walk(mgr.cfg.cacheDir, func(p string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
return os.Remove(p)
})
if err != nil {
log.Error().Err(err).Msg("failed to clear envoy file cache")
}
}
// FileDataSource returns an envoy config data source based on a file.
func (mgr *Manager) FileDataSource(filePath string) *envoy_config_core_v3.DataSource {
data, err := ioutil.ReadFile(filePath)
if err != nil {
return inlineFilename(filePath)
}
return mgr.BytesDataSource(filepath.Base(filePath), data)
}
func inlineBytes(data []byte) *envoy_config_core_v3.DataSource {
return &envoy_config_core_v3.DataSource{
Specifier: &envoy_config_core_v3.DataSource_InlineBytes{
InlineBytes: data,
},
}
}
func inlineFilename(name string) *envoy_config_core_v3.DataSource {
return &envoy_config_core_v3.DataSource{
Specifier: &envoy_config_core_v3.DataSource_Filename{
Filename: name,
},
}
}