3pp-mirror/pomerium
1
0
Fork 0
mirror of https://github.com/pomerium/pomerium.git synced 2025-05-10 07:37:33 +02:00
Code Issues Projects Releases Packages Wiki Activity

databroker: rename cache service (#1790)

Browse source 
* rename cache folder

* rename cache service everywhere

* skip yaml in examples

* Update docs/docs/topics/data-storage.md

Co-authored-by: Travis Groth <travisgroth@users.noreply.github.com>

Co-authored-by: Travis Groth <travisgroth@users.noreply.github.com>
This commit is contained in:
Caleb Doxsey 2021-01-21 08:41:22 -07:00 • committed by GitHub
parent 0adb9e5dde
commit 70b4497595
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
27 changed files with 115 additions and 108 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.pre-commit-config.yaml
View file

@ -4,7 +4,9 @@ repos:
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
exclude: "docs/.*"
- id: check-yaml
exclude: "examples/.*"
- id: check-added-large-files
- repo: https://github.com/syntaqx/git-hooks
rev: v0.0.16

2
authenticate/authenticate_test.go
View file

@ -115,7 +115,7 @@ func TestNew(t *testing.T) {
{"empty opts", &config.Options{}, true},
{"fails to validate", badRedirectURL, true},
{"bad provider", badProvider, true},
{"bad cache url", badGRPCConn, true},
{"bad databroker url", badGRPCConn, true},
{"empty provider url", emptyProviderURL, true},
{"good signing key", goodSigningKey, false},
{"bad signing key", badSigningKey, true},

12
authorize/authorize_test.go
View file

@ -22,7 +22,7 @@ func TestNew(t *testing.T) {
"good",
config.Options{
AuthenticateURL: mustParseURL("https://authN.example.com"),
DataBrokerURL: mustParseURL("https://cache.example.com"),
DataBrokerURL: mustParseURL("https://databroker.example.com"),
SharedKey: "2p/Wi2Q6bYDfzmoSEbKqYKtg+DUoLWTEHHs7vOhvL7w=",
Policies: policies,
},
@ -32,7 +32,7 @@ func TestNew(t *testing.T) {
"bad shared secret",
config.Options{
AuthenticateURL: mustParseURL("https://authN.example.com"),
DataBrokerURL: mustParseURL("https://cache.example.com"),
DataBrokerURL: mustParseURL("https://databroker.example.com"),
SharedKey: "AZA85podM73CjLCjViDNz1EUvvejKpWp7Hysr0knXA==",
Policies: policies,
},
@ -42,7 +42,7 @@ func TestNew(t *testing.T) {
"really bad shared secret",
config.Options{
AuthenticateURL: mustParseURL("https://authN.example.com"),
DataBrokerURL: mustParseURL("https://cache.example.com"),
DataBrokerURL: mustParseURL("https://databroker.example.com"),
SharedKey: "sup",
Policies: policies,
},
@ -52,7 +52,7 @@ func TestNew(t *testing.T) {
"validation error, short secret",
config.Options{
AuthenticateURL: mustParseURL("https://authN.example.com"),
DataBrokerURL: mustParseURL("https://cache.example.com"),
DataBrokerURL: mustParseURL("https://databroker.example.com"),
SharedKey: "AZA85podM73CjLCjViDNz1EUvvejKpWp7Hysr0knXA==",
Policies: policies,
},
@ -60,7 +60,7 @@ func TestNew(t *testing.T) {
},
{"empty options", config.Options{}, true},
{
"bad cache url",
"bad databroker url",
config.Options{
AuthenticateURL: mustParseURL("https://authN.example.com"),
DataBrokerURL: &url.URL{},
@ -101,7 +101,7 @@ func TestAuthorize_OnConfigChange(t *testing.T) {
t.Parallel()
o := &config.Options{
AuthenticateURL: mustParseURL("https://authN.example.com"),
DataBrokerURL: mustParseURL("https://cache.example.com"),
DataBrokerURL: mustParseURL("https://databroker.example.com"),
SharedKey: tc.SharedKey,
Policies: tc.Policies,
}

2
authorize/grpc_test.go
View file

@ -360,7 +360,7 @@ func TestSync(t *testing.T) {
}
o := &config.Options{
AuthenticateURL: mustParseURL("https://authN.example.com"),
DataBrokerURL: mustParseURL("https://cache.example.com"),
DataBrokerURL: mustParseURL("https://databroker.example.com"),
SharedKey: "gXK6ggrlIW2HyKyUF9rUO4azrDgxhDPWqw9y+lJU7B8=",
Policies: testPolicies(t),
}

10
config/helpers.go
View file

@ -11,6 +11,8 @@ const (
ServiceAuthenticate = "authenticate"
// ServiceCache represents running the cache service component
ServiceCache = "cache"
// ServiceDataBroker represents running the databroker service component
ServiceDataBroker = "databroker"
// StorageRedisName is the name of the redis storage backend
StorageRedisName = "redis"
// StorageInMemoryName is the name of the in-memory storage backend
@ -25,6 +27,7 @@ func IsValidService(s string) bool {
ServiceAuthenticate,
ServiceAuthorize,
ServiceCache,
ServiceDataBroker,
ServiceProxy:
return true
}
@ -64,12 +67,13 @@ func IsProxy(s string) bool {
return false
}
// IsCache checks to see if we should be running the proxy service
func IsCache(s string) bool {
// IsDataBroker checks to see if we should be running the databroker service
func IsDataBroker(s string) bool {
switch s {
case
ServiceAll,
ServiceCache:
ServiceCache,
ServiceDataBroker:
return true
}
return false

9
config/helpers_test.go
View file

@ -16,7 +16,7 @@ func Test_isValidService(t *testing.T) {
{"authenticate bad case", "AuThenticate", false},
{"authorize implemented", "authorize", true},
{"jiberish", "xd23", false},
{"cache", "cache", true},
{"databroker", "databroker", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@ -94,7 +94,7 @@ func Test_IsProxy(t *testing.T) {
}
}
func Test_IsCache(t *testing.T) {
func Test_IsDataBroker(t *testing.T) {
t.Parallel()
tests := []struct {
name string
@ -107,11 +107,12 @@ func Test_IsCache(t *testing.T) {
{"proxy bad case", "PrOxY", false},
{"jiberish", "xd23", false},
{"cache", "cache", true},
{"databroker", "databroker", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := IsCache(tt.service); got != tt.want {
t.Errorf("IsCache() = %v, want %v", got, tt.want)
if got := IsDataBroker(tt.service); got != tt.want {
t.Errorf("IsDataBroker() = %v, want %v", got, tt.want)
}
})
}

4
config/options.go
View file

@ -523,7 +523,7 @@ func (o *Options) Validate() error {
return errors.New("config: unknown databroker storage backend type")
}
if IsAuthorize(o.Services) || IsCache(o.Services) {
if IsAuthorize(o.Services) || IsDataBroker(o.Services) {
// if authorize is set, we don't really need a http server
// but we'll still set one up incase the user wants to use
// the HTTP health check api
@ -568,7 +568,7 @@ func (o *Options) Validate() error {
if o.DataBrokerURLString != "" {
u, err := urlutil.ParseAndValidateURL(o.DataBrokerURLString)
if err != nil {
return fmt.Errorf("config: bad cache service url %s : %w", o.DataBrokerURLString, err)
return fmt.Errorf("config: bad databroker service url %s : %w", o.DataBrokerURLString, err)
}
o.DataBrokerURL = u
}

6
config/options_test.go
View file

@ -318,9 +318,9 @@ func Test_NewOptionsFromConfigEnvVar(t *testing.T) {
{"bad no certs no insecure mode set", map[string]string{"SHARED_SECRET": "YixWi1MYh77NMECGGIJQevoonYtVF+ZPRkQZrrmeRqM="}, true},
{"good disable headers ", map[string]string{"HEADERS": "disable:true", "INSECURE_SERVER": "true", "SHARED_SECRET": "YixWi1MYh77NMECGGIJQevoonYtVF+ZPRkQZrrmeRqM="}, false},
{"bad whitespace in secret", map[string]string{"INSECURE_SERVER": "true", "SERVICES": "authenticate", "SHARED_SECRET": "YixWi1MYh77NMECGGIJQevoonYtVF+ZPRkQZrrmeRqM=\n"}, true},
{"good forward auth url", map[string]string{"FORWARD_AUTH_URL": "https://cache.example", "INSECURE_SERVER": "true", "SHARED_SECRET": "YixWi1MYh77NMECGGIJQevoonYtVF+ZPRkQZrrmeRqM="}, false},
{"bad forward auth url", map[string]string{"FORWARD_AUTH_URL": "cache.example", "INSECURE_SERVER": "true", "SHARED_SECRET": "YixWi1MYh77NMECGGIJQevoonYtVF+ZPRkQZrrmeRqM="}, true},
{"same addr and grpc addr", map[string]string{"SERVICES": "cache", "ADDRESS": "0", "GRPC_ADDRESS": "0", "INSECURE_SERVER": "true", "SHARED_SECRET": "YixWi1MYh77NMECGGIJQevoonYtVF+ZPRkQZrrmeRqM="}, false},
{"good forward auth url", map[string]string{"FORWARD_AUTH_URL": "https://databroker.example", "INSECURE_SERVER": "true", "SHARED_SECRET": "YixWi1MYh77NMECGGIJQevoonYtVF+ZPRkQZrrmeRqM="}, false},
{"bad forward auth url", map[string]string{"FORWARD_AUTH_URL": "databroker.example", "INSECURE_SERVER": "true", "SHARED_SECRET": "YixWi1MYh77NMECGGIJQevoonYtVF+ZPRkQZrrmeRqM="}, true},
{"same addr and grpc addr", map[string]string{"SERVICES": "databroker", "ADDRESS": "0", "GRPC_ADDRESS": "0", "INSECURE_SERVER": "true", "SHARED_SECRET": "YixWi1MYh77NMECGGIJQevoonYtVF+ZPRkQZrrmeRqM="}, false},
{"bad cert files", map[string]string{"INSECURE_SERVER": "true", "SHARED_SECRET": "YixWi1MYh77NMECGGIJQevoonYtVF+ZPRkQZrrmeRqM=", "CERTIFICATES": "./test-data/example-cert.pem"}, true},
{"good cert file", map[string]string{"CERTIFICATE_FILE": "./testdata/example-cert.pem", "CERTIFICATE_KEY_FILE": "./testdata/example-key.pem", "INSECURE_SERVER": "true", "SHARED_SECRET": "YixWi1MYh77NMECGGIJQevoonYtVF+ZPRkQZrrmeRqM="}, false},
{"bad cert file", map[string]string{"CERTIFICATE_FILE": "./testdata/example-cert-bad.pem", "CERTIFICATE_KEY_FILE": "./testdata/example-key-bad.pem", "INSECURE_SERVER": "true", "SHARED_SECRET": "YixWi1MYh77NMECGGIJQevoonYtVF+ZPRkQZrrmeRqM="}, true},

34
cache/cache.go → databroker/cache.go
View file

@ -1,7 +1,7 @@
// Package cache is a pomerium service that handles the storage of user
// Package databroker is a pomerium service that handles the storage of user
// session state. It communicates over RPC with other pomerium services,
// and can be configured to use a number of different backend cache stores.
package cache
// and can be configured to use a number of different backend databroker stores.
package databroker
import (
"context"
@ -27,9 +27,9 @@ import (
"github.com/pomerium/pomerium/pkg/grpcutil"
)
// Cache represents the cache service. The cache service is a simple interface
// DataBroker represents the databroker service. The databroker service is a simple interface
// for storing keyed blobs (bytes) of unstructured data.
type Cache struct {
type DataBroker struct {
dataBrokerServer *dataBrokerServer
manager *manager.Manager
@ -43,8 +43,8 @@ type Cache struct {
directoryProvider directory.Provider
}
// New creates a new cache service.
func New(cfg *config.Config) (*Cache, error) {
// New creates a new databroker service.
func New(cfg *config.Config) (*DataBroker, error) {
localListener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return nil, err
@ -82,7 +82,7 @@ func New(cfg *config.Config) (*Cache, error) {
dataBrokerServer := newDataBrokerServer(cfg)
c := &Cache{
c := &DataBroker{
dataBrokerServer: dataBrokerServer,
localListener: localListener,
localGRPCServer: localGRPCServer,
@ -101,23 +101,23 @@ func New(cfg *config.Config) (*Cache, error) {
}
// OnConfigChange is called whenever configuration is changed.
func (c *Cache) OnConfigChange(cfg *config.Config) {
func (c *DataBroker) OnConfigChange(cfg *config.Config) {
err := c.update(cfg)
if err != nil {
log.Error().Err(err).Msg("cache: error updating configuration")
log.Error().Err(err).Msg("databroker: error updating configuration")
}
c.dataBrokerServer.OnConfigChange(cfg)
}
// Register registers all the gRPC services with the given server.
func (c *Cache) Register(grpcServer *grpc.Server) {
func (c *DataBroker) Register(grpcServer *grpc.Server) {
databroker.RegisterDataBrokerServiceServer(grpcServer, c.dataBrokerServer)
directory.RegisterDirectoryServiceServer(grpcServer, c)
}
// Run runs the cache components.
func (c *Cache) Run(ctx context.Context) error {
// Run runs the databroker components.
func (c *DataBroker) Run(ctx context.Context) error {
eg, ctx := errgroup.WithContext(ctx)
eg.Go(func() error {
return c.localGRPCServer.Serve(c.localListener)
@ -133,14 +133,14 @@ func (c *Cache) Run(ctx context.Context) error {
return eg.Wait()
}
func (c *Cache) update(cfg *config.Config) error {
func (c *DataBroker) update(cfg *config.Config) error {
if err := validate(cfg.Options); err != nil {
return fmt.Errorf("cache: bad option: %w", err)
return fmt.Errorf("databroker: bad option: %w", err)
}
authenticator, err := identity.NewAuthenticator(cfg.Options.GetOauthOptions())
if err != nil {
return fmt.Errorf("cache: failed to create authenticator: %w", err)
return fmt.Errorf("databroker: failed to create authenticator: %w", err)
}
directoryProvider := directory.GetProvider(directory.Options{
@ -175,7 +175,7 @@ func (c *Cache) update(cfg *config.Config) error {
}
// validate checks that proper configuration settings are set to create
// a cache instance
// a databroker instance
func validate(o *config.Options) error {
if _, err := cryptutil.NewAEADCipherFromBase64(o.SharedKey); err != nil {
return fmt.Errorf("invalid 'SHARED_SECRET': %w", err)

4
cache/cache_test.go → databroker/cache_test.go
View file

@ -1,4 +1,4 @@
package cache
package databroker
import (
"io/ioutil"
@ -25,7 +25,7 @@ func TestNew(t *testing.T) {
}{
{"good", config.Options{SharedKey: cryptutil.NewBase64Key(), DataBrokerURL: &url.URL{Scheme: "http", Host: "example"}}, false},
{"bad shared secret", config.Options{SharedKey: string([]byte(cryptutil.NewBase64Key())[:31]), DataBrokerURL: &url.URL{Scheme: "http", Host: "example"}}, true},
{"bad cache url", config.Options{SharedKey: cryptutil.NewBase64Key(), DataBrokerURL: &url.URL{}}, true},
{"bad databroker url", config.Options{SharedKey: cryptutil.NewBase64Key(), DataBrokerURL: &url.URL{}}, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {

2
cache/databroker.go → databroker/databroker.go
View file

@ -1,4 +1,4 @@
package cache
package databroker
import (
"context"

2
cache/databroker_test.go → databroker/databroker_test.go
View file

@ -1,4 +1,4 @@
package cache
package databroker
import (
"context"

4
cache/directory.go → databroker/directory.go
View file

@ -1,4 +1,4 @@
package cache
package databroker
import (
"context"
@ -12,7 +12,7 @@ import (
)
// RefreshUser refreshes a user's directory information.
func (c *Cache) RefreshUser(ctx context.Context, req *directory.RefreshUserRequest) (*emptypb.Empty, error) {
func (c *DataBroker) RefreshUser(ctx context.Context, req *directory.RefreshUserRequest) (*emptypb.Empty, error) {
c.mu.Lock()
dp := c.directoryProvider
c.mu.Unlock()

4
docs/docs/architecture.md
View file

@ -29,13 +29,13 @@ Pomerium is composed of 4 logical components:
- Handles authentication flow to your IdP as needed
- Handles identity verification after initial Authentication
- Establishes user session cookie
- Stores user OIDC tokens in cache service
- Stores user OIDC tokens in databroker service
- Authorization Service
- Processes policy to determine permissions for each service
- Handles authorization check for all user sessions
- Directs Proxy service to initiate Authentication flow as required
- Provides additional security releated headers for upstream services to consume
- Cache Service
- Data Broker Service
- Retrieves identity provider related data such as group membership
- Stores and refreshes identity provider access and refresh tokens
- Provides streaming authoritative session and identity data to Authorize service

14
docs/docs/topics/data-storage.md
View file

@ -19,11 +19,11 @@ Pomerium keeps persistent state out of most components, but an identity-aware ac
- Group membership was fixed from session creation
- Slow initial authentication flow to fetch user data
To address these limitations, the Pomerium `cache` service runs a number of internal services responsible for maintaining data and state.
To address these limitations, the Pomerium `databroker` service runs a number of internal services responsible for maintaining data and state.
#### Design
Internal to the `cache` service, the `databroker` is responsible for providing a stateful storage layer. Services which require high performance maintain a streaming local cache of the contents of the `databroker`, while others may call `databroker` in real time. Only the `databroker` is expected to maintain authoritative state.
The `databroker` is responsible for providing a stateful storage layer. Services which require high performance maintain a streaming local cache of the contents of the `databroker`, while others may call `databroker` in real time. Only the `databroker` is expected to maintain authoritative state.
## Persistence
@ -33,27 +33,27 @@ To prevent early session loss in production deployments, persistent storage back
## Backends
Configuration options for each backend are detailed in [cache configuration reference](/reference/#cache-service).
Configuration options for each backend are detailed in [databroker configuration reference](/reference/#databroker-service).
In all backends, Pomerium encrypts record values. This ensures security of all records at rest, regardless of data store capabilities. While this prevents many classes of attack vector, additional security measures should always be taken to secure data in transit and minimize access to the backends themselves.
Please see Pomerium backend and upstream storage system documentation for best practices.
### In-Memory
- Cache Service HA: `no`
- Data Broker Service HA: `no`
- Data Store HA: `no`
- Data Persistence: `no`
The default storage backend for `databroker` is memory based. This backend provides
easy deployment semantics but is not persistent or highly available. Running more than one `cache` instance configured for memory backed storage is not supported and will lead to non-deterministic behavior.
easy deployment semantics but is not persistent or highly available. Running more than one `databroker` instance configured for memory backed storage is not supported and will lead to non-deterministic behavior.
### Redis
- Cache Service HA: `yes`
- Data Broker Service HA: `yes`
- Data Store HA: `yes`
- Data Persistence: `yes`
The Redis based backend supports multiple `cache` instances and persistence across restarts. We recommend a dedicated redis instance for Pomerium to provide the strongest security and performance guarantees.
The Redis based backend supports multiple `databroker` instances and persistence across restarts. We recommend a dedicated redis instance for Pomerium to provide the strongest security and performance guarantees.
#### High Availability
Redis should be configured to provide high availability via [replication](https://redis.io/topics/replication) and failover. Sentinal and cluster are not supported at this time.

18
docs/docs/topics/production-deployment.md
View file

@ -28,7 +28,7 @@ In dedicated service mode, you have the opportunity to scale the components of P
All of Pomerium's components are designed to be stateless, and may all be scaled horizontally or vertically. In general, horizontal scaling is recommended. Vertical scaling will lead to diminished returns after ~8 vCPUs.
The Cache service, which is responsible for session and identity related data, must be [configured for external persistence](/docs/topics/data-storage.md) to be fully stateless.
The Data Broker service, which is responsible for session and identity related data, must be [configured for external persistence](/docs/topics/data-storage.md) to be fully stateless.
### Proxy
@ -48,14 +48,14 @@ The Authenticate service handles session cookie setup, session storage, and auth
Authenticate requires significantly fewer resources than other components due to the only-occasional requirement to establish new sessions. Add resources to the Authenticate service if you have a high session/user churn rate. The requests should be constant time and complexity, but may vary by Identity Provider implementation.
### Cache
### Data Broker
The Cache service is responsible for background identity data retrieval and storage. It is in the hot path for user authentication. However, it does not directly handle user traffic and is not in-path for authorization decisions.
The Data Broker service is responsible for background identity data retrieval and storage. It is in the hot path for user authentication. However, it does not directly handle user traffic and is not in-path for authorization decisions.
The Cache service does not require significant resources, as it provides streaming updates of state changes to the Authorize service. There will be utilization spikes when Authorize services are restarted and perform an initial synchronization. Add resources if running many Authorize services and performing restarts in large batches. In many deployments, 2 replicas of Cache is enough to provide resilient service.
The Data Broker service does not require significant resources, as it provides streaming updates of state changes to the Authorize service. There will be utilization spikes when Authorize services are restarted and perform an initial synchronization. Add resources if running many Authorize services and performing restarts in large batches. In many deployments, 2 replicas of Data Broker is enough to provide resilient service.
::: warning
In a production configuration, Cache CPU/IO utilization also translates to IO load on the [underlying storage system](/docs/topics/data-storage.md). Ensure it is scaled accordingly!
In a production configuration, Data Broker CPU/IO utilization also translates to IO load on the [underlying storage system](/docs/topics/data-storage.md). Ensure it is scaled accordingly!
:::
## Load Balancing
@ -74,11 +74,11 @@ You should provide a TCP or HTTP(s) load balancer between end users and the Auth
Authenticate is compatible with any L4 or L7/HTTP load balancer. Session stickiness should not be required and it is typical to have Authenticate be a named vhost on the same L7 load balancer as the Proxy service.
### Authorize and Cache
### Authorize and Data Broker
You do **not** need to provide a load balancer in front of Authorize and Cache services. Both utilize GRPC, and thus has special requirements if you should choose to use an external load balancer. GRPC can perform client based load balancing, and in most configurations is the best architecture.
You do **not** need to provide a load balancer in front of Authorize and Data Broker services. Both utilize GRPC, and thus has special requirements if you should choose to use an external load balancer. GRPC can perform client based load balancing, and in most configurations is the best architecture.
By default, Pomerium gRPC clients will automatically connect to all IPs returned by a DNS query for the name of an upstream service. They will then regularly re-query DNS for changes to the Authorize or Cache service cluster. Health checks and failover are automatic.
By default, Pomerium gRPC clients will automatically connect to all IPs returned by a DNS query for the name of an upstream service. They will then regularly re-query DNS for changes to the Authorize or Data Broker service cluster. Health checks and failover are automatic.
**Many load balancers do not support HTTP2 yet. Please verify with your hardware, software or cloud provider**
@ -99,7 +99,7 @@ Regardless of the service mode, it is recommended you run 2+ instances of Pomeri
Ensure that you have enough spare capacity to handle the scope of your failure domains.
::: warning
Multiple replicas of Cache or all-in-one service are only supported with [external storage](/docs/topics/data-storage.md) configured
Multiple replicas of Data Broker or all-in-one service are only supported with [external storage](/docs/topics/data-storage.md) configured
:::
## SSL/TLS Certificates

12
docs/reference/readme.md
View file

@ -511,7 +511,7 @@ Proxy log level sets the logging level for the pomerium proxy service access log
- Config File Key: `services`
- Type: `string`
- Default: `all`
- Options: `all` `authenticate` `authorize` `cache` or `proxy`
- Options: `all` `authenticate` `authorize` `databroker` or `proxy`
Service mode sets which service(s) to run. If testing, you may want to set to `all` and run pomerium in "all-in-one mode." In production, you'll likely want to spin up several instances of each service mode for high availability.
@ -855,26 +855,26 @@ Refresh cooldown is the minimum amount of time between allowed manually refreshe
Do not append proxy IP address to `x-forwarded-for` HTTP header. See [Envoy](https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers.html?highlight=skip_xff_append#x-forwarded-for) docs for more detail.
## Cache Service
The cache service is used for storing user session data.
## Data Broker Service
The databroker service is used for storing user session data.
### Data Broker Service URL
- Environmental Variable: `DATABROKER_SERVICE_URL`
- Config File Key: `databroker_service_url`
- Type: `URL`
- Example: `https://cache.corp.example.com`
- Example: `https://databroker.corp.example.com`
- Default: in all-in-one mode, `http://localhost:5443`
The data broker service URL points to a data broker which is responsible for storing associated authorization context (e.g. sessions, users and user groups).
By default, the `cache` service uses an in-memory databroker.
By default, the `databroker` service uses an in-memory databroker.
To create your own data broker, implement the following gRPC interface:
- [pkg/grpc/databroker/databroker.proto](https://github.com/pomerium/pomerium/blob/master/pkg/grpc/databroker/databroker.proto)
For an example implementation, the in-memory database used by the cache service can be found here:
For an example implementation, the in-memory database used by the databroker service can be found here:
- [pkg/databroker/memory](https://github.com/pomerium/pomerium/tree/master/pkg/databroker/memory)

12
docs/reference/settings.yaml
View file

@ -581,7 +581,7 @@ settings:
- Config File Key: `services`
- Type: `string`
- Default: `all`
- Options: `all` `authenticate` `authorize` `cache` or `proxy`
- Options: `all` `authenticate` `authorize` `databroker` or `proxy`
doc: |
Service mode sets which service(s) to run. If testing, you may want to set to `all` and run pomerium in "all-in-one mode." In production, you'll likely want to spin up several instances of each service mode for high availability.
shortdoc: |
@ -967,9 +967,9 @@ settings:
Do not append proxy IP address to `x-forwarded-for` HTTP header. See [Envoy](https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers.html?highlight=skip_xff_append#x-forwarded-for) docs for more detail.
shortdoc: |
Do not append proxy IP address to [x-forwarded-for](https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers.html?highlight=skip_xff_append#x-forwarded-for).
- name: "Cache Service"
- name: "Data Broker Service"
doc: |
The cache service is used for storing user session data.
The databroker service is used for storing user session data.
settings:
- name: "Data Broker Service URL"
keys: ["databroker_service_url"]
@ -977,18 +977,18 @@ settings:
- Environmental Variable: `DATABROKER_SERVICE_URL`
- Config File Key: `databroker_service_url`
- Type: `URL`
- Example: `https://cache.corp.example.com`
- Example: `https://databroker.corp.example.com`
- Default: in all-in-one mode, `http://localhost:5443`
doc: |
The data broker service URL points to a data broker which is responsible for storing associated authorization context (e.g. sessions, users and user groups).
By default, the `cache` service uses an in-memory databroker.
By default, the `databroker` service uses an in-memory databroker.
To create your own data broker, implement the following gRPC interface:
- [pkg/grpc/databroker/databroker.proto](https://github.com/pomerium/pomerium/blob/master/pkg/grpc/databroker/databroker.proto)
For an example implementation, the in-memory database used by the cache service can be found here:
For an example implementation, the in-memory database used by the databroker service can be found here:
- [pkg/databroker/memory](https://github.com/pomerium/pomerium/tree/master/pkg/databroker/memory)
- name: "Data Broker Storage Type"

2
examples/config/config.example.env
View file

@ -11,7 +11,7 @@ export AUTHENTICATE_SERVICE_URL=https://authenticate.corp.beyondperimeter.com
# AUTHORIZE_SERVICE_URL service url will default to localhost in all-in-one mode,
# otherwise it should be set to a "behind-the-ingress" routable url
# export AUTHORIZE_SERVICE_URL=https://pomerium-authorize-service.default.svc.cluster.local
# export CACHE_SERVICE_URL=https://pomerium-cache-service.default.svc.cluster.local
# export DATABROKER_SERVICE_URL=https://pomerium-databroker-service.default.svc.cluster.local
# Certificates can be loaded as files or base64 encoded bytes.
# See : https://www.pomerium.io/docs/reference/certificates

2
examples/config/config.example.yaml
View file

@ -9,7 +9,7 @@ authenticate_service_url: https://authenticate.localhost.pomerium.io
# authorize service url will default to localhost in all-in-one mode, otherwise
# it should be set to a "behind-the-ingress" routable url
# authorize_service_url: https://pomerium-authorize-service.default.svc.cluster.local
# databroker_service_url: https://pomerium-cache-service.default.svc.cluster.local
# databroker_service_url: https://pomerium-databroker-service.default.svc.cluster.local
# Certificates can be loaded as files or base64 encoded bytes.
# certificate_file: "./cert.pem" # optional, defaults to `./cert.pem`

6
examples/docker/nginx.docker-compose.yml
View file

@ -31,7 +31,7 @@ services:
- VIRTUAL_PROTO=http
- VIRTUAL_HOST=authenticate.corp.beyondperimeter.com
- VIRTUAL_PORT=443
- DATABROKER_SERVICE_URL=http://pomerium-cache:443
- DATABROKER_SERVICE_URL=http://pomerium-databroker:443
volumes:
- ../config/config.example.yaml:/pomerium/config.yaml:ro
@ -76,11 +76,11 @@ services:
expose:
- 443
pomerium-cache:
pomerium-databroker:
image: pomerium/pomerium:latest # or `build: .` to build from source
restart: always
environment:
- SERVICES=cache
- SERVICES=databroker
- SHARED_SECRET=aDducXQzK2tPY3R4TmdqTGhaYS80eGYxcTUvWWJDb2M=
- GRPC_INSECURE=TRUE
- GRPC_ADDRESS=:443

2
examples/kubernetes/kubernetes-config.yaml
View file

@ -6,7 +6,7 @@ grpc_address: ":80"
authenticate_service_url: https://authenticate.localhost.pomerium.io
authorize_service_url: http://pomerium-authorize-service.default.svc.cluster.local
databroker_service_url: http://pomerium-cache-service.default.svc.cluster.local
databroker_service_url: http://pomerium-databroker-service.default.svc.cluster.local
override_certificate_name: "*.localhost.pomerium.io"

2
examples/kubernetes/kubernetes_gke.sh
View file

@ -32,7 +32,7 @@ echo "=> deploy pomerium proxy, authorize, and authenticate"
kubectl apply -f pomerium-proxy.yml
kubectl apply -f pomerium-authenticate.yml
kubectl apply -f pomerium-authorize.yml
kubectl apply -f pomerium-cache.yml
kubectl apply -f pomerium-databroker.yml
echo "=> deploy our test app, httpbin"
kubectl apply -f httpbin.yml

18
examples/kubernetes/pomerium-cache.yml → examples/kubernetes/pomerium-databroker.yml
View file

@ -1,35 +1,35 @@
apiVersion: v1
kind: Service
metadata:
name: pomerium-cache-service
name: pomerium-databroker-service
spec:
clusterIP: None # cache is a headless service!
clusterIP: None # databroker is a headless service!
ports:
- port: 80
name: grpc
selector:
app: pomerium-cache
app: pomerium-databroker
type: ClusterIP
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: pomerium-cache
name: pomerium-databroker
labels:
app: pomerium-cache
app: pomerium-databroker
spec:
replicas: 1
selector:
matchLabels:
app: pomerium-cache
app: pomerium-databroker
template:
metadata:
labels:
app: pomerium-cache
app: pomerium-databroker
spec:
containers:
- image: pomerium/pomerium:master
name: pomerium-cache
name: pomerium-databroker
args:
- --config=/etc/pomerium/config.yaml
ports:
@ -38,7 +38,7 @@ spec:
protocol: TCP
env:
- name: SERVICES
value: cache
value: databroker
- name: SHARED_SECRET
valueFrom:
secretKeyRef:

6
integration/manifests/lib/pomerium.libsonnet
View file

@ -184,7 +184,7 @@ local PomeriumConfigMap = function() {
AUTHENTICATE_SERVICE_URL: 'https://authenticate.localhost.pomerium.io',
AUTHENTICATE_CALLBACK_PATH: '/oauth2/callback',
AUTHORIZE_SERVICE_URL: 'https://authorize.default.svc.cluster.local:5443',
DATABROKER_SERVICE_URL: 'https://cache.default.svc.cluster.local:5443',
DATABROKER_SERVICE_URL: 'https://databroker.default.svc.cluster.local:5443',
FORWARD_AUTH_URL: 'https://forward-authenticate.localhost.pomerium.io',
HEADERS: 'X-Frame-Options:SAMEORIGIN',
JWT_CLAIMS_HEADERS: 'email',
@ -497,8 +497,8 @@ local PomeriumForwardAuthIngress = function() {
PomeriumDeployment('authenticate'),
PomeriumService('authorize'),
PomeriumDeployment('authorize'),
PomeriumService('cache'),
PomeriumDeployment('cache'),
PomeriumService('databroker'),
PomeriumDeployment('databroker'),
PomeriumService('proxy'),
PomeriumDeployment('proxy'),
PomeriumNodePortServce(),

22
internal/cmd/pomerium/pomerium.go
View file

@ -16,8 +16,8 @@ import (
"github.com/pomerium/pomerium/authenticate"
"github.com/pomerium/pomerium/authorize"
"github.com/pomerium/pomerium/cache"
"github.com/pomerium/pomerium/config"
databroker_service "github.com/pomerium/pomerium/databroker"
"github.com/pomerium/pomerium/internal/autocert"
"github.com/pomerium/pomerium/internal/controlplane"
"github.com/pomerium/pomerium/internal/databroker"
@ -91,9 +91,9 @@ func Run(ctx context.Context, configFile string) error {
return err
}
}
var cacheServer *cache.Cache
if config.IsCache(src.GetConfig().Options.Services) {
cacheServer, err = setupCache(src, controlPlane)
var dataBrokerServer *databroker_service.DataBroker
if config.IsDataBroker(src.GetConfig().Options.Services) {
dataBrokerServer, err = setupDataBroker(src, controlPlane)
if err != nil {
return err
}
@ -125,7 +125,7 @@ func Run(ctx context.Context, configFile string) error {
})
// in non-all-in-one mode we will wait for the initial sync to complete before starting
// the control plane
if cacheServer == nil {
if dataBrokerServer == nil {
if err := authorizeServer.WaitForInitialSync(ctx); err != nil {
return err
}
@ -134,9 +134,9 @@ func Run(ctx context.Context, configFile string) error {
eg.Go(func() error {
return controlPlane.Run(ctx)
})
if cacheServer != nil {
if dataBrokerServer != nil {
eg.Go(func() error {
return cacheServer.Run(ctx)
return dataBrokerServer.Run(ctx)
})
}
return eg.Wait()
@ -174,13 +174,13 @@ func setupAuthorize(src config.Source, controlPlane *controlplane.Server) (*auth
return svc, nil
}
func setupCache(src config.Source, controlPlane *controlplane.Server) (*cache.Cache, error) {
svc, err := cache.New(src.GetConfig())
func setupDataBroker(src config.Source, controlPlane *controlplane.Server) (*databroker_service.DataBroker, error) {
svc, err := databroker_service.New(src.GetConfig())
if err != nil {
return nil, fmt.Errorf("error creating config service: %w", err)
return nil, fmt.Errorf("error creating databroker service: %w", err)
}
svc.Register(controlPlane.GRPCServer)
log.Info().Msg("enabled cache service")
log.Info().Msg("enabled databroker service")
src.OnConfigChange(svc.OnConfigChange)
svc.OnConfigChange(src.GetConfig())
return svc, nil

6
internal/controlplane/xds_listeners.go
View file

@ -45,7 +45,7 @@ func (srv *Server) buildListeners(options *config.Options) []*envoy_config_liste
listeners = append(listeners, srv.buildMainListener(options))
}
if config.IsAuthorize(options.Services) || config.IsCache(options.Services) {
if config.IsAuthorize(options.Services) || config.IsDataBroker(options.Services) {
listeners = append(listeners, srv.buildGRPCListener(options))
}
@ -146,7 +146,7 @@ func buildMainHTTPConnectionManagerFilter(options *config.Options, domains []str
if options.Addr == options.GRPCAddr {
// if this is a gRPC service domain and we're supposed to handle that, add those routes
if (config.IsAuthorize(options.Services) && hostMatchesDomain(options.GetAuthorizeURL(), domain)) ||
(config.IsCache(options.Services) && hostMatchesDomain(options.GetDataBrokerURL(), domain)) {
(config.IsDataBroker(options.Services) && hostMatchesDomain(options.GetDataBrokerURL(), domain)) {
vh.Routes = append(vh.Routes, buildGRPCRoutes()...)
}
}
@ -433,7 +433,7 @@ func getAllRouteableDomains(options *config.Options, addr string) []string {
lookup[h] = struct{}{}
}
}
if config.IsCache(options.Services) && addr == options.GRPCAddr {
if config.IsDataBroker(options.Services) && addr == options.GRPCAddr {
for _, h := range urlutil.GetDomainsForURL(options.GetDataBrokerURL()) {
lookup[h] = struct{}{}
}