config: allow setting directory sync interval and timeout (#1098)

Updates #567

cache/cache.go

@@ -78,7 +78,15 @@ func New(opts config.Options) (*Cache, error) {
 	userServer := NewUserServer(localGRPCServer, dataBrokerClient)
 	userClient := user.NewUserServiceClient(localGRPCConnection)
 
-	manager := manager.New(authenticator, directoryProvider, sessionClient, userClient, dataBrokerClient)
+	manager := manager.New(
+		authenticator,
+		directoryProvider,
+		sessionClient,
+		userClient,
+		dataBrokerClient,
+		manager.WithGroupRefreshInterval(opts.RefreshDirectoryInterval),
+		manager.WithGroupRefreshTimeout(opts.RefreshDirectoryTimeout),
+	)
 
 	return &Cache{
 		dataBrokerServer: dataBrokerServer,

config/options.go

@@ -123,6 +123,9 @@ type Options struct {
 	ProviderURL    string   `mapstructure:"idp_provider_url" yaml:"idp_provider_url,omitempty"`
 	Scopes         []string `mapstructure:"idp_scopes" yaml:"idp_scopes,omitempty"`
 	ServiceAccount string   `mapstructure:"idp_service_account" yaml:"idp_service_account,omitempty"`
+	// Identity provider refresh directory interval/timeout settings.
+	RefreshDirectoryTimeout  time.Duration `mapstructure:"idp_refresh_directory_timeout" yaml:"idp_refresh_directory_timeout,omitempty"`
+	RefreshDirectoryInterval time.Duration `mapstructure:"idp_refresh_directory_interval" yaml:"idp_refresh_directory_interval,omitempty"`
 
 	// RequestParams are custom request params added to the signin request as
 	// part of an Oauth2 code flow.
@@ -270,6 +273,8 @@ var defaultOptions = Options{
 	GRPCServerMaxConnectionAgeGrace: 5 * time.Minute,
 	AuthenticateCallbackPath:        "/oauth2/callback",
 	TracingSampleRate:               0.0001,
+	RefreshDirectoryInterval:        10 * time.Minute,
+	RefreshDirectoryTimeout:         1 * time.Minute,
 
 	AutocertOptions: AutocertOptions{
 		Folder: dataDir(),

config/options_test.go

@@ -233,7 +233,10 @@ func TestOptionsFromViper(t *testing.T) {
 					"Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload",
 					"X-Frame-Options":           "SAMEORIGIN",
 					"X-XSS-Protection":          "1; mode=block",
-				}},
+				},
+				RefreshDirectoryTimeout:  1 * time.Minute,
+				RefreshDirectoryInterval: 10 * time.Minute,
+			},
 			false},
 		{"good disable header",
 			[]byte(`{"autocert_dir":"","insecure_server":true,"headers": {"disable":"true"},"policy":[{"from": "https://from.example","to":"https://to.example"}]}`),
@@ -246,7 +249,10 @@ func TestOptionsFromViper(t *testing.T) {
 				InsecureServer:                  true,
 				GRPCServerMaxConnectionAge:      5 * time.Minute,
 				GRPCServerMaxConnectionAgeGrace: 5 * time.Minute,
-				Headers:                         map[string]string{}},
+				Headers:                         map[string]string{},
+				RefreshDirectoryTimeout:         1 * time.Minute,
+				RefreshDirectoryInterval:        10 * time.Minute,
+			},
 			false},
 		{"bad url", []byte(`{"policy":[{"from": "https://","to":"https://to.example"}]}`), nil, true},
 		{"bad policy", []byte(`{"policy":[{"allow_public_unauthenticated_access": "dog","to":"https://to.example"}]}`), nil, true},

docs/configuration/readme.md

@@ -668,6 +668,17 @@ For more information see:
 - [Microsoft Azure Request params](https://docs.microsoft.com/en-us/azure/active-directory/develop/v2-oauth2-auth-code-flow#request-an-authorization-code)
 - [Google Authentication URI parameters](https://developers.google.com/identity/protocols/oauth2/openid-connect)
 
+### Identity Provider Refresh Directory Settings
+
+- Environmental Variables: `IDP_REFRESH_DIRECTORY_INTERVAL` `IDP_REFRESH_DIRECTORY_TIMEOUT`
+- Config File Key: `idp_refresh_directory_interval` `idp_refresh_directory_timeout`
+- Type: [Go Duration](https://golang.org/pkg/time/#Duration.String) `string`
+- Example: `IDP_REFRESH_DIRECTORY_INTERVAL=30m`
+- Defaults: `IDP_REFRESH_DIRECTORY_INTERVAL=10m` `IDP_REFRESH_DIRECTORY_TIMEOUT=1m`
+
+Refresh directory interval is the time that pomerium will sync your IDP diretory, while refresh directory timeout is the
+maximum time allow each run. Use it at your ownn risk, if you set a too low value, you may reach IDP API rate limit.
+
 ## Proxy Service
 
 ### Authenticate Service URL

internal/identity/manager/config.go

@@ -4,12 +4,14 @@ import "time"
 
 var (
 	defaultGroupRefreshInterval          = 10 * time.Minute
+	defaultGroupRefreshTimeout           = 1 * time.Minute
 	defaultSessionRefreshGracePeriod     = 1 * time.Minute
 	defaultSessionRefreshCoolOffDuration = 10 * time.Second
 )
 
 type config struct {
 	groupRefreshInterval          time.Duration
+	groupRefreshTimeout           time.Duration
 	sessionRefreshGracePeriod     time.Duration
 	sessionRefreshCoolOffDuration time.Duration
 }
@@ -17,6 +19,7 @@ type config struct {
 func newConfig(options ...Option) *config {
 	cfg := new(config)
 	WithGroupRefreshInterval(defaultGroupRefreshInterval)(cfg)
+	WithGroupRefreshTimeout(defaultGroupRefreshTimeout)(cfg)
 	WithSessionRefreshGracePeriod(defaultSessionRefreshGracePeriod)(cfg)
 	WithSessionRefreshCoolOffDuration(defaultSessionRefreshCoolOffDuration)(cfg)
 	for _, option := range options {
@@ -35,6 +38,13 @@ func WithGroupRefreshInterval(interval time.Duration) Option {
 	}
 }
 
+// WithGroupRefreshTimeout sets the group refresh timeout used by the manager.
+func WithGroupRefreshTimeout(timeout time.Duration) Option {
+	return func(cfg *config) {
+		cfg.groupRefreshTimeout = timeout
+	}
+}
+
 // WithSessionRefreshGracePeriod sets the session refresh grace period used by the manager.
 func WithSessionRefreshGracePeriod(dur time.Duration) Option {
 	return func(cfg *config) {

internal/identity/manager/manager.go

@@ -188,7 +188,7 @@ func (mgr *Manager) refreshLoop(
 func (mgr *Manager) refreshDirectoryUsers(ctx context.Context) {
 	mgr.log.Info().Msg("refreshing directory users")
 
-	ctx, clearTimeout := context.WithTimeout(ctx, time.Minute)
+	ctx, clearTimeout := context.WithTimeout(ctx, mgr.cfg.groupRefreshTimeout)
 	defer clearTimeout()
 
 	directoryUsers, err := mgr.directory.UserGroups(ctx)