config: Expose and set default GRPC Server Keepalive Parameters (#509)

cmd/pomerium/main.go

@@ -7,10 +7,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/fsnotify/fsnotify"
-	"github.com/gorilla/mux"
-	"google.golang.org/grpc"
-
 	"github.com/pomerium/pomerium/authenticate"
 	"github.com/pomerium/pomerium/authorize"
 	"github.com/pomerium/pomerium/cache"
@@ -27,6 +23,11 @@ import (
 	"github.com/pomerium/pomerium/internal/urlutil"
 	"github.com/pomerium/pomerium/internal/version"
 	"github.com/pomerium/pomerium/proxy"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/gorilla/mux"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/keepalive"
 )
 
 var versionFlag = flag.Bool("version", false, "prints the version")
@@ -147,7 +148,13 @@ func newGRPCServer(opt config.Options, as *authorize.Authorize, cs *cache.Cache,
 
 		}
 	}
-	so := &pgrpc.ServerOptions{Addr: opt.GRPCAddr}
+	so := &pgrpc.ServerOptions{
+		Addr: opt.GRPCAddr,
+		KeepaliveParams: keepalive.ServerParameters{
+			MaxConnectionAge:      opt.GRPCServerMaxConnectionAge,
+			MaxConnectionAgeGrace: opt.GRPCServerMaxConnectionAgeGrace,
+		},
+	}
 	if !opt.GRPCInsecure {
 		so.TLSCertificate = opt.TLSCertificate
 	}

config/options.go

@@ -171,6 +171,11 @@ type Options struct {
 	GRPCClientTimeout       time.Duration `mapstructure:"grpc_client_timeout" yaml:"grpc_client_timeout,omitempty"`
 	GRPCClientDNSRoundRobin bool          `mapstructure:"grpc_client_dns_roundrobin" yaml:"grpc_client_dns_roundrobin,omitempty"`
 
+	//GRPCServerMaxConnectionAge sets MaxConnectionAge in the grpc ServerParameters used to create GRPC Services
+	GRPCServerMaxConnectionAge time.Duration `mapstructure:"grpc_server_max_connection_age" yaml:"grpc_server_max_connection_age,omitempty"`
+	//GRPCServerMaxConnectionAgeGrace sets MaxConnectionAgeGrace in the grpc ServerParameters used to create GRPC Services
+	GRPCServerMaxConnectionAgeGrace time.Duration `mapstructure:"grpc_server_max_connection_age_grace,omitempty" yaml:"grpc_server_max_connection_age_grace,omitempty"` //nolint: lll
+
 	// ForwardAuthEndpoint allows for a given route to be used as a forward-auth
 	// endpoint instead of a reverse proxy. Some third-party proxies that do not
 	// have rich access control capabilities (nginx, envoy, ambassador, traefik)
@@ -226,6 +231,8 @@ var defaultOptions = Options{
 	GRPCAddr:                        ":443",
 	GRPCClientTimeout:               10 * time.Second, // Try to withstand transient service failures for a single request
 	GRPCClientDNSRoundRobin:         true,
+	GRPCServerMaxConnectionAge:      5 * time.Minute,
+	GRPCServerMaxConnectionAgeGrace: 5 * time.Minute,
 	CacheStore:                      "autocache",
 	AuthenticateCallbackPath:        "/oauth2/callback",
 }

config/options_test.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
@@ -224,6 +225,8 @@ func TestOptionsFromViper(t *testing.T) {
 				CookieSecure:                    true,
 				InsecureServer:                  true,
 				CookieHTTPOnly:                  true,
+				GRPCServerMaxConnectionAge:      5 * time.Minute,
+				GRPCServerMaxConnectionAgeGrace: 5 * time.Minute,
 				AuthenticateCallbackPath:        "/oauth2/callback",
 				Headers: map[string]string{
 					"Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload",
@@ -240,6 +243,8 @@ func TestOptionsFromViper(t *testing.T) {
 				CookieSecure:                    true,
 				CookieHTTPOnly:                  true,
 				InsecureServer:                  true,
+				GRPCServerMaxConnectionAge:      5 * time.Minute,
+				GRPCServerMaxConnectionAgeGrace: 5 * time.Minute,
 				Headers:                         map[string]string{}},
 			false},
 		{"bad url", []byte(`{"policy":[{"from": "https://","to":"https://to.example"}]}`), nil, true},

docs/configuration/readme.md

@@ -198,6 +198,30 @@ Enable grpc DNS based round robin load balancing. This method uses DNS to resolv
 - Type: `bool`
 - Default: `true`
 
+#### GRPC Server Max Connection Age
+
+Set max connection age for GRPC servers.  After this interval, servers ask clients to reconnect and perform any rediscovery for new/updated endpoints from DNS.  
+
+See https://godoc.org/google.golang.org/grpc/keepalive#ServerParameters for details
+
+- Environmental Variable: `GRPC_SERVER_MAX_CONNECTION_AGE`
+- Config File Key: `grpc_server_max_connection_age`
+- Type: [Go Duration](https://golang.org/pkg/time/#Duration.String) `string`
+- Default: `5m`
+
+
+#### GRPC Server Max Connection Age Grace
+
+Additive period with `grpc_server_max_connection_age`, after which servers will force connections to close.
+
+See https://godoc.org/google.golang.org/grpc/keepalive#ServerParameters for details
+
+- Environmental Variable: `GRPC_SERVER_MAX_CONNECTION_AGE_GRACE`
+- Config File Key: `grpc_server_max_connection_age_grace`
+- Type: [Go Duration](https://golang.org/pkg/time/#Duration.String) `string`
+- Default: `5m`
+
+
 ### Cookie options
 
 These settings control the Pomerium session cookies sent to users's

internal/grpc/server.go

@@ -13,6 +13,7 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/keepalive"
 )
 
 // NewServer creates a new gRPC serve.
@@ -27,7 +28,10 @@ func NewServer(opt *ServerOptions, registrationFn func(s *grpc.Server), wg *sync
 	if err != nil {
 		return nil, err
 	}
-	grpcOpts := []grpc.ServerOption{grpc.StatsHandler(metrics.NewGRPCServerStatsHandler(opt.Addr))}
+	grpcOpts := []grpc.ServerOption{
+		grpc.StatsHandler(metrics.NewGRPCServerStatsHandler(opt.Addr)),
+		grpc.KeepaliveParams(opt.KeepaliveParams),
+	}
 
 	if opt.TLSCertificate != nil {
 		log.Debug().Str("addr", opt.Addr).Msg("internal/grpc: serving over TLS")
@@ -65,6 +69,8 @@ type ServerOptions struct {
 	// In this mode, Pomerium is susceptible to man-in-the-middle attacks.
 	// This should be used only for testing.
 	InsecureServer bool
+
+	KeepaliveParams keepalive.ServerParameters
 }
 
 var defaultServerOptions = &ServerOptions{

internal/grpc/server_test.go

@@ -10,7 +10,9 @@ import (
 	"time"
 
 	"github.com/pomerium/pomerium/internal/cryptutil"
+
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/keepalive"
 )
 
 const privKey = `-----BEGIN EC PRIVATE KEY-----
@@ -46,6 +48,7 @@ func TestNewServer(t *testing.T) {
 		wantErr        bool
 	}{
 		{"simple", &ServerOptions{Addr: ":0"}, func(s *grpc.Server) {}, &sync.WaitGroup{}, false, false},
+		{"simple keepalive options", &ServerOptions{Addr: ":0", KeepaliveParams: keepalive.ServerParameters{MaxConnectionAge: 5 * time.Minute}}, func(s *grpc.Server) {}, &sync.WaitGroup{}, false, false},
 		{"bad tcp port", &ServerOptions{Addr: ":9999999"}, func(s *grpc.Server) {}, &sync.WaitGroup{}, true, true},
 		{"with certs", &ServerOptions{Addr: ":0", TLSCertificate: certb64}, func(s *grpc.Server) {}, &sync.WaitGroup{}, false, false},
 	}