proxy: disable control-plane robots.txt for public unauthenticated routes (#1361)

authorize/grpc.go

@@ -6,7 +6,6 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
-	"regexp"
 	"strings"
 
 	"github.com/golang/protobuf/ptypes"
@@ -243,35 +242,10 @@ func (a *Authorize) getMatchingPolicy(requestURL *url.URL) *config.Policy {
 	options := a.currentOptions.Load()
 
 	for _, p := range options.Policies {
-		if p.Source == nil {
+		if p.Matches(requestURL) {
-			continue
-		}
-
-		if p.Source.Host != requestURL.Host {
-			continue
-		}
-
-		if p.Prefix != "" {
-			if !strings.HasPrefix(requestURL.Path, p.Prefix) {
-				continue
-			}
-		}
-
-		if p.Path != "" {
-			if requestURL.Path != p.Path {
-				continue
-			}
-		}
-
-		if p.Regex != "" {
-			re, err := regexp.Compile(p.Regex)
-			if err == nil && !re.MatchString(requestURL.String()) {
-				continue
-			}
-		}
-
 			return &p
 		}
+	}
 
 	return nil
 }

config/policy.go

@@ -8,6 +8,8 @@ import (
 	"io/ioutil"
 	"net/url"
 	"os"
+	"regexp"
+	"strings"
 	"time"
 
 	"github.com/cespare/xxhash/v2"
@@ -319,6 +321,39 @@ func (p *Policy) String() string {
 	return fmt.Sprintf("%s → %s", p.Source.String(), p.Destination.String())
 }
 
+// Matches returns true if the policy would match the given URL.
+func (p *Policy) Matches(requestURL *url.URL) bool {
+	// handle nils by always returning false
+	if p.Source == nil || requestURL == nil {
+		return false
+	}
+
+	if p.Source.Host != requestURL.Host {
+		return false
+	}
+
+	if p.Prefix != "" {
+		if !strings.HasPrefix(requestURL.Path, p.Prefix) {
+			return false
+		}
+	}
+
+	if p.Path != "" {
+		if requestURL.Path != p.Path {
+			return false
+		}
+	}
+
+	if p.Regex != "" {
+		re, err := regexp.Compile(p.Regex)
+		if err == nil && !re.MatchString(requestURL.String()) {
+			return false
+		}
+	}
+
+	return true
+}
+
 // StringURL stores a URL as a string in json.
 type StringURL struct {
 	*url.URL

internal/controlplane/xds_listeners_test.go

@@ -86,21 +86,6 @@ func Test_buildMainHTTPConnectionManagerFilter(t *testing.T) {
 						"name": "example.com",
 						"domains": ["example.com"],
 						"routes": [
-							{
-								"name": "pomerium-path-/robots.txt",
-								"match": {
-									"path": "/robots.txt"
-								},
-								"route": {
-									"cluster": "pomerium-control-plane-http"
-								},
-								"typedPerFilterConfig": {
-									"envoy.filters.http.ext_authz": {
-										"@type": "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute",
-										"disabled": true
-									}
-								}
-							},
 							{
 								"name": "pomerium-path-/ping",
 								"match": {
@@ -190,6 +175,21 @@ func Test_buildMainHTTPConnectionManagerFilter(t *testing.T) {
 										"disabled": true
 									}
 								}
+							},
+							{
+								"name": "pomerium-path-/robots.txt",
+								"match": {
+									"path": "/robots.txt"
+								},
+								"route": {
+									"cluster": "pomerium-control-plane-http"
+								},
+								"typedPerFilterConfig": {
+									"envoy.filters.http.ext_authz": {
+										"@type": "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute",
+										"disabled": true
+									}
+								}
 							}
 						]
 					},
@@ -197,21 +197,6 @@ func Test_buildMainHTTPConnectionManagerFilter(t *testing.T) {
 						"name": "catch-all",
 						"domains": ["*"],
 						"routes": [
-							{
-								"name": "pomerium-path-/robots.txt",
-								"match": {
-									"path": "/robots.txt"
-								},
-								"route": {
-									"cluster": "pomerium-control-plane-http"
-								},
-								"typedPerFilterConfig": {
-									"envoy.filters.http.ext_authz": {
-										"@type": "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute",
-										"disabled": true
-									}
-								}
-							},
 							{
 								"name": "pomerium-path-/ping",
 								"match": {
@@ -301,6 +286,21 @@ func Test_buildMainHTTPConnectionManagerFilter(t *testing.T) {
 										"disabled": true
 									}
 								}
+							},
+							{
+								"name": "pomerium-path-/robots.txt",
+								"match": {
+									"path": "/robots.txt"
+								},
+								"route": {
+									"cluster": "pomerium-control-plane-http"
+								},
+								"typedPerFilterConfig": {
+									"envoy.filters.http.ext_authz": {
+										"@type": "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute",
+										"disabled": true
+									}
+								}
 							}
 						]
 					}

internal/controlplane/xds_routes.go

@@ -2,6 +2,7 @@ package controlplane
 
 import (
 	"fmt"
+	"net/url"
 
 	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	envoy_config_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
@@ -41,7 +42,6 @@ func buildGRPCRoutes() []*envoy_config_route_v3.Route {
 
 func buildPomeriumHTTPRoutes(options *config.Options, domain string) []*envoy_config_route_v3.Route {
 	routes := []*envoy_config_route_v3.Route{
-		buildControlPlanePathRoute("/robots.txt"),
 		buildControlPlanePathRoute("/ping"),
 		buildControlPlanePathRoute("/healthz"),
 		buildControlPlanePathRoute("/.pomerium"),
@@ -49,6 +49,10 @@ func buildPomeriumHTTPRoutes(options *config.Options, domain string) []*envoy_co
 		buildControlPlanePathRoute("/.well-known/pomerium"),
 		buildControlPlanePrefixRoute("/.well-known/pomerium/"),
 	}
+	// per #837, only add robots.txt if there are no unauthenticated routes
+	if !hasPublicPolicyMatchingURL(options, mustParseURL("https://"+domain+"/robots.txt")) {
+		routes = append(routes, buildControlPlanePathRoute("/robots.txt"))
+	}
 	// if we're handling authentication, add the oauth2 callback url
 	if config.IsAuthenticate(options.Services) && hostMatchesDomain(options.GetAuthenticateURL(), domain) {
 		routes = append(routes, buildControlPlanePathRoute(options.AuthenticateCallbackPath))
@@ -246,3 +250,20 @@ func getPrefixRewrite(policy *config.Policy) string {
 	}
 	return prefixRewrite
 }
+
+func hasPublicPolicyMatchingURL(options *config.Options, requestURL *url.URL) bool {
+	for _, policy := range options.Policies {
+		if policy.AllowPublicUnauthenticatedAccess && policy.Matches(requestURL) {
+			return true
+		}
+	}
+	return false
+}
+
+func mustParseURL(str string) *url.URL {
+	u, err := url.Parse(str)
+	if err != nil {
+		panic(err)
+	}
+	return u
+}

internal/controlplane/xds_routes_test.go

@@ -2,7 +2,6 @@ package controlplane
 
 import (
 	"fmt"
-	"net/url"
 	"testing"
 	"time"
 
@@ -43,137 +42,94 @@ func Test_buildGRPCRoutes(t *testing.T) {
 }
 
 func Test_buildPomeriumHTTPRoutes(t *testing.T) {
-	routes := buildPomeriumHTTPRoutes(&config.Options{
+	routeString := func(typ, name string) string {
+		return `{
+				"name": "pomerium-` + typ + `-` + name + `",
+				"match": {
+					"` + typ + `": "` + name + `"
+				},
+				"route": {
+					"cluster": "pomerium-control-plane-http"
+				},
+				"typedPerFilterConfig": {
+					"envoy.filters.http.ext_authz": {
+						"@type": "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute",
+						"disabled": true
+					}
+				}
+			}`
+	}
+
+	t.Run("authenticate", func(t *testing.T) {
+		options := &config.Options{
 			Services:                 "all",
 			AuthenticateURL:          mustParseURL("https://authenticate.example.com"),
 			AuthenticateCallbackPath: "/oauth2/callback",
 			ForwardAuthURL:           mustParseURL("https://forward-auth.example.com"),
-	}, "authenticate.example.com")
+		}
+		routes := buildPomeriumHTTPRoutes(options, "authenticate.example.com")
 
-	testutil.AssertProtoJSONEqual(t, `
+		testutil.AssertProtoJSONEqual(t, `[
-		[
+			`+routeString("path", "/ping")+`,
-			{
+			`+routeString("path", "/healthz")+`,
-				"name": "pomerium-path-/robots.txt",
+			`+routeString("path", "/.pomerium")+`,
-				"match": {
+			`+routeString("prefix", "/.pomerium/")+`,
-					"path": "/robots.txt"
+			`+routeString("path", "/.well-known/pomerium")+`,
-				},
+			`+routeString("prefix", "/.well-known/pomerium/")+`,
-				"route": {
+			`+routeString("path", "/robots.txt")+`,
-					"cluster": "pomerium-control-plane-http"
+			`+routeString("path", "/oauth2/callback")+`
-				},
+		]`, routes)
-				"typedPerFilterConfig": {
+	})
-					"envoy.filters.http.ext_authz": {
+
-						"@type": "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute",
+	t.Run("with robots", func(t *testing.T) {
-						"disabled": true
+		options := &config.Options{
+			Services:                 "all",
+			AuthenticateURL:          mustParseURL("https://authenticate.example.com"),
+			AuthenticateCallbackPath: "/oauth2/callback",
+			ForwardAuthURL:           mustParseURL("https://forward-auth.example.com"),
+			Policies: []config.Policy{{
+				From: "https://from.example.com",
+				To:   "https://to.example.com",
+			}},
 		}
+		_ = options.Policies[0].Validate()
+		routes := buildPomeriumHTTPRoutes(options, "from.example.com")
+
+		testutil.AssertProtoJSONEqual(t, `[
+			`+routeString("path", "/ping")+`,
+			`+routeString("path", "/healthz")+`,
+			`+routeString("path", "/.pomerium")+`,
+			`+routeString("prefix", "/.pomerium/")+`,
+			`+routeString("path", "/.well-known/pomerium")+`,
+			`+routeString("prefix", "/.well-known/pomerium/")+`,
+			`+routeString("path", "/robots.txt")+`
+		]`, routes)
+	})
+
+	t.Run("without robots", func(t *testing.T) {
+		options := &config.Options{
+			Services:                 "all",
+			AuthenticateURL:          mustParseURL("https://authenticate.example.com"),
+			AuthenticateCallbackPath: "/oauth2/callback",
+			ForwardAuthURL:           mustParseURL("https://forward-auth.example.com"),
+			Policies: []config.Policy{{
+				From:                             "https://from.example.com",
+				To:                               "https://to.example.com",
+				AllowPublicUnauthenticatedAccess: true,
+			}},
 		}
-			},
+		_ = options.Policies[0].Validate()
-			{
+		routes := buildPomeriumHTTPRoutes(options, "from.example.com")
-				"name": "pomerium-path-/ping",
+
-				"match": {
+		testutil.AssertProtoJSONEqual(t, `[
-					"path": "/ping"
+			`+routeString("path", "/ping")+`,
-				},
+			`+routeString("path", "/healthz")+`,
-				"route": {
+			`+routeString("path", "/.pomerium")+`,
-					"cluster": "pomerium-control-plane-http"
+			`+routeString("prefix", "/.pomerium/")+`,
-				},
+			`+routeString("path", "/.well-known/pomerium")+`,
-				"typedPerFilterConfig": {
+			`+routeString("prefix", "/.well-known/pomerium/")+`
-					"envoy.filters.http.ext_authz": {
+		]`, routes)
-						"@type": "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute",
+	})
-						"disabled": true
-					}
-				}
-			},
-			{
-				"name": "pomerium-path-/healthz",
-				"match": {
-					"path": "/healthz"
-				},
-				"route": {
-					"cluster": "pomerium-control-plane-http"
-				},
-				"typedPerFilterConfig": {
-					"envoy.filters.http.ext_authz": {
-						"@type": "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute",
-						"disabled": true
-					}
-				}
-			},
-			{
-				"name": "pomerium-path-/.pomerium",
-				"match": {
-					"path": "/.pomerium"
-				},
-				"route": {
-					"cluster": "pomerium-control-plane-http"
-				},
-				"typedPerFilterConfig": {
-					"envoy.filters.http.ext_authz": {
-						"@type": "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute",
-						"disabled": true
-					}
-				}
-			},
-			{
-				"name": "pomerium-prefix-/.pomerium/",
-				"match": {
-					"prefix": "/.pomerium/"
-				},
-				"route": {
-					"cluster": "pomerium-control-plane-http"
-				},
-				"typedPerFilterConfig": {
-					"envoy.filters.http.ext_authz": {
-						"@type": "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute",
-						"disabled": true
-					}
-				}
-			},
-			{
-				"name": "pomerium-path-/.well-known/pomerium",
-				"match": {
-					"path": "/.well-known/pomerium"
-				},
-				"route": {
-					"cluster": "pomerium-control-plane-http"
-				},
-				"typedPerFilterConfig": {
-					"envoy.filters.http.ext_authz": {
-						"@type": "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute",
-						"disabled": true
-					}
-				}
-			},
-			{
-				"name": "pomerium-prefix-/.well-known/pomerium/",
-				"match": {
-					"prefix": "/.well-known/pomerium/"
-				},
-				"route": {
-					"cluster": "pomerium-control-plane-http"
-				},
-				"typedPerFilterConfig": {
-					"envoy.filters.http.ext_authz": {
-						"@type": "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute",
-						"disabled": true
-					}
-				}
-			},
-			{
-				"name": "pomerium-path-/oauth2/callback",
-				"match": {
-					"path": "/oauth2/callback"
-				},
-				"route": {
-					"cluster": "pomerium-control-plane-http"
-				},
-				"typedPerFilterConfig": {
-					"envoy.filters.http.ext_authz": {
-						"@type": "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute",
-						"disabled": true
-					}
-				}
-			}
-		]
-	`, routes)
 }
 
 func Test_buildControlPlanePathRoute(t *testing.T) {
@@ -589,11 +545,3 @@ func Test_buildPolicyRoutesWithDestinationPath(t *testing.T) {
 		]
 	`, routes)
 }
-
-func mustParseURL(str string) *url.URL {
-	u, err := url.Parse(str)
-	if err != nil {
-		panic(err)
-	}
-	return u
-}