core/authorize: result denied improvements (#4952)

* core/authorize: result denied improvements

* add authenticate robots.txt

* fix tests

proxy/handlers.go

@@ -2,7 +2,6 @@ package proxy
 
 import (
 	"errors"
-	"fmt"
 	"io"
 	"net/http"
 	"net/url"
@@ -43,13 +42,6 @@ func (p *Proxy) registerDashboardHandlers(r *mux.Router) *mux.Router {
 	return r
 }
 
-// RobotsTxt sets the User-Agent header in the response to be "Disallow"
-func (p *Proxy) RobotsTxt(w http.ResponseWriter, _ *http.Request) {
-	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
-	w.WriteHeader(http.StatusOK)
-	fmt.Fprintf(w, "User-agent: *\nDisallow: /")
-}
-
 // SignOut clears the local session and redirects the request to the sign out url.
 // It's the responsibility of the authenticate service to revoke the remote session and clear
 // the authenticate service's session state.

proxy/handlers_test.go

@@ -16,20 +16,6 @@ import (
 	"github.com/pomerium/pomerium/internal/urlutil"
 )
 
-func TestProxy_RobotsTxt(t *testing.T) {
-	proxy := Proxy{}
-	req := httptest.NewRequest(http.MethodGet, "/robots.txt", nil)
-	rr := httptest.NewRecorder()
-	proxy.RobotsTxt(rr, req)
-	if status := rr.Code; status != http.StatusOK {
-		t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusOK)
-	}
-	expected := "User-agent: *\nDisallow: /"
-	if rr.Body.String() != expected {
-		t.Errorf("handler returned wrong body: got %v want %v", rr.Body.String(), expected)
-	}
-}
-
 func TestProxy_SignOut(t *testing.T) {
 	t.Parallel()
 	tests := []struct {

proxy/proxy.go

@@ -112,7 +112,6 @@ func (p *Proxy) setHandlers(opts *config.Options) error {
 	})
 	r.SkipClean(true)
 	r.StrictSlash(true)
-	r.HandleFunc("/robots.txt", p.RobotsTxt).Methods(http.MethodGet)
 	// dashboard handlers are registered to all routes
 	r = p.registerDashboardHandlers(r)