telemetry: add tracing

- telemetry/tace: add traces throughout code
- telemetry/metrics: nest metrics and trace under telemetry
- telemetry/tace: add service name span to HTTPMetricsHandler.
- telemetry/metrics: removed chain dependency middleware_tests.
- telemetry/metrics: wrap and encapsulate variatic view registration.
- telemetry/tace: add jaeger support for tracing.
- cmd/pomerium: move `parseOptions` to internal/config.
- cmd/pomerium: offload server handling to httputil and sub pkgs.
- httputil: standardize creation/shutdown of http listeners.
- httputil: prefer curve X25519 to P256 when negotiating TLS.
- fileutil: use standardized Getw

Signed-off-by: Bobby DeSimone <bobbydesimone@gmail.com>

CHANGELOG.md

@@ -4,27 +4,33 @@
 
 ### New
 
+#### Telemetry [GH-35]
+
+- **Tracing** [GH-230] aka distributed tracing, provides insight into the full lifecycles, aka traces, of requests to the system, allowing you to pinpoint failures and performance issues.
+
+  - Add [Jaeger](https://opencensus.io/exporters/supported-exporters/go/jaeger/) support. [GH-230]
+
+- **Metrics** provide quantitative information about processes running inside the system, including counters, gauges, and histograms.
+
+  - Add informational metrics. [GH-227]
+  - GRPC Metrics Implementation. [GH-218]
+
+    - Additional GRPC server metrics and request sizes
+    - Improved GRPC metrics implementation internals
+    - The GRPC method label is now 'grpc_method' and GRPC status is now `grpc_client_status` and `grpc_server_status`
+
+  - HTTP Metrics Implementation. [GH-220]
+
+    - Support HTTP request sizes on client and server side of proxy
+    - Improved HTTP metrics implementation internals
+    - The HTTP method label is now `http_method`, and HTTP status label is now `http_status`
+
 ### Changed
 
-- GRPC Metrics Implementation [GH-218]
-
-  - Additional GRPC server metrics and request sizes
-  - Improved GRPC metrics implementation internals
-  - The GRPC method label is now 'grpc_method' and GRPC status is now `grpc_client_status` and `grpc_server_status`
-
 - GRPC version upgraded to v1.22 [GH-219]
-
-- HTTP Metrics Implementation [GH-220]
-
-  - Support HTTP request sizes on client and server side of proxy
-  - Improved HTTP metrics implementation internals
-  - The HTTP method label is now `http_method`, and HTTP status label is now `http_status`
-
 - Add support for large cookie sessions by chunking. [GH-211]
-
 - Prefer [curve](https://wiki.mozilla.org/Security/Server_Side_TLS) X25519 to P256 for TLS connections. [GH-233]
-
-- Add informational metrics. [GH-227]
+- Pomerium and its services will gracefully shutdown on [interrupt signal](http://man7.org/linux/man-pages/man7/signal.7.html). [GH-230]
 - [Google](https://developers.google.com/identity/protocols/OpenIDConnect) now prompts the user to select a user account (by adding `select_account` to the sign in url). This allows a user who has multiple accounts at the authorization server to select amongst the multiple accounts that they may have current sessions for.
 
 ## v0.1.0

authenticate/grpc.go

@@ -6,11 +6,14 @@ import (
 	"fmt"
 
 	"github.com/pomerium/pomerium/internal/sessions"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	pb "github.com/pomerium/pomerium/proto/authenticate"
 )
 
 // Authenticate takes an encrypted code, and returns the authentication result.
 func (p *Authenticate) Authenticate(ctx context.Context, in *pb.AuthenticateRequest) (*pb.Session, error) {
+	_, span := trace.StartSpan(ctx, "authenticate.grpc.Validate")
+	defer span.End()
 	session, err := sessions.UnmarshalSession(in.Code, p.cipher)
 	if err != nil {
 		return nil, fmt.Errorf("authenticate/grpc: authenticate %v", err)
@@ -25,6 +28,9 @@ func (p *Authenticate) Authenticate(ctx context.Context, in *pb.AuthenticateRequ
 // Validate locally validates a JWT id_token; does NOT do nonce or revokation validation.
 // https://openid.net/specs/openid-connect-core-1_0.html#IDTokenValidation
 func (p *Authenticate) Validate(ctx context.Context, in *pb.ValidateRequest) (*pb.ValidateReply, error) {
+	ctx, span := trace.StartSpan(ctx, "authenticate.grpc.Validate")
+	defer span.End()
+
 	isValid, err := p.provider.Validate(ctx, in.IdToken)
 	if err != nil {
 		return &pb.ValidateReply{IsValid: false}, fmt.Errorf("authenticate/grpc: validate %v", err)
@@ -35,10 +41,8 @@ func (p *Authenticate) Validate(ctx context.Context, in *pb.ValidateRequest) (*p
 // Refresh renews a user's session checks if the session has been revoked using an access token
 // without reprompting the user.
 func (p *Authenticate) Refresh(ctx context.Context, in *pb.Session) (*pb.Session, error) {
-	// todo(bdd): add request id from incoming context
-	// md, _ := metadata.FromIncomingContext(ctx)
-	// sublogger := log.With().Str("req_id", md.Get("req_id")[0]).WithContext(ctx)
-	// sublogger.Info().Msg("tracing sucks!")
+	ctx, span := trace.StartSpan(ctx, "authenticate.grpc.Refresh")
+	defer span.End()
 	if in == nil {
 		return nil, fmt.Errorf("authenticate/grpc: session cannot be nil")
 	}

authorize/authorize.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/pomerium/pomerium/internal/config"
 	"github.com/pomerium/pomerium/internal/log"
-	"github.com/pomerium/pomerium/internal/metrics"
+	"github.com/pomerium/pomerium/internal/telemetry/metrics"
 )
 
 // ValidateOptions checks to see if configuration values are valid for the

authorize/gprc.go

@@ -4,12 +4,16 @@ package authorize // import "github.com/pomerium/pomerium/authorize"
 import (
 	"context"
 
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	pb "github.com/pomerium/pomerium/proto/authorize"
 )
 
 // Authorize validates the user identity, device, and context of a request for
 // a given route. Currently only checks identity.
 func (a *Authorize) Authorize(ctx context.Context, in *pb.Identity) (*pb.AuthorizeReply, error) {
+	_, span := trace.StartSpan(ctx, "authorize.grpc.Authorize")
+	defer span.End()
+
 	ok := a.ValidIdentity(in.Route,
 		&Identity{
 			User:              in.User,
@@ -23,6 +27,8 @@ func (a *Authorize) Authorize(ctx context.Context, in *pb.Identity) (*pb.Authori
 
 // IsAdmin validates the user is an administrative user.
 func (a *Authorize) IsAdmin(ctx context.Context, in *pb.Identity) (*pb.IsAdminReply, error) {
+	_, span := trace.StartSpan(ctx, "authorize.grpc.IsAdmin")
+	defer span.End()
 	ok := a.identityAccess.IsAdmin(
 		&Identity{
 			Email:  in.Email,

cmd/pomerium/main.go

@@ -1,12 +1,10 @@
 package main // import "github.com/pomerium/pomerium/cmd/pomerium"
 
 import (
-	"errors"
 	"flag"
 	"fmt"
 	"net/http"
 	"os"
-	"strconv"
 	"time"
 
 	"github.com/fsnotify/fsnotify"
@@ -18,8 +16,9 @@ import (
 	"github.com/pomerium/pomerium/internal/config"
 	"github.com/pomerium/pomerium/internal/httputil"
 	"github.com/pomerium/pomerium/internal/log"
-	"github.com/pomerium/pomerium/internal/metrics"
 	"github.com/pomerium/pomerium/internal/middleware"
+	"github.com/pomerium/pomerium/internal/telemetry/metrics"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/urlutil"
 	"github.com/pomerium/pomerium/internal/version"
 	pbAuthenticate "github.com/pomerium/pomerium/proto/authenticate"
@@ -36,17 +35,18 @@ func main() {
 		fmt.Println(version.FullVersion())
 		os.Exit(0)
 	}
-	opt, err := parseOptions(*configFile)
+	opt, err := config.ParseOptions(*configFile)
 	if err != nil {
 		log.Fatal().Err(err).Msg("cmd/pomerium: options")
 	}
 	log.Info().Str("version", version.FullVersion()).Msg("cmd/pomerium")
-	grpcAuth := middleware.NewSharedSecretCred(opt.SharedKey)
-	grpcOpts := []grpc.ServerOption{grpc.UnaryInterceptor(grpcAuth.ValidateRequest), grpc.StatsHandler(metrics.NewGRPCServerStatsHandler(opt.Services))}
-	grpcServer := grpc.NewServer(grpcOpts...)
+
+	setupMetrics(opt)
+	setupTracing(opt)
+	setupHTTPRedirectServer(opt)
 
 	mux := http.NewServeMux()
-
+	grpcServer := setupGRPCServer(opt)
 	_, err = newAuthenticateService(*opt, mux, grpcServer)
 	if err != nil {
 		log.Fatal().Err(err).Msg("cmd/pomerium: authenticate")
@@ -61,65 +61,23 @@ func main() {
 	if err != nil {
 		log.Fatal().Err(err).Msg("cmd/pomerium: proxy")
 	}
+	defer proxy.AuthenticateClient.Close()
+	defer proxy.AuthorizeClient.Close()
+
 	go viper.WatchConfig()
 
 	viper.OnConfigChange(func(e fsnotify.Event) {
-		log.Info().
-			Str("file", e.Name).
-			Msg("cmd/pomerium: configuration file changed")
-
-		opt = handleConfigUpdate(opt, []config.OptionsUpdater{authz, proxy})
+		log.Info().Str("file", e.Name).Msg("cmd/pomerium: config file changed")
+		opt = config.HandleConfigUpdate(*configFile, opt, []config.OptionsUpdater{authz, proxy})
 	})
-	// defer statements ignored anyway :  https://stackoverflow.com/a/17888654
-	// defer proxyService.AuthenticateClient.Close()
-	// defer proxyService.AuthorizeClient.Close()
 
-	httpOpts := &httputil.Options{
-		Addr:              opt.Addr,
-		Cert:              opt.Cert,
-		Key:               opt.Key,
-		CertFile:          opt.CertFile,
-		KeyFile:           opt.KeyFile,
-		ReadTimeout:       opt.ReadTimeout,
-		WriteTimeout:      opt.WriteTimeout,
-		ReadHeaderTimeout: opt.ReadHeaderTimeout,
-		IdleTimeout:       opt.IdleTimeout,
+	srv, err := httputil.NewTLSServer(configToServerOptions(opt), mainHandler(opt, mux), grpcServer)
+	if err != nil {
+		log.Fatal().Err(err).Msg("cmd/pomerium: couldn't start pomerium")
 	}
+	httputil.Shutdown(srv)
 
-	if opt.MetricsAddr != "" {
-		go newPromListener(opt.MetricsAddr)
-		metrics.SetBuildInfo(opt.Services)
-	}
-
-	if srv, err := startRedirectServer(opt.HTTPRedirectAddr); err != nil {
-		log.Debug().Str("cause", err.Error()).Msg("cmd/pomerium: http redirect server not started")
-	} else {
-		defer srv.Close()
-	}
-
-	if err := httputil.ListenAndServeTLS(httpOpts, wrapMiddleware(opt, mux), grpcServer); err != nil {
-		log.Fatal().Err(err).Msg("cmd/pomerium: https server")
-	}
-}
-
-// startRedirectServer starts a http server that redirect HTTP to HTTPS traffic
-func startRedirectServer(addr string) (*http.Server, error) {
-	if addr == "" {
-		return nil, errors.New("no http redirect addr provided")
-	}
-	srv := &http.Server{
-		Addr:         addr,
-		ReadTimeout:  5 * time.Second,
-		WriteTimeout: 5 * time.Second,
-		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			w.Header().Set("Connection", "close")
-			url := fmt.Sprintf("https://%s%s", urlutil.StripPort(r.Host), r.URL.String())
-			http.Redirect(w, r, url, http.StatusMovedPermanently)
-		}),
-	}
-	log.Info().Str("Addr", addr).Msg("cmd/pomerium: http redirect server started")
-	go func() { log.Error().Err(srv.ListenAndServe()).Msg("cmd/pomerium: http server closed") }()
-	return srv, nil
+	os.Exit(0)
 }
 
 func newAuthenticateService(opt config.Options, mux *http.ServeMux, rpc *grpc.Server) (*authenticate.Authenticate, error) {
@@ -159,21 +117,9 @@ func newProxyService(opt config.Options, mux *http.ServeMux) (*proxy.Proxy, erro
 	return service, nil
 }
 
-func newPromListener(addr string) {
-	metrics.RegisterView(metrics.HTTPClientViews)
-	metrics.RegisterView(metrics.HTTPServerViews)
-	metrics.RegisterView(metrics.GRPCClientViews)
-	metrics.RegisterView(metrics.GRPCServerViews)
-	metrics.RegisterInfoMetrics()
-	metrics.RegisterView(metrics.InfoViews)
-
-	log.Info().Str("MetricsAddr", addr).Msg("cmd/pomerium: starting prometheus endpoint")
-	log.Error().Err(metrics.NewPromHTTPListener(addr)).Str("MetricsAddr", addr).Msg("cmd/pomerium: could not start metrics exporter")
-}
-
-func wrapMiddleware(o *config.Options, mux http.Handler) http.Handler {
+func mainHandler(o *config.Options, mux http.Handler) http.Handler {
 	c := middleware.NewChain()
-	c = c.Append(metrics.HTTPMetricsHandler("proxy"))
+	c = c.Append(metrics.HTTPMetricsHandler(o.Services))
 	c = c.Append(log.NewHandler(log.Logger))
 	c = c.Append(log.AccessHandler(func(r *http.Request, status, size int, duration time.Duration) {
 		log.FromRequest(r).Debug().
@@ -199,60 +145,61 @@ func wrapMiddleware(o *config.Options, mux http.Handler) http.Handler {
 	return c.Then(mux)
 }
 
-func parseOptions(configFile string) (*config.Options, error) {
-	o, err := config.OptionsFromViper(configFile)
-	if err != nil {
-		return nil, err
+func configToServerOptions(opt *config.Options) *httputil.ServerOptions {
+	return &httputil.ServerOptions{
+		Addr:              opt.Addr,
+		Cert:              opt.Cert,
+		Key:               opt.Key,
+		CertFile:          opt.CertFile,
+		KeyFile:           opt.KeyFile,
+		ReadTimeout:       opt.ReadTimeout,
+		WriteTimeout:      opt.WriteTimeout,
+		ReadHeaderTimeout: opt.ReadHeaderTimeout,
+		IdleTimeout:       opt.IdleTimeout,
 	}
-	if o.Debug {
-		log.SetDebugMode()
-	}
-	if o.LogLevel != "" {
-		log.SetLevel(o.LogLevel)
-	}
-	metrics.AddPolicyCountCallback(o.Services, func() int64 {
-		return int64(len(o.Policies))
-	})
-
-	checksumDec, err := strconv.ParseUint(o.Checksum(), 16, 64)
-	if err != nil {
-		log.Warn().Err(err).Msg("Could not parse config checksum into decimal")
-	}
-	metrics.SetConfigChecksum(o.Services, checksumDec)
-
-	return o, nil
 }
 
-func handleConfigUpdate(opt *config.Options, services []config.OptionsUpdater) *config.Options {
-	newOpt, err := parseOptions(*configFile)
-	if err != nil {
-		log.Error().Err(err).Msg("cmd/pomerium: could not reload configuration")
-		metrics.SetConfigInfo(opt.Services, false, "")
-		return opt
-	}
-	optChecksum := opt.Checksum()
-	newOptChecksum := newOpt.Checksum()
-
-	log.Debug().
-		Str("old-checksum", optChecksum).
-		Str("new-checksum", newOptChecksum).
-		Msg("cmd/pomerium: configuration file changed")
-
-	if newOptChecksum == optChecksum {
-		log.Debug().Msg("cmd/pomerium: loaded configuration has not changed")
-		return opt
-	}
-
-	log.Info().Str("checksum", newOptChecksum).Msg("cmd/pomerium: checksum changed")
-	for _, service := range services {
-		if err := service.UpdateOptions(*newOpt); err != nil {
-			log.Error().Err(err).Msg("cmd/pomerium: could not update options")
-			metrics.SetConfigInfo(opt.Services, false, "")
+func setupMetrics(opt *config.Options) {
+	if opt.MetricsAddr != "" {
+		if handler, err := metrics.PrometheusHandler(); err != nil {
+			log.Error().Err(err).Msg("cmd/pomerium: couldn't start metrics server")
+		} else {
+			serverOpts := &httputil.ServerOptions{Addr: opt.MetricsAddr}
+			srv := httputil.NewHTTPServer(serverOpts, handler)
+			go httputil.Shutdown(srv)
 		}
 	}
-	metrics.AddPolicyCountCallback(newOpt.Services, func() int64 {
-		return int64(len(newOpt.Policies))
-	})
-	metrics.SetConfigInfo(newOpt.Services, true, newOptChecksum)
-	return newOpt
+}
+
+func setupTracing(opt *config.Options) {
+	if opt.TracingProvider != "" {
+		tracingOpts := &trace.TracingOptions{
+			Provider:                opt.TracingProvider,
+			Service:                 opt.Services,
+			Debug:                   opt.TracingDebug,
+			JaegerAgentEndpoint:     opt.TracingJaegerAgentEndpoint,
+			JaegerCollectorEndpoint: opt.TracingJaegerCollectorEndpoint,
+		}
+		if err := trace.RegisterTracing(tracingOpts); err != nil {
+			log.Error().Err(err).Msg("cmd/pomerium: couldn't register tracing")
+		} else {
+			log.Info().Interface("options", tracingOpts).Msg("cmd/pomerium: metrics configured")
+		}
+	}
+}
+
+func setupHTTPRedirectServer(opt *config.Options) {
+	if opt.HTTPRedirectAddr != "" {
+		serverOpts := httputil.ServerOptions{Addr: opt.HTTPRedirectAddr}
+		srv := httputil.NewHTTPServer(&serverOpts, httputil.RedirectHandler())
+		go httputil.Shutdown(srv)
+	}
+}
+
+func setupGRPCServer(opt *config.Options) *grpc.Server {
+	grpcAuth := middleware.NewSharedSecretCred(opt.SharedKey)
+	grpcOpts := []grpc.ServerOption{
+		grpc.UnaryInterceptor(grpcAuth.ValidateRequest),
+		grpc.StatsHandler(metrics.NewGRPCServerStatsHandler(opt.Services))}
+	return grpc.NewServer(grpcOpts...)
 }

cmd/pomerium/main_test.go

@@ -7,48 +7,19 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"os"
+	"os/signal"
 	"reflect"
-	"strings"
+	"syscall"
 	"testing"
+	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/pomerium/pomerium/internal/config"
+	"github.com/pomerium/pomerium/internal/httputil"
 	"github.com/pomerium/pomerium/internal/middleware"
 	"google.golang.org/grpc"
 )
 
-func Test_startRedirectServer(t *testing.T) {
-
-	tests := []struct {
-		name    string
-		addr    string
-		want    string
-		wantErr bool
-	}{
-		{"empty", "", "", true},
-		{":http", ":http", ":http", false},
-		{"localhost:80", "localhost:80", "localhost:80", false},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := startRedirectServer(tt.addr)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("startRedirectServer() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if got != nil {
-				defer got.Close()
-				ts := httptest.NewServer(got.Handler)
-				defer ts.Close()
-				_, err := http.Get(ts.URL)
-				if !strings.Contains(err.Error(), "https") {
-					t.Errorf("startRedirectServer() = %v, want %v", err, tt.want)
-					return
-				}
-			}
-		})
-	}
-}
-
 func Test_newAuthenticateService(t *testing.T) {
 	grpcAuth := middleware.NewSharedSecretCred("test")
 	grpcOpts := []grpc.ServerOption{grpc.UnaryInterceptor(grpcAuth.ValidateRequest)}
@@ -193,7 +164,7 @@ func Test_newProxyeService(t *testing.T) {
 	}
 }
 
-func Test_wrapMiddleware(t *testing.T) {
+func Test_mainHandler(t *testing.T) {
 	o := config.Options{
 		Services: "all",
 		Headers: map[string]string{
@@ -214,7 +185,7 @@ func Test_wrapMiddleware(t *testing.T) {
 	})
 
 	mux.Handle("/404", h)
-	out := wrapMiddleware(&o, mux)
+	out := mainHandler(&o, mux)
 	out.ServeHTTP(rr, req)
 	expected := fmt.Sprintf("OK")
 	body := rr.Body.String()
@@ -223,87 +194,106 @@ func Test_wrapMiddleware(t *testing.T) {
 		t.Errorf("handler returned unexpected body: got %v want %v", body, expected)
 	}
 }
-func Test_parseOptions(t *testing.T) {
+
+func Test_configToServerOptions(t *testing.T) {
 	tests := []struct {
-		name             string
-		envKey           string
-		envValue         string
-		servicesEnvKey   string
-		servicesEnvValue string
-		wantSharedKey    string
-		wantErr          bool
+		name string
+		opt  *config.Options
+		want *httputil.ServerOptions
 	}{
-		{"no shared secret", "", "", "SERVICES", "authenticate", "skip", true},
-		{"no shared secret in all mode", "", "", "", "", "", false},
-		{"good", "SHARED_SECRET", "YixWi1MYh77NMECGGIJQevoonYtVF+ZPRkQZrrmeRqM=", "", "", "YixWi1MYh77NMECGGIJQevoonYtVF+ZPRkQZrrmeRqM=", false},
+		{"simple convert", &config.Options{Addr: ":http"}, &httputil.ServerOptions{Addr: ":http"}},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			os.Setenv(tt.servicesEnvKey, tt.servicesEnvValue)
-			os.Setenv(tt.envKey, tt.envValue)
-			defer os.Unsetenv(tt.envKey)
-			defer os.Unsetenv(tt.servicesEnvKey)
-
-			got, err := parseOptions("")
-			if (err != nil) != tt.wantErr {
-				t.Errorf("parseOptions() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if got != nil && got.Services != "all" && got.SharedKey != tt.wantSharedKey {
-				t.Errorf("parseOptions()\n")
-				t.Errorf("got: %+v\n", got.SharedKey)
-				t.Errorf("want: %+v\n", tt.wantSharedKey)
-
+			if diff := cmp.Diff(configToServerOptions(tt.opt), tt.want); diff != "" {
+				t.Errorf("configToServerOptions() = \n %s", diff)
 			}
 		})
 	}
 }
 
-type mockService struct {
-	fail    bool
-	Updated bool
-}
-
-func (m *mockService) UpdateOptions(o config.Options) error {
-
-	m.Updated = true
-	if m.fail {
-		return fmt.Errorf("failed")
-	}
-	return nil
-}
-
-func Test_handleConfigUpdate(t *testing.T) {
-	os.Clearenv()
-	os.Setenv("SHARED_SECRET", "foo")
-	defer os.Unsetenv("SHARED_SECRET")
-
-	blankOpts, err := config.NewOptions("https://authenticate.example", "https://authorize.example")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	goodOpts, err := config.OptionsFromViper("")
-	if err != nil {
-		t.Fatal(err)
-	}
+func Test_setupGRPCServer(t *testing.T) {
 	tests := []struct {
-		name       string
-		service    *mockService
-		oldOpts    config.Options
-		wantUpdate bool
+		name     string
+		opt      *config.Options
+		dontWant *grpc.Server
 	}{
-		{"good", &mockService{fail: false}, *blankOpts, true},
-		{"bad", &mockService{fail: true}, *blankOpts, true},
-		{"no change", &mockService{fail: false}, *goodOpts, false},
+		{"good", &config.Options{SharedKey: "test"}, nil},
 	}
-
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			handleConfigUpdate(&tt.oldOpts, []config.OptionsUpdater{tt.service})
-			if tt.service.Updated != tt.wantUpdate {
-				t.Errorf("Failed to update config on service")
+			if diff := cmp.Diff(setupGRPCServer(tt.opt), tt.dontWant); diff == "" {
+				t.Errorf("setupGRPCServer() = \n %s", diff)
 			}
 		})
 	}
 }
+
+func Test_setupTracing(t *testing.T) {
+	tests := []struct {
+		name string
+		opt  *config.Options
+	}{
+		{"good jaeger", &config.Options{TracingProvider: "jaeger", TracingJaegerAgentEndpoint: "localhost:0", TracingJaegerCollectorEndpoint: "localhost:0"}},
+		{"dont register aything", &config.Options{}},
+		{"bad provider", &config.Options{TracingProvider: "bad provider"}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			setupTracing(tt.opt)
+		})
+	}
+}
+
+func Test_setupMetrics(t *testing.T) {
+	tests := []struct {
+		name string
+		opt  *config.Options
+	}{
+		{"dont register aything", &config.Options{}},
+		{"good metrics server", &config.Options{MetricsAddr: "localhost:0"}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := make(chan os.Signal, 1)
+			signal.Notify(c, syscall.SIGINT)
+			defer signal.Stop(c)
+			setupMetrics(tt.opt)
+			syscall.Kill(syscall.Getpid(), syscall.SIGINT)
+			waitSig(t, c, syscall.SIGINT)
+
+		})
+	}
+}
+
+func Test_setupHTTPRedirectServer(t *testing.T) {
+	tests := []struct {
+		name string
+		opt  *config.Options
+	}{
+		{"dont register aything", &config.Options{}},
+		{"good redirect server", &config.Options{HTTPRedirectAddr: "localhost:0"}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := make(chan os.Signal, 1)
+			signal.Notify(c, syscall.SIGINT)
+			defer signal.Stop(c)
+			setupHTTPRedirectServer(tt.opt)
+			syscall.Kill(syscall.Getpid(), syscall.SIGINT)
+			waitSig(t, c, syscall.SIGINT)
+
+		})
+	}
+}
+
+func waitSig(t *testing.T, c <-chan os.Signal, sig os.Signal) {
+	select {
+	case s := <-c:
+		if s != sig {
+			t.Fatalf("signal was %v, want %v", s, sig)
+		}
+	case <-time.After(1 * time.Second):
+		t.Fatalf("timeout waiting for %v", sig)
+	}
+}

docs/reference/readme.md

@@ -154,13 +154,17 @@ If set, the HTTP Redirect Address specifies the host and port to redirect http t
 - Environmental Variable: `METRICS_ADDRESS`
 - Config File Key: `metrics_address`
 - Type: `string`
-- Example: `:8080`, `127.0.0.1:9090`, ``
+- Example: `:9090`, `127.0.0.1:9090`
 - Default: `disabled`
 - Optional
 
 Expose a prometheus format HTTP endpoint on the specified port. Disabled by default.
 
-**Use with caution:** the endpoint can expose frontend and backend server names or addresses. Do not expose the metrics port if this is sensitive information.
+:::warning
+
+**Use with caution:** the endpoint can expose frontend and backend server names or addresses. Do not externally expose the metrics if this is sensitive information.
+
+:::
 
 #### Metrics tracked
 
@@ -187,6 +191,38 @@ pomerium_config_last_reload_success           | Gauge     | Whether the last con
 pomerium_config_last_reload_success_timestamp | Gauge     | The timestamp of the last successful configuration reload by service
 pomerium_build_info                           | Gauge     | Pomerium build metadata by git revision, service, version and goversion
 
+### Tracing
+
+Tracing tracks the progression of a single user request as it is handled by Pomerium.
+
+Each unit work is called a Span in a trace. Spans include metadata about the work, including the time spent in the step (latency), status, time events, attributes, links. You can use tracing to debug errors and latency issues in your applications, including in downstream connections.
+
+#### Shared Tracing Settings
+
+Config Key       | Description                                                       | Required
+:--------------- | :---------------------------------------------------------------- | --------
+tracing_provider | The name of the tracing provider. (e.g. jaeger)                   | ✅
+tracing_debug    | Will disable [sampling](https://opencensus.io/tracing/sampling/). | ❌
+
+#### Jaeger
+
+[Jaeger](https://www.jaegertracing.io/) is a distributed tracing system released as open source by Uber Technologies. It is used for monitoring and troubleshooting microservices-based distributed systems, including:
+
+- Distributed context propagation
+- Distributed transaction monitoring
+- Root cause analysis
+- Service dependency analysis
+- Performance / latency optimization
+
+Config Key                        | Description                                 | Required
+:-------------------------------- | :------------------------------------------ | --------
+tracing_jaeger_collector_endpoint | Url to the Jaeger HTTP Thrift collector.    | ✅
+tracing_jaeger_agent_endpoint     | Send spans to jaeger-agent at this address. | ✅
+
+##### Example
+
+![jaeger example trace](./tracing/jaeger.png) pomerium_config_last_reload_success_timestamp | Gauge | The timestamp of the last successful configuration reload by service pomerium_build_info | Gauge | Pomerium build metadata by git revision, service, version and goversion
+
 ### Policy
 
 - Environmental Variable: `POLICY`

go.mod

@@ -4,6 +4,7 @@ go 1.12
 
 require (
 	cloud.google.com/go v0.40.0 // indirect
+	contrib.go.opencensus.io/exporter/jaeger v0.1.0
 	contrib.go.opencensus.io/exporter/prometheus v0.1.0
 	github.com/fsnotify/fsnotify v1.4.7
 	github.com/golang/mock v1.3.1

go.sum

@@ -4,13 +4,19 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 cloud.google.com/go v0.40.0 h1:FjSY7bOj+WzJe6TZRVtXI2b9kAYvtNg4lMbcH2+MUkk=
 cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlro=
+contrib.go.opencensus.io/exporter/jaeger v0.1.0 h1:WNc9HbA38xEQmsI40Tjd/MNU/g8byN2Of7lwIjv0Jdc=
+contrib.go.opencensus.io/exporter/jaeger v0.1.0/go.mod h1:VYianECmuFPwU37O699Vc1GOcy+y8kOsfaxHRImmjbA=
 contrib.go.opencensus.io/exporter/prometheus v0.1.0 h1:SByaIoWwNgMdPSgl5sMqM2KDE5H/ukPWBRo314xiDvg=
 contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
+github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/apache/thrift v0.12.0 h1:pODnxUFNcjP9UTLZGTdeh+j16A8lJbRvD3rOtrk/7bs=
+github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
@@ -28,6 +34,9 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
+github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
+github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -36,6 +45,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
@@ -49,6 +59,7 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -57,6 +68,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
+github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
@@ -67,6 +80,7 @@ github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
@@ -91,10 +105,15 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg=
 github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
+github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -105,20 +124,25 @@ github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 h1:J9b7z+QKAm
 github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
+github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
 github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.14.3 h1:4EGfSkR2hJDB0s3oFfrlPqjU1e4WLncergLil3nEKW0=
@@ -151,6 +175,7 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@@ -171,10 +196,12 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -195,11 +222,14 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -216,6 +246,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -226,6 +257,8 @@ golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBn
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
+google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.6.0 h1:2tJEkRfnZL5g1GeBUlITh/rqT5HG3sFcoVCUUxmgJ2g=
 google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4=
@@ -243,6 +276,7 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn
 google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
 google.golang.org/genproto v0.0.0-20190611190212-a7e196e89fd3 h1:0LGHEA/u5XLibPOx6D7D8FBT/ax6wT57vNKY0QckCwo=
 google.golang.org/genproto v0.0.0-20190611190212-a7e196e89fd3/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
+google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -253,13 +287,16 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/square/go-jose.v2 v2.3.1 h1:SK5KegNXmKmqE342YYN2qPHEnUYeoMiXXl1poUlI+o4=
 gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099 h1:XJP7lxbSxWLOMNdBE4B/STaqVy6L73o0knwj2vIlxnw=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

internal/config/helpers.go

@@ -1,16 +1,5 @@
 package config // import "github.com/pomerium/pomerium/internal/config"
 
-import "os"
-
-// findPwd returns best guess at current working directory
-func findPwd() string {
-	p, err := os.Getwd()
-	if err != nil {
-		return "."
-	}
-	return p
-}
-
 // IsValidService checks to see if a service is a valid service mode
 func IsValidService(s string) bool {
 	switch s {

internal/config/options.go

@@ -7,11 +7,14 @@ import (
 	"net/url"
 	"path/filepath"
 	"reflect"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/pomerium/pomerium/internal/cryptutil"
+	"github.com/pomerium/pomerium/internal/fileutil"
 	"github.com/pomerium/pomerium/internal/log"
+	"github.com/pomerium/pomerium/internal/telemetry/metrics"
 	"github.com/pomerium/pomerium/internal/urlutil"
 
 	"github.com/mitchellh/hashstructure"
@@ -129,6 +132,19 @@ type Options struct {
 
 	// Address/Port to bind to for prometheus metrics
 	MetricsAddr string `mapstructure:"metrics_address"`
+
+	// Tracing shared settings
+	TracingProvider string `mapstructure:"tracing_provider"`
+	TracingDebug    bool   `mapstructure:"tracing_debug"`
+
+	//  Jaeger
+
+	// CollectorEndpoint is the full url to the Jaeger HTTP Thrift collector.
+	// For example, http://localhost:14268/api/traces
+	TracingJaegerCollectorEndpoint string `mapstructure:"tracing_jaeger_collector_endpoint"`
+	// AgentEndpoint instructs exporter to send spans to jaeger-agent at this address.
+	// For example, localhost:6831.
+	TracingJaegerAgentEndpoint string `mapstructure:"tracing_jaeger_agent_endpoint"`
 }
 
 var defaultOptions = Options{
@@ -148,8 +164,8 @@ var defaultOptions = Options{
 		"Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload",
 	},
 	Addr:              ":https",
-	CertFile:          filepath.Join(findPwd(), "cert.pem"),
-	KeyFile:           filepath.Join(findPwd(), "privkey.pem"),
+	CertFile:          filepath.Join(fileutil.Getwd(), "cert.pem"),
+	KeyFile:           filepath.Join(fileutil.Getwd(), "privkey.pem"),
 	ReadHeaderTimeout: 10 * time.Second,
 	ReadTimeout:       30 * time.Second,
 	WriteTimeout:      0, // support streaming by default
@@ -339,3 +355,56 @@ func (o *Options) Checksum() string {
 	}
 	return fmt.Sprintf("%x", hash)
 }
+
+func ParseOptions(configFile string) (*Options, error) {
+	o, err := OptionsFromViper(configFile)
+	if err != nil {
+		return nil, err
+	}
+	if o.Debug {
+		log.SetDebugMode()
+	}
+	if o.LogLevel != "" {
+		log.SetLevel(o.LogLevel)
+	}
+	metrics.AddPolicyCountCallback(o.Services, func() int64 {
+		return int64(len(o.Policies))
+	})
+
+	checksumDec, err := strconv.ParseUint(o.Checksum(), 16, 64)
+	if err != nil {
+		log.Warn().Err(err).Msg("Could not parse config checksum into decimal")
+	}
+	metrics.SetConfigChecksum(o.Services, checksumDec)
+
+	return o, nil
+}
+
+func HandleConfigUpdate(configFile string, opt *Options, services []OptionsUpdater) *Options {
+	newOpt, err := ParseOptions(configFile)
+	if err != nil {
+		log.Error().Err(err).Msg("cmd/pomerium: could not reload configuration")
+		return opt
+	}
+	optChecksum := opt.Checksum()
+	newOptChecksum := newOpt.Checksum()
+
+	log.Debug().
+		Str("old-checksum", optChecksum).
+		Str("new-checksum", newOptChecksum).
+		Msg("cmd/pomerium: configuration file changed")
+
+	if newOptChecksum == optChecksum {
+		log.Debug().Msg("cmd/pomerium: loaded configuration has not changed")
+		return opt
+	}
+
+	log.Info().Str("checksum", newOptChecksum).Msg("cmd/pomerium: checksum changed")
+	for _, service := range services {
+		if err := service.UpdateOptions(*newOpt); err != nil {
+			log.Error().Err(err).Msg("cmd/pomerium: could not update options")
+		}
+	}
+
+	return newOpt
+}

internal/config/options_test.go

@@ -408,3 +408,99 @@ func TestOptionsFromViper(t *testing.T) {
 		})
 	}
 }
+
+func Test_parseOptions(t *testing.T) {
+	viper.Reset()
+
+	tests := []struct {
+		name             string
+		envKey           string
+		envValue         string
+		servicesEnvKey   string
+		servicesEnvValue string
+		wantSharedKey    string
+		wantErr          bool
+	}{
+		{"no shared secret", "", "", "SERVICES", "authenticate", "skip", true},
+		{"no shared secret in all mode", "", "", "", "", "", false},
+		{"good", "SHARED_SECRET", "YixWi1MYh77NMECGGIJQevoonYtVF+ZPRkQZrrmeRqM=", "", "", "YixWi1MYh77NMECGGIJQevoonYtVF+ZPRkQZrrmeRqM=", false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			os.Setenv(tt.servicesEnvKey, tt.servicesEnvValue)
+			os.Setenv(tt.envKey, tt.envValue)
+			defer os.Unsetenv(tt.envKey)
+			defer os.Unsetenv(tt.servicesEnvKey)
+
+			got, err := ParseOptions("")
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ParseOptions() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != nil && got.Services != "all" && got.SharedKey != tt.wantSharedKey {
+				t.Errorf("ParseOptions()\n")
+				t.Errorf("got: %+v\n", got.SharedKey)
+				t.Errorf("want: %+v\n", tt.wantSharedKey)
+
+			}
+		})
+	}
+}
+
+type mockService struct {
+	fail    bool
+	Updated bool
+}
+
+func (m *mockService) UpdateOptions(o Options) error {
+
+	m.Updated = true
+	if m.fail {
+		return fmt.Errorf("failed")
+	}
+	return nil
+}
+
+func Test_HandleConfigUpdate(t *testing.T) {
+	os.Clearenv()
+	os.Setenv("SHARED_SECRET", "foo")
+	defer os.Unsetenv("SHARED_SECRET")
+
+	blankOpts, err := NewOptions("https://authenticate.example", "https://authorize.example")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	goodOpts, err := OptionsFromViper("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	tests := []struct {
+		name       string
+		envarKey   string
+		envarValue string
+		service    *mockService
+		oldOpts    Options
+		wantUpdate bool
+	}{
+		{"good", "", "", &mockService{fail: false}, *blankOpts, true},
+		{"good set debug", "POMERIUM_DEBUG", "true", &mockService{fail: false}, *blankOpts, true},
+		{"bad", "", "", &mockService{fail: true}, *blankOpts, true},
+		{"no change", "", "", &mockService{fail: false}, *goodOpts, false},
+		{"bad policy file unmarshal error", "POLICY", base64.StdEncoding.EncodeToString([]byte("{json:}")), &mockService{fail: false}, *blankOpts, false},
+		{"bad header key", "SERVICES", "error", &mockService{fail: false}, *blankOpts, false},
+		{"bad header header value", "HEADERS", "x;y;z", &mockService{fail: false}, *blankOpts, false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			os.Setenv(tt.envarKey, tt.envarValue)
+			defer os.Unsetenv(tt.envarKey)
+
+			HandleConfigUpdate("", &tt.oldOpts, []OptionsUpdater{tt.service})
+			if tt.service.Updated != tt.wantUpdate {
+				t.Errorf("Failed to update config on service")
+			}
+		})
+	}
+}

internal/fileutil/fileutil.go

@@ -30,3 +30,17 @@ func IsReadableFile(path string) (bool, error) {
 	fd.Close()
 	return true, nil // Item exists and is readable.
 }
+
+// Getwd returns a rooted path name corresponding to the
+// current directory. If the current directory can be
+// reached via multiple paths (due to symbolic links),
+// Getwd may return any one of them.
+//
+// On failure, will return "."
+func Getwd() string {
+	p, err := os.Getwd()
+	if err != nil {
+		return "."
+	}
+	return p
+}

internal/fileutil/fileutil_test.go

@@ -1,6 +1,9 @@
-package fileutil // import "github.com/pomerium/pomerium/internal/fileutil"
+package fileutil
 
-import "testing"
+import (
+	"strings"
+	"testing"
+)
 
 func TestIsReadableFile(t *testing.T) {
 
@@ -27,3 +30,19 @@ func TestIsReadableFile(t *testing.T) {
 		})
 	}
 }
+
+func TestGetwd(t *testing.T) {
+	tests := []struct {
+		name string
+		want string
+	}{
+		{"most basic example", "internal/fileutil"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := Getwd(); strings.Contains(tt.want, got) {
+				t.Errorf("Getwd() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

internal/httputil/errors.go

@@ -23,21 +23,11 @@ func (h Error) Error() string {
 	return fmt.Sprintf("%d %s: %s", h.Code, http.StatusText(h.Code), h.Message)
 }
 
-// CodeForError maps an error type and returns a corresponding http.Status
-func CodeForError(err error) int {
-	switch err {
-	case ErrTokenRevoked:
-		return http.StatusUnauthorized
-	}
-	return http.StatusInternalServerError
-}
-
 // ErrorResponse renders an error page for errors given a message and a status code.
 // If no message is passed, defaults to the text of the status code.
 func ErrorResponse(rw http.ResponseWriter, r *http.Request, e *Error) {
-	requestID := ""
-	id, ok := log.IDFromRequest(r)
-	if ok {
+	var requestID string
+	if id, ok := log.IDFromRequest(r); ok {
 		requestID = id
 	}
 	if r.Header.Get("Accept") == "application/json" {

internal/httputil/errors_test.go

@@ -0,0 +1,49 @@
+package httputil
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestErrorResponse(t *testing.T) {
+	tests := []struct {
+		name string
+		rw   http.ResponseWriter
+		r    *http.Request
+		e    *Error
+	}{
+		{"good", httptest.NewRecorder(), &http.Request{Method: http.MethodGet}, &Error{Code: http.StatusBadRequest, Message: "missing id token"}},
+		{"good json", httptest.NewRecorder(), &http.Request{Method: http.MethodGet, Header: http.Header{"Accept": []string{"application/json"}}}, &Error{Code: http.StatusBadRequest, Message: "missing id token"}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ErrorResponse(tt.rw, tt.r, tt.e)
+		})
+	}
+}
+
+func TestError_Error(t *testing.T) {
+
+	tests := []struct {
+		name     string
+		Message  string
+		Code     int
+		CanDebug bool
+		want     string
+	}{
+		{"good", "short and stout", http.StatusTeapot, false, "418 I'm a teapot: short and stout"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			h := Error{
+				Message:  tt.Message,
+				Code:     tt.Code,
+				CanDebug: tt.CanDebug,
+			}
+			if got := h.Error(); got != tt.want {
+				t.Errorf("Error.Error() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

internal/httputil/http.go

@@ -0,0 +1,76 @@
+package httputil // import "github.com/pomerium/pomerium/internal/httputil"
+
+import (
+	"context"
+	"fmt"
+	stdlog "log"
+	"net/http"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/pomerium/pomerium/internal/log"
+	"github.com/pomerium/pomerium/internal/urlutil"
+)
+
+// NewHTTPServer starts a http server given a set of options and a handler.
+//
+// It is the caller's responsibility to Close() or Shutdown() the returned
+// server.
+func NewHTTPServer(opt *ServerOptions, h http.Handler) *http.Server {
+	if opt == nil {
+		opt = defaultHTTPServerOptions
+	} else {
+		opt.applyHTTPDefaults()
+	}
+	sublogger := log.With().Str("addr", opt.Addr).Logger()
+	srv := http.Server{
+		Addr:              opt.Addr,
+		ReadHeaderTimeout: opt.ReadHeaderTimeout,
+		ReadTimeout:       opt.ReadTimeout,
+		WriteTimeout:      opt.WriteTimeout,
+		IdleTimeout:       opt.IdleTimeout,
+		Handler:           h,
+		ErrorLog:          stdlog.New(&log.StdLogWrapper{Logger: &sublogger}, "", 0),
+	}
+
+	go func() {
+		if err := srv.ListenAndServe(); err != http.ErrServerClosed {
+			log.Error().Str("addr", opt.Addr).Err(err).Msg("internal/httputil: unexpected shutdown")
+		}
+	}()
+	return &srv
+}
+
+func RedirectHandler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Connection", "close")
+		url := fmt.Sprintf("https://%s%s", urlutil.StripPort(r.Host), r.URL.String())
+		http.Redirect(w, r, url, http.StatusMovedPermanently)
+	})
+}
+
+// Shutdown attempts to shut down the server when a os interrupt or sigterm
+// signal are received without interrupting any
+// active connections. Shutdown works by first closing all open
+// listeners, then closing all idle connections, and then waiting
+// indefinitely for connections to return to idle and then shut down.
+// If the provided context expires before the shutdown is complete,
+// Shutdown returns the context's error, otherwise it returns any
+// error returned from closing the Server's underlying Listener(s).
+//
+// When Shutdown is called, Serve, ListenAndServe, and
+// ListenAndServeTLS immediately return ErrServerClosed.
+func Shutdown(srv *http.Server) {
+	sigint := make(chan os.Signal, 1)
+	signal.Notify(sigint, os.Interrupt)
+	signal.Notify(sigint, syscall.SIGTERM)
+	rec := <-sigint
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	log.Info().Str("signal", rec.String()).Msg("internal/httputil: shutting down servers")
+	if err := srv.Shutdown(ctx); err != nil {
+		log.Error().Err(err).Msg("internal/httputil: shutdown failed")
+	}
+}

internal/httputil/http_test.go

@@ -0,0 +1,49 @@
+package httputil
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestNewHTTPServer(t *testing.T) {
+	tests := []struct {
+		name string
+		opts *ServerOptions
+		// wantErr bool
+	}{
+		{"localhost:9232", &ServerOptions{Addr: "localhost:9232"}},
+		{"localhost:65536", &ServerOptions{Addr: "localhost:-1"}}, // will fail, but won't err
+		{"empty", &ServerOptions{}},
+		{"empty", nil},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			srv := NewHTTPServer(tt.opts, RedirectHandler())
+
+			defer srv.Close()
+
+			// we cheat a little bit here and use the httptest server to test the client
+			ts := httptest.NewServer(srv.Handler)
+			defer ts.Close()
+			client := ts.Client()
+			client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+				return http.ErrUseLastResponse
+			}
+			res, err := client.Get(ts.URL)
+			if err != nil {
+				log.Fatal(err)
+			}
+			greeting, err := ioutil.ReadAll(res.Body)
+			res.Body.Close()
+			if err != nil {
+				log.Fatal(err)
+			}
+			fmt.Printf("%s", greeting)
+
+		})
+	}
+}

internal/httputil/options.go

@@ -0,0 +1,87 @@
+package httputil // import "github.com/pomerium/pomerium/internal/httputil"
+
+import (
+	"path/filepath"
+	"time"
+
+	"github.com/pomerium/pomerium/internal/fileutil"
+)
+
+// ServerOptions contains the configurations settings for a http server.
+type ServerOptions struct {
+	// Addr specifies the host and port on which the server should serve
+	// HTTPS requests. If empty, ":https" is used.
+	Addr string
+
+	// TLS certificates to use.
+	Cert     string
+	Key      string
+	CertFile string
+	KeyFile  string
+
+	// Timeouts
+	ReadHeaderTimeout time.Duration
+	ReadTimeout       time.Duration
+	WriteTimeout      time.Duration
+	IdleTimeout       time.Duration
+}
+
+var defaultTLSServerOptions = &ServerOptions{
+	Addr:              ":https",
+	CertFile:          filepath.Join(fileutil.Getwd(), "cert.pem"),
+	KeyFile:           filepath.Join(fileutil.Getwd(), "privkey.pem"),
+	ReadHeaderTimeout: 10 * time.Second,
+	ReadTimeout:       30 * time.Second,
+	WriteTimeout:      0, // support streaming by default
+	IdleTimeout:       5 * time.Minute,
+}
+
+func (o *ServerOptions) applyTLSDefaults() {
+	if o.Addr == "" {
+		o.Addr = defaultTLSServerOptions.Addr
+	}
+	if o.Cert == "" && o.CertFile == "" {
+		o.CertFile = defaultTLSServerOptions.CertFile
+	}
+	if o.Key == "" && o.KeyFile == "" {
+		o.KeyFile = defaultTLSServerOptions.KeyFile
+	}
+	if o.ReadHeaderTimeout == 0 {
+		o.ReadHeaderTimeout = defaultTLSServerOptions.ReadHeaderTimeout
+	}
+	if o.ReadTimeout == 0 {
+		o.ReadTimeout = defaultTLSServerOptions.ReadTimeout
+	}
+	if o.WriteTimeout == 0 {
+		o.WriteTimeout = defaultTLSServerOptions.WriteTimeout
+	}
+	if o.IdleTimeout == 0 {
+		o.IdleTimeout = defaultTLSServerOptions.IdleTimeout
+	}
+}
+
+var defaultHTTPServerOptions = &ServerOptions{
+	Addr:              ":http",
+	ReadHeaderTimeout: 10 * time.Second,
+	ReadTimeout:       5 * time.Second,
+	WriteTimeout:      5 * time.Second,
+	IdleTimeout:       5 * time.Minute,
+}
+
+func (o *ServerOptions) applyHTTPDefaults() {
+	if o.Addr == "" {
+		o.Addr = defaultHTTPServerOptions.Addr
+	}
+	if o.ReadHeaderTimeout == 0 {
+		o.ReadHeaderTimeout = defaultHTTPServerOptions.ReadHeaderTimeout
+	}
+	if o.ReadTimeout == 0 {
+		o.ReadTimeout = defaultHTTPServerOptions.ReadTimeout
+	}
+	if o.WriteTimeout == 0 {
+		o.WriteTimeout = defaultHTTPServerOptions.WriteTimeout
+	}
+	if o.IdleTimeout == 0 {
+		o.IdleTimeout = defaultHTTPServerOptions.IdleTimeout
+	}
+}

internal/httputil/test_data/cert.pem

@@ -0,0 +1,10 @@
+-----BEGIN CERTIFICATE-----
+MIIBeDCCAR+gAwIBAgIUUGE8w2S7XzpkVLbNq5QUxyVOwqEwCgYIKoZIzj0EAwIw
+ETEPMA0GA1UEAwwGdW51c2VkMCAXDTE5MDcxNTIzNDQyOVoYDzQ3NTcwNjExMjM0
+NDI5WjARMQ8wDQYDVQQDDAZ1bnVzZWQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNC
+AAQW6Z1KsR712c8RRTcu7ILyXowzo9582ClKxEvgasPbZchMyOoMoWuOolN/QWjV
+labi/4R2zqzzyuwvMQL5wotFo1MwUTAdBgNVHQ4EFgQURYdcaniRqBHXeaM79LtV
+pyJ4EwAwHwYDVR0jBBgwFoAURYdcaniRqBHXeaM79LtVpyJ4EwAwDwYDVR0TAQH/
+BAUwAwEB/zAKBggqhkjOPQQDAgNHADBEAiBHbhVnGbwXqaMZ1dB8eBAK56jyeWDZ
+2PWXmFMTu7+RywIgaZ7UwVNB2k7KjEEBiLm0PIRcpJmczI2cP9+ZMIkPHHw=
+-----END CERTIFICATE-----

internal/httputil/test_data/privkey.pem

@@ -0,0 +1,5 @@
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIMQiDy26/R4ca/OdnjIf8OEDeHcw8yB5SDV9FD500CW5oAoGCCqGSM49
+AwEHoUQDQgAEFumdSrEe9dnPEUU3LuyC8l6MM6PefNgpSsRL4GrD22XITMjqDKFr
+jqJTf0Fo1ZWm4v+Eds6s88rsLzEC+cKLRQ==
+-----END EC PRIVATE KEY-----

internal/httputil/tls.go

@@ -7,83 +7,20 @@ import (
 	stdlog "log"
 	"net"
 	"net/http"
-	"os"
-	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/pomerium/pomerium/internal/fileutil"
 	"github.com/pomerium/pomerium/internal/log"
 )
 
-// Options contains the configurations settings for a TLS http server.
-type Options struct {
-	// Addr specifies the host and port on which the server should serve
-	// HTTPS requests. If empty, ":https" is used.
-	Addr string
-
-	// TLS certificates to use.
-	Cert     string
-	Key      string
-	CertFile string
-	KeyFile  string
-
-	// Timeouts
-	ReadHeaderTimeout time.Duration
-	ReadTimeout       time.Duration
-	WriteTimeout      time.Duration
-	IdleTimeout       time.Duration
-}
-
-var defaultOptions = &Options{
-	Addr:              ":https",
-	CertFile:          filepath.Join(findKeyDir(), "cert.pem"),
-	KeyFile:           filepath.Join(findKeyDir(), "privkey.pem"),
-	ReadHeaderTimeout: 10 * time.Second,
-	ReadTimeout:       30 * time.Second,
-	WriteTimeout:      0, // support streaming by default
-	IdleTimeout:       5 * time.Minute,
-}
-
-func findKeyDir() string {
-	p, err := os.Getwd()
-	if err != nil {
-		return "."
-	}
-	return p
-}
-
-func (o *Options) applyDefaults() {
-	if o.Addr == "" {
-		o.Addr = defaultOptions.Addr
-	}
-	if o.Cert == "" && o.CertFile == "" {
-		o.CertFile = defaultOptions.CertFile
-	}
-	if o.Key == "" && o.KeyFile == "" {
-		o.KeyFile = defaultOptions.KeyFile
-	}
-	if o.ReadHeaderTimeout == 0 {
-		o.ReadHeaderTimeout = defaultOptions.ReadHeaderTimeout
-	}
-	if o.ReadTimeout == 0 {
-		o.ReadTimeout = defaultOptions.ReadTimeout
-	}
-	if o.WriteTimeout == 0 {
-		o.WriteTimeout = defaultOptions.WriteTimeout
-	}
-	if o.IdleTimeout == 0 {
-		o.IdleTimeout = defaultOptions.IdleTimeout
-	}
-}
-
-// ListenAndServeTLS serves the provided handlers by HTTPS
-// using the provided options.
-func ListenAndServeTLS(opt *Options, httpHandler http.Handler, grpcHandler http.Handler) error {
+// NewTLSServer creates a new TLS server given a set of options, handlers, and
+// optionally a set of gRPC endpoints as well.
+// It is the callers responsibility to close the resturned server.
+func NewTLSServer(opt *ServerOptions, httpHandler http.Handler, grpcHandler http.Handler) (*http.Server, error) {
 	if opt == nil {
-		opt = defaultOptions
+		opt = defaultTLSServerOptions
 	} else {
-		opt.applyDefaults()
+		opt.applyTLSDefaults()
 	}
 	var cert *tls.Certificate
 	var err error
@@ -93,12 +30,12 @@ func ListenAndServeTLS(opt *Options, httpHandler http.Handler, grpcHandler http.
 		cert, err = readCertificateFile(opt.CertFile, opt.KeyFile)
 	}
 	if err != nil {
-		return fmt.Errorf("https: failed loading x509 certificate: %v", err)
+		return nil, fmt.Errorf("internal/httputil: failed loading x509 certificate: %v", err)
 	}
 	config := newDefaultTLSConfig(cert)
 	ln, err := net.Listen("tcp", opt.Addr)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	ln = tls.NewListener(ln, config)
@@ -112,7 +49,7 @@ func ListenAndServeTLS(opt *Options, httpHandler http.Handler, grpcHandler http.
 	sublogger := log.With().Str("addr", opt.Addr).Logger()
 
 	// Set up the main server.
-	server := &http.Server{
+	srv := &http.Server{
 		ReadHeaderTimeout: opt.ReadHeaderTimeout,
 		ReadTimeout:       opt.ReadTimeout,
 		WriteTimeout:      opt.WriteTimeout,
@@ -121,8 +58,13 @@ func ListenAndServeTLS(opt *Options, httpHandler http.Handler, grpcHandler http.
 		Handler:           h,
 		ErrorLog:          stdlog.New(&log.StdLogWrapper{Logger: &sublogger}, "", 0),
 	}
+	go func() {
+		if err := srv.Serve(ln); err != http.ErrServerClosed {
+			log.Error().Err(err).Msg("internal/httputil: tls server crashed")
+		}
+	}()
 
-	return server.Serve(ln)
+	return srv, nil
 }
 
 func decodeCertificate(cert, key string) (*tls.Certificate, error) {
@@ -189,8 +131,8 @@ func newDefaultTLSConfig(cert *tls.Certificate) *tls.Config {
 	return tlsConfig
 }
 
-// grpcHandlerFunc splits request serving between gRPC and HTTPS depending on the request type.
-// Requires HTTP/2.
+// grpcHandlerFunc splits request serving between gRPC and HTTPS depending on
+// the request type. Requires HTTP/2 to be enabled.
 func grpcHandlerFunc(rpcServer http.Handler, other http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ct := r.Header.Get("Content-Type")

internal/httputil/tls_test.go

@@ -0,0 +1,210 @@
+package httputil
+
+import (
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"os/signal"
+	"syscall"
+	"testing"
+	"time"
+)
+
+const privKey = `-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIMQiDy26/R4ca/OdnjIf8OEDeHcw8yB5SDV9FD500CW5oAoGCCqGSM49
+AwEHoUQDQgAEFumdSrEe9dnPEUU3LuyC8l6MM6PefNgpSsRL4GrD22XITMjqDKFr
+jqJTf0Fo1ZWm4v+Eds6s88rsLzEC+cKLRQ==
+-----END EC PRIVATE KEY-----`
+const pubKey = `-----BEGIN CERTIFICATE-----
+MIIBeDCCAR+gAwIBAgIUUGE8w2S7XzpkVLbNq5QUxyVOwqEwCgYIKoZIzj0EAwIw
+ETEPMA0GA1UEAwwGdW51c2VkMCAXDTE5MDcxNTIzNDQyOVoYDzQ3NTcwNjExMjM0
+NDI5WjARMQ8wDQYDVQQDDAZ1bnVzZWQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNC
+AAQW6Z1KsR712c8RRTcu7ILyXowzo9582ClKxEvgasPbZchMyOoMoWuOolN/QWjV
+labi/4R2zqzzyuwvMQL5wotFo1MwUTAdBgNVHQ4EFgQURYdcaniRqBHXeaM79LtV
+pyJ4EwAwHwYDVR0jBBgwFoAURYdcaniRqBHXeaM79LtVpyJ4EwAwDwYDVR0TAQH/
+BAUwAwEB/zAKBggqhkjOPQQDAgNHADBEAiBHbhVnGbwXqaMZ1dB8eBAK56jyeWDZ
+2PWXmFMTu7+RywIgaZ7UwVNB2k7KjEEBiLm0PIRcpJmczI2cP9+ZMIkPHHw=
+-----END CERTIFICATE-----`
+
+func TestNewTLSServer(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name        string
+		opt         *ServerOptions
+		httpHandler http.Handler
+		grpcHandler http.Handler
+		// want        *http.Server
+		wantErr bool
+	}{
+		{"good basic http handler",
+			&ServerOptions{
+				Addr: "127.0.0.1:0",
+				Cert: base64.StdEncoding.EncodeToString([]byte(pubKey)),
+				Key:  base64.StdEncoding.EncodeToString([]byte(privKey)),
+			},
+			http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				fmt.Fprintln(w, "Hello, http")
+			}),
+			nil,
+			false},
+		{"good basic http and grpc handler",
+			&ServerOptions{
+				Addr: "127.0.0.1:0",
+				Cert: base64.StdEncoding.EncodeToString([]byte(pubKey)),
+				Key:  base64.StdEncoding.EncodeToString([]byte(privKey)),
+			},
+			http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				fmt.Fprintln(w, "Hello, http")
+			}),
+			http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				fmt.Fprintln(w, "Hello, grpc")
+			}),
+			false},
+		{"good with cert files",
+			&ServerOptions{
+				Addr:     "127.0.0.1:0",
+				CertFile: "test_data/cert.pem",
+				KeyFile:  "test_data/privkey.pem",
+			},
+			http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				fmt.Fprintln(w, "Hello, http")
+			}),
+			http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				fmt.Fprintln(w, "Hello, grpc")
+			}),
+			false},
+		{"unreadable cert file",
+			&ServerOptions{
+				Addr:     "127.0.0.1:0",
+				CertFile: "test_data",
+				KeyFile:  "test_data/privkey.pem",
+			},
+			http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				fmt.Fprintln(w, "Hello, http")
+			}),
+			http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				fmt.Fprintln(w, "Hello, grpc")
+			}),
+			true},
+		{"unreadable key file",
+			&ServerOptions{
+				Addr:     "127.0.0.1:0",
+				CertFile: "./test_data/cert.pem",
+				KeyFile:  "./test_data",
+			},
+			http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				fmt.Fprintln(w, "Hello, http")
+			}),
+			http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				fmt.Fprintln(w, "Hello, grpc")
+			}),
+			true},
+		{"unreadable key file",
+			&ServerOptions{
+				Addr:     "127.0.0.1:0",
+				CertFile: "./test_data/cert.pem",
+				KeyFile:  "./test_data/file-does-not-exist",
+			},
+			http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				fmt.Fprintln(w, "Hello, http")
+			}),
+			http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				fmt.Fprintln(w, "Hello, grpc")
+			}),
+			true},
+		{"bad private key base64",
+			&ServerOptions{
+				Addr: "127.0.0.1:0",
+				Cert: base64.StdEncoding.EncodeToString([]byte(pubKey)),
+				Key:  "bad guy",
+			}, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				fmt.Fprintln(w, "Hello, http")
+			}),
+			nil,
+			true},
+		{"bad public key base64",
+			&ServerOptions{
+				Addr: "127.0.0.1:9999",
+				Key:  base64.StdEncoding.EncodeToString([]byte(pubKey)),
+				Cert: "bad guy",
+			}, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				fmt.Fprintln(w, "Hello, http")
+			}),
+			nil,
+			true},
+		{"bad port - invalid port range ",
+			&ServerOptions{
+				Addr: "127.0.0.1:65536",
+				Cert: base64.StdEncoding.EncodeToString([]byte(pubKey)),
+				Key:  base64.StdEncoding.EncodeToString([]byte(privKey)),
+			}, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				fmt.Fprintln(w, "Hello, http")
+			}),
+			nil,
+			true},
+		{"nil apply default but will fail",
+			nil,
+			http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				fmt.Fprintln(w, "Hello, http")
+			}),
+			nil,
+			true},
+		{"empty, apply defaults to missing",
+			&ServerOptions{},
+			http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				fmt.Fprintln(w, "Hello, http")
+			}),
+			nil,
+			true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			srv, err := NewTLSServer(tt.opt, tt.httpHandler, tt.grpcHandler)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("NewTLSServer() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if err == nil {
+				// we cheat a little bit here and use the httptest server to test the client
+				ts := httptest.NewTLSServer(srv.Handler)
+				defer ts.Close()
+				client := ts.Client()
+				res, err := client.Get(ts.URL)
+				if err != nil {
+					log.Fatal(err)
+				}
+				greeting, err := ioutil.ReadAll(res.Body)
+				res.Body.Close()
+				if err != nil {
+					log.Fatal(err)
+				}
+				fmt.Printf("%s", greeting)
+			}
+			if srv != nil {
+				// simulate a sigterm and cleanup the server
+				c := make(chan os.Signal, 1)
+				signal.Notify(c, syscall.SIGINT)
+				defer signal.Stop(c)
+				go Shutdown(srv)
+				syscall.Kill(syscall.Getpid(), syscall.SIGINT)
+				waitSig(t, c, syscall.SIGINT)
+			}
+
+		})
+	}
+}
+func waitSig(t *testing.T, c <-chan os.Signal, sig os.Signal) {
+	select {
+	case s := <-c:
+		if s != sig {
+			t.Fatalf("signal was %v, want %v", s, sig)
+		}
+	case <-time.After(1 * time.Second):
+		t.Fatalf("timeout waiting for %v", sig)
+	}
+}

internal/identity/providers.go

@@ -9,11 +9,12 @@ import (
 	"net/url"
 	"time"
 
-	oidc "github.com/pomerium/go-oidc"
-	"golang.org/x/oauth2"
-
 	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/internal/sessions"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
+
+	oidc "github.com/pomerium/go-oidc"
+	"golang.org/x/oauth2"
 )
 
 const (
@@ -117,6 +118,8 @@ func (p *Provider) GetSignInURL(state string) string {
 // Validate does NOT check if revoked.
 // https://openid.net/specs/openid-connect-core-1_0.html#IDTokenValidation
 func (p *Provider) Validate(ctx context.Context, idToken string) (bool, error) {
+	ctx, span := trace.StartSpan(ctx, "identity.provider.Validate")
+	defer span.End()
 	_, err := p.verifier.Verify(ctx, idToken)
 	if err != nil {
 		log.Error().Err(err).Msg("identity: failed to verify session state")

internal/metrics/exporter.go

@@ -1,30 +0,0 @@
-package metrics // import "github.com/pomerium/pomerium/internal/metrics"
-
-import (
-	"net/http"
-
-	ocProm "contrib.go.opencensus.io/exporter/prometheus"
-	prom "github.com/prometheus/client_golang/prometheus"
-	"go.opencensus.io/stats/view"
-)
-
-//NewPromHTTPListener creates a prometheus exporter on ListenAddr
-func NewPromHTTPListener(addr string) error {
-	return http.ListenAndServe(addr, newPromHTTPHandler())
-}
-
-// newPromHTTPHandler creates a new prometheus exporter handler for /metrics
-func newPromHTTPHandler() http.Handler {
-	// TODO this is a cheap way to get thorough go process
-	// stats.  It will not work with additional exporters.
-	// It should turn into an FR to the OC framework
-	reg := prom.DefaultRegisterer.(*prom.Registry)
-	pe, _ := ocProm.NewExporter(ocProm.Options{
-		Namespace: "pomerium",
-		Registry:  reg,
-	})
-	view.RegisterExporter(pe)
-	mux := http.NewServeMux()
-	mux.Handle("/metrics", pe)
-	return mux
-}

internal/metrics/middleware.go

@@ -1,151 +0,0 @@
-package metrics // import "github.com/pomerium/pomerium/internal/metrics"
-
-import (
-	"net/http"
-
-	"go.opencensus.io/plugin/ochttp"
-
-	"github.com/pomerium/pomerium/internal/log"
-
-	"github.com/pomerium/pomerium/internal/tripper"
-	"go.opencensus.io/stats/view"
-	"go.opencensus.io/tag"
-)
-
-var (
-	httpSizeDistribution = view.Distribution(
-		1, 256, 512, 1024, 2048, 8192, 16384, 32768, 65536, 131072, 262144, 524288,
-		1048576, 2097152, 4194304, 8388608,
-	)
-
-	httpLatencyDistrubtion = view.Distribution(
-		1, 2, 5, 7, 10, 25, 500, 750,
-		100, 250, 500, 750,
-		1000, 2500, 5000, 7500,
-		10000, 25000, 50000, 75000,
-		100000,
-	)
-
-	// httpClientRequestCount    = stats.Int64("http_client_requests_total", "Total HTTP Client Requests", "1")
-	// httpClientResponseSize    = stats.Int64("http_client_response_size_bytes", "HTTP Client Response Size in bytes", "bytes")
-	// httpClientRequestDuration = stats.Int64("http_client_request_duration_ms", "HTTP Client Request duration in ms", "ms")
-
-	// HTTPServerRequestCountView is an OpenCensus View that tracks HTTP server requests by pomerium service, host, method and status
-	HTTPServerRequestCountView = &view.View{
-		Name:        "http_server_requests_total",
-		Measure:     ochttp.ServerLatency,
-		Description: "Total HTTP Requests",
-		TagKeys:     []tag.Key{keyService, keyHost, keyHTTPMethod, ochttp.StatusCode},
-		Aggregation: view.Count(),
-	}
-
-	// HTTPServerRequestDurationView is an OpenCensus view that tracks HTTP server request duration by pomerium service, host, method and status
-	HTTPServerRequestDurationView = &view.View{
-		Name:        "http_server_request_duration_ms",
-		Measure:     ochttp.ServerLatency,
-		Description: "HTTP Request duration in ms",
-		TagKeys:     []tag.Key{keyService, keyHost, keyHTTPMethod, ochttp.StatusCode},
-		Aggregation: httpLatencyDistrubtion,
-	}
-
-	// HTTPServerRequestSizeView is an OpenCensus view that tracks HTTP server request size by pomerium service, host and method
-	HTTPServerRequestSizeView = &view.View{
-		Name:        "http_server_request_size_bytes",
-		Measure:     ochttp.ServerRequestBytes,
-		Description: "HTTP Server Request Size in bytes",
-		TagKeys:     []tag.Key{keyService, keyHost, keyHTTPMethod},
-		Aggregation: httpSizeDistribution,
-	}
-
-	// HTTPServerResponseSizeView is an OpenCensus view that tracks HTTP server response size by pomerium service, host, method and status
-	HTTPServerResponseSizeView = &view.View{
-		Name:        "http_server_response_size_bytes",
-		Measure:     ochttp.ServerResponseBytes,
-		Description: "HTTP Server Response Size in bytes",
-		TagKeys:     []tag.Key{keyService, keyHost, keyHTTPMethod, ochttp.StatusCode},
-		Aggregation: httpSizeDistribution,
-	}
-
-	// HTTPClientRequestCountView is an OpenCensus View that tracks HTTP client requests by pomerium service, destination, host, method and status
-	HTTPClientRequestCountView = &view.View{
-		Name:        "http_client_requests_total",
-		Measure:     ochttp.ClientRoundtripLatency,
-		Description: "Total HTTP Client Requests",
-		TagKeys:     []tag.Key{keyService, keyHost, keyHTTPMethod, ochttp.StatusCode, keyDestination},
-		Aggregation: view.Count(),
-	}
-
-	// HTTPClientRequestDurationView is an OpenCensus view that tracks HTTP client request duration by pomerium service, destination, host, method and status
-	HTTPClientRequestDurationView = &view.View{
-		Name:        "http_client_request_duration_ms",
-		Measure:     ochttp.ClientRoundtripLatency,
-		Description: "HTTP Client Request duration in ms",
-		TagKeys:     []tag.Key{keyService, keyHost, keyHTTPMethod, ochttp.StatusCode, keyDestination},
-		Aggregation: httpLatencyDistrubtion,
-	}
-
-	// HTTPClientResponseSizeView is an OpenCensus view that tracks HTTP client response size by pomerium service, destination, host, method and status
-	HTTPClientResponseSizeView = &view.View{
-		Name:        "http_client_response_size_bytes",
-		Measure:     ochttp.ClientReceivedBytes,
-		Description: "HTTP Client Response Size in bytes",
-		TagKeys:     []tag.Key{keyService, keyHost, keyHTTPMethod, ochttp.StatusCode, keyDestination},
-		Aggregation: httpSizeDistribution,
-	}
-
-	// HTTPClientRequestSizeView is an OpenCensus view that tracks HTTP client request size by pomerium service, destination, host and method
-	HTTPClientRequestSizeView = &view.View{
-		Name:        "http_client_response_size_bytes",
-		Measure:     ochttp.ClientSentBytes,
-		Description: "HTTP Client Response Size in bytes",
-		TagKeys:     []tag.Key{keyService, keyHost, keyHTTPMethod, keyDestination},
-		Aggregation: httpSizeDistribution,
-	}
-)
-
-// HTTPMetricsHandler creates a metrics middleware for incoming HTTP requests
-func HTTPMetricsHandler(service string) func(next http.Handler) http.Handler {
-
-	return func(next http.Handler) http.Handler {
-		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			ctx, tagErr := tag.New(
-				r.Context(),
-				tag.Insert(keyService, service),
-				tag.Insert(keyHost, r.Host),
-				tag.Insert(keyHTTPMethod, r.Method),
-			)
-			if tagErr != nil {
-				log.Warn().Err(tagErr).Str("context", "HTTPMetricsHandler").Msg("internal/metrics: Failed to create metrics context tag")
-				next.ServeHTTP(w, r)
-				return
-			}
-
-			ocHandler := ochttp.Handler{Handler: next}
-			ocHandler.ServeHTTP(w, r.WithContext(ctx))
-		})
-	}
-}
-
-// HTTPMetricsRoundTripper creates a metrics tracking tripper for outbound HTTP Requests
-func HTTPMetricsRoundTripper(service string, destination string) func(next http.RoundTripper) http.RoundTripper {
-	return func(next http.RoundTripper) http.RoundTripper {
-		return tripper.RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
-
-			ctx, tagErr := tag.New(
-				r.Context(),
-				tag.Insert(keyService, service),
-				tag.Insert(keyHost, r.Host),
-				tag.Insert(keyHTTPMethod, r.Method),
-				tag.Insert(keyDestination, destination),
-			)
-
-			if tagErr != nil {
-				log.Warn().Err(tagErr).Str("context", "HTTPMetricsRoundTripper").Msg("internal/metrics: Failed to create context tag")
-				return next.RoundTrip(r)
-			}
-
-			ocTransport := ochttp.Transport{Base: next}
-			return ocTransport.RoundTrip(r.WithContext(ctx))
-		})
-	}
-}

internal/metrics/tags.go

@@ -1,14 +0,0 @@
-package metrics
-
-import (
-	"go.opencensus.io/tag"
-)
-
-var (
-	keyHTTPMethod  tag.Key = tag.MustNewKey("http_method")
-	keyService     tag.Key = tag.MustNewKey("service")
-	keyGRPCService tag.Key = tag.MustNewKey("grpc_service")
-	keyGRPCMethod  tag.Key = tag.MustNewKey("grpc_method")
-	keyHost        tag.Key = tag.MustNewKey("host")
-	keyDestination tag.Key = tag.MustNewKey("destination")
-)

internal/metrics/view.go

@@ -1,32 +0,0 @@
-package metrics
-
-import (
-	"github.com/pomerium/pomerium/internal/log"
-	"go.opencensus.io/stats/view"
-)
-
-var (
-	// HTTPClientViews contains opencensus views for HTTP Client metrics
-	HTTPClientViews = []*view.View{HTTPClientRequestCountView, HTTPClientRequestDurationView, HTTPClientResponseSizeView}
-	// HTTPServerViews contains opencensus views for HTTP Server metrics
-	HTTPServerViews = []*view.View{HTTPServerRequestCountView, HTTPServerRequestDurationView, HTTPServerRequestSizeView, HTTPServerResponseSizeView}
-	// GRPCClientViews contains opencensus views for GRPC Client metrics
-	GRPCClientViews = []*view.View{GRPCClientRequestCountView, GRPCClientRequestDurationView, GRPCClientResponseSizeView, GRPCClientRequestSizeView}
-	// GRPCServerViews contains opencensus views for GRPC Server metrics
-	GRPCServerViews = []*view.View{GRPCServerRequestCountView, GRPCServerRequestDurationView, GRPCServerResponseSizeView, GRPCServerRequestSizeView}
-	// InfoViews contains opencensus views for Info metrics
-	InfoViews = []*view.View{ConfigLastReloadView, ConfigLastReloadSuccessView}
-)
-
-// RegisterView registers one of the defined metrics views.  It must be called for metrics to see metrics
-// in the configured exporters
-func RegisterView(v []*view.View) {
-	if err := view.Register(v...); err != nil {
-		log.Warn().Str("context", "RegisterView").Err(err).Msg("internal/metrics: Could not register view")
-	}
-}
-
-// UnRegisterView unregisters one of the defined metrics views.
-func UnRegisterView(v []*view.View) {
-	view.Unregister(v...)
-}

internal/metrics/view_test.go

@@ -1,25 +0,0 @@
-package metrics
-
-import (
-	"testing"
-
-	"go.opencensus.io/stats/view"
-)
-
-func Test_RegisterView(t *testing.T) {
-	RegisterView(HTTPClientViews)
-	for _, v := range HTTPClientViews {
-		if view.Find(v.Name) != v {
-			t.Errorf("Failed to find registered view %s", v.Name)
-		}
-	}
-}
-
-func Test_UnregisterView(t *testing.T) {
-	UnRegisterView(HTTPClientViews)
-	for _, v := range HTTPClientViews {
-		if view.Find(v.Name) == v {
-			t.Errorf("Found unregistered view %s", v.Name)
-		}
-	}
-}

internal/middleware/grpc.go

@@ -3,6 +3,8 @@ package middleware // import "github.com/pomerium/pomerium/internal/middleware"
 import (
 	"context"
 
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
+
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/metadata"
@@ -30,6 +32,9 @@ func (s SharedSecretCred) RequireTransportSecurity() bool { return false }
 // handler and returns an error. Otherwise, the interceptor invokes the unary
 // handler.
 func (s SharedSecretCred) ValidateRequest(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
+	ctx, span := trace.StartSpan(ctx, "middleware.grpc.ValidateRequest")
+	defer span.End()
+
 	md, ok := metadata.FromIncomingContext(ctx)
 	if !ok {
 		return nil, status.Errorf(codes.InvalidArgument, "missing metadata")

internal/middleware/middleware.go

@@ -12,6 +12,8 @@ import (
 
 	"github.com/pomerium/pomerium/internal/cryptutil"
 	"github.com/pomerium/pomerium/internal/httputil"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
+
 	"golang.org/x/net/publicsuffix"
 )
 
@@ -19,10 +21,12 @@ import (
 func SetHeaders(securityHeaders map[string]string) func(next http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			ctx, span := trace.StartSpan(r.Context(), "middleware.SetHeaders")
+			defer span.End()
 			for key, val := range securityHeaders {
 				w.Header().Set(key, val)
 			}
-			next.ServeHTTP(w, r)
+			next.ServeHTTP(w, r.WithContext(ctx))
 		})
 	}
 }
@@ -32,6 +36,9 @@ func SetHeaders(securityHeaders map[string]string) func(next http.Handler) http.
 func ValidateClientSecret(sharedSecret string) func(next http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			ctx, span := trace.StartSpan(r.Context(), "middleware.ValidateClientSecret")
+			defer span.End()
+
 			if err := r.ParseForm(); err != nil {
 				httpErr := &httputil.Error{Message: err.Error(), Code: http.StatusBadRequest}
 				httputil.ErrorResponse(w, r, httpErr)
@@ -47,7 +54,7 @@ func ValidateClientSecret(sharedSecret string) func(next http.Handler) http.Hand
 				httputil.ErrorResponse(w, r, &httputil.Error{Code: http.StatusInternalServerError})
 				return
 			}
-			next.ServeHTTP(w, r)
+			next.ServeHTTP(w, r.WithContext(ctx))
 		})
 	}
 }
@@ -57,6 +64,8 @@ func ValidateClientSecret(sharedSecret string) func(next http.Handler) http.Hand
 func ValidateRedirectURI(rootDomain *url.URL) func(next http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			ctx, span := trace.StartSpan(r.Context(), "middleware.ValidateRedirectURI")
+			defer span.End()
 			err := r.ParseForm()
 			if err != nil {
 				httpErr := &httputil.Error{
@@ -80,7 +89,7 @@ func ValidateRedirectURI(rootDomain *url.URL) func(next http.Handler) http.Handl
 				httputil.ErrorResponse(w, r, httpErr)
 				return
 			}
-			next.ServeHTTP(w, r)
+			next.ServeHTTP(w, r.WithContext(ctx))
 		})
 	}
 }
@@ -103,6 +112,9 @@ func SameDomain(u, j *url.URL) bool {
 func ValidateSignature(sharedSecret string) func(next http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			ctx, span := trace.StartSpan(r.Context(), "middleware.ValidateSignature")
+			defer span.End()
+
 			err := r.ParseForm()
 			if err != nil {
 				httpErr := &httputil.Error{Message: err.Error(), Code: http.StatusBadRequest}
@@ -120,7 +132,7 @@ func ValidateSignature(sharedSecret string) func(next http.Handler) http.Handler
 				return
 			}
 
-			next.ServeHTTP(w, r)
+			next.ServeHTTP(w, r.WithContext(ctx))
 		})
 	}
 }
@@ -129,11 +141,14 @@ func ValidateSignature(sharedSecret string) func(next http.Handler) http.Handler
 func ValidateHost(validHost func(host string) bool) func(next http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			ctx, span := trace.StartSpan(r.Context(), "middleware.ValidateHost")
+			defer span.End()
+
 			if !validHost(r.Host) {
 				httputil.ErrorResponse(w, r, &httputil.Error{Code: http.StatusNotFound})
 				return
 			}
-			next.ServeHTTP(w, r)
+			next.ServeHTTP(w, r.WithContext(ctx))
 		})
 	}
 }
@@ -145,13 +160,16 @@ func ValidateHost(validHost func(host string) bool) func(next http.Handler) http
 func Healthcheck(endpoint, msg string) func(http.Handler) http.Handler {
 	f := func(next http.Handler) http.Handler {
 		fn := func(w http.ResponseWriter, r *http.Request) {
+			ctx, span := trace.StartSpan(r.Context(), "middleware.Healthcheck")
+			defer span.End()
+
 			if r.Method == "GET" && strings.EqualFold(r.URL.Path, endpoint) {
 				w.Header().Set("Content-Type", "text/plain")
 				w.WriteHeader(http.StatusOK)
 				w.Write([]byte(msg))
 				return
 			}
-			next.ServeHTTP(w, r)
+			next.ServeHTTP(w, r.WithContext(ctx))
 		}
 		return http.HandlerFunc(fn)
 	}

internal/middleware/reverse_proxy.go

@@ -6,11 +6,14 @@ import (
 
 	"github.com/pomerium/pomerium/internal/cryptutil"
 	"github.com/pomerium/pomerium/internal/log"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 )
 
 func SignRequest(signer cryptutil.JWTSigner, id, email, groups, header string) func(next http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			ctx, span := trace.StartSpan(r.Context(), "middleware.SignRequest")
+			defer span.End()
 			jwt, err := signer.SignJWT(
 				r.Header.Get(id),
 				r.Header.Get(email),
@@ -20,7 +23,7 @@ func SignRequest(signer cryptutil.JWTSigner, id, email, groups, header string) f
 			} else {
 				r.Header.Set(header, jwt)
 			}
-			next.ServeHTTP(w, r)
+			next.ServeHTTP(w, r.WithContext(ctx))
 		})
 	}
 }
@@ -29,6 +32,9 @@ func SignRequest(signer cryptutil.JWTSigner, id, email, groups, header string) f
 func StripPomeriumCookie(cookieName string) func(next http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			ctx, span := trace.StartSpan(r.Context(), "middleware.SignRequest")
+			defer span.End()
+
 			headers := make([]string, len(r.Cookies()))
 			for _, cookie := range r.Cookies() {
 				if cookie.Name != cookieName {
@@ -36,7 +42,7 @@ func StripPomeriumCookie(cookieName string) func(next http.Handler) http.Handler
 				}
 			}
 			r.Header.Set("Cookie", strings.Join(headers, ";"))
-			next.ServeHTTP(w, r)
+			next.ServeHTTP(w, r.WithContext(ctx))
 		})
 	}
 }

internal/telemetry/metrics/const.go

@@ -0,0 +1,41 @@
+package metrics // import "github.com/pomerium/pomerium/internal/telemetry/metrics"
+
+import (
+	"go.opencensus.io/plugin/ocgrpc"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
+)
+
+// The following tags are applied to stats recorded by this package.
+var (
+	TagKeyHTTPMethod  tag.Key = tag.MustNewKey("http_method")
+	TagKeyService     tag.Key = tag.MustNewKey("service")
+	TagKeyGRPCService tag.Key = tag.MustNewKey("grpc_service")
+	TagKeyGRPCMethod  tag.Key = tag.MustNewKey("grpc_method")
+	TagKeyHost        tag.Key = tag.MustNewKey("host")
+	TagKeyDestination tag.Key = tag.MustNewKey("destination")
+)
+
+// Default distributions used by views in this package.
+var (
+	DefaulHTTPSizeDistribution = view.Distribution(
+		1, 256, 512, 1024, 2048, 8192, 16384, 32768, 65536, 131072, 262144,
+		524288, 1048576, 2097152, 4194304, 8388608)
+	DefaultHTTPLatencyDistrubtion = view.Distribution(
+		1, 2, 5, 7, 10, 25, 500, 750, 100, 250, 500, 750, 1000, 2500, 5000,
+		7500, 10000, 25000, 50000, 75000, 100000)
+	grpcSizeDistribution = view.Distribution(
+		1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024,
+		2048, 4096, 8192, 16384,
+	)
+	DefaultMillisecondsDistribution = ocgrpc.DefaultMillisecondsDistribution
+)
+
+// DefaultViews are a set of default views to view HTTP and GRPC metrics.
+var (
+	DefaultViews = [][]*view.View{
+		GRPCServerViews,
+		HTTPServerViews,
+		GRPCClientViews,
+		GRPCServerViews}
+)

internal/telemetry/metrics/grpc.go

@@ -1,4 +1,4 @@
-package metrics // import "github.com/pomerium/pomerium/internal/metrics"
+package metrics // import "github.com/pomerium/pomerium/internal/telemetry/metrics"
 
 import (
 	"context"
@@ -12,93 +12,98 @@ import (
 	grpcstats "google.golang.org/grpc/stats"
 )
 
+// GRPC Views
 var (
-	grpcSizeDistribution = view.Distribution(
-		1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024,
-		2048, 4096, 8192, 16384,
-	)
-	grcpLatencyDistribution = view.Distribution(
-		1, 2, 5, 7, 10, 25, 50, 75,
-		100, 250, 500, 750, 1000,
-	)
+	// GRPCClientViews contains opencensus views for GRPC Client metrics.
+	GRPCClientViews = []*view.View{
+		GRPCClientRequestCountView,
+		GRPCClientRequestDurationView,
+		GRPCClientResponseSizeView,
+		GRPCClientRequestSizeView}
+	// GRPCServerViews contains opencensus views for GRPC Server metrics.
+	GRPCServerViews = []*view.View{
+		GRPCServerRequestCountView,
+		GRPCServerRequestDurationView,
+		GRPCServerResponseSizeView,
+		GRPCServerRequestSizeView}
 
 	// GRPCServerRequestCountView is an OpenCensus view which counts GRPC Server
 	// requests by pomerium service, grpc service, grpc method, and status
 	GRPCServerRequestCountView = &view.View{
-		Name:        "grpc_server_requests_total",
+		Name:        "grpc/server/requests_total",
 		Measure:     ocgrpc.ServerLatency,
 		Description: "Total grpc Requests",
-		TagKeys:     []tag.Key{keyService, keyGRPCMethod, ocgrpc.KeyServerStatus, keyGRPCService},
+		TagKeys:     []tag.Key{TagKeyService, TagKeyGRPCMethod, ocgrpc.KeyServerStatus, TagKeyGRPCService},
 		Aggregation: view.Count(),
 	}
 
 	// GRPCServerRequestDurationView is an OpenCensus view which tracks GRPC Server
 	// request duration by pomerium service, grpc service, grpc method, and status
 	GRPCServerRequestDurationView = &view.View{
-		Name:        "grpc_server_request_duration_ms",
+		Name:        "grpc/server/request_duration_ms",
 		Measure:     ocgrpc.ServerLatency,
 		Description: "grpc Request duration in ms",
-		TagKeys:     []tag.Key{keyService, keyGRPCMethod, ocgrpc.KeyServerStatus, keyGRPCService},
-		Aggregation: grcpLatencyDistribution,
+		TagKeys:     []tag.Key{TagKeyService, TagKeyGRPCMethod, ocgrpc.KeyServerStatus, TagKeyGRPCService},
+		Aggregation: DefaultMillisecondsDistribution,
 	}
 
 	// GRPCServerResponseSizeView is an OpenCensus view which tracks GRPC Server
 	// response size by pomerium service, grpc service, grpc method, and status
 	GRPCServerResponseSizeView = &view.View{
-		Name:        "grpc_server_response_size_bytes",
+		Name:        "grpc/server/response_size_bytes",
 		Measure:     ocgrpc.ServerSentBytesPerRPC,
 		Description: "grpc Server Response Size in bytes",
-		TagKeys:     []tag.Key{keyService, keyGRPCMethod, ocgrpc.KeyServerStatus, keyGRPCService},
+		TagKeys:     []tag.Key{TagKeyService, TagKeyGRPCMethod, ocgrpc.KeyServerStatus, TagKeyGRPCService},
 		Aggregation: grpcSizeDistribution,
 	}
 
 	// GRPCServerRequestSizeView is an OpenCensus view which tracks GRPC Server
 	// request size by pomerium service, grpc service, grpc method, and status
 	GRPCServerRequestSizeView = &view.View{
-		Name:        "grpc_server_request_size_bytes",
+		Name:        "grpc/server/request_size_bytes",
 		Measure:     ocgrpc.ServerReceivedBytesPerRPC,
 		Description: "grpc Server Request Size in bytes",
-		TagKeys:     []tag.Key{keyService, keyGRPCMethod, ocgrpc.KeyServerStatus, keyGRPCService},
+		TagKeys:     []tag.Key{TagKeyService, TagKeyGRPCMethod, ocgrpc.KeyServerStatus, TagKeyGRPCService},
 		Aggregation: grpcSizeDistribution,
 	}
 
 	// GRPCClientRequestCountView is an OpenCensus view which tracks GRPC Client
 	// requests by pomerium service, target host, grpc service, grpc method, and status
 	GRPCClientRequestCountView = &view.View{
-		Name:        "grpc_client_requests_total",
+		Name:        "grpc/client/requests_total",
 		Measure:     ocgrpc.ClientRoundtripLatency,
 		Description: "Total grpc Client Requests",
-		TagKeys:     []tag.Key{keyService, keyHost, keyGRPCMethod, keyGRPCService, ocgrpc.KeyClientStatus},
+		TagKeys:     []tag.Key{TagKeyService, TagKeyHost, TagKeyGRPCMethod, TagKeyGRPCService, ocgrpc.KeyClientStatus},
 		Aggregation: view.Count(),
 	}
 
 	// GRPCClientRequestDurationView is an OpenCensus view which tracks GRPC Client
 	// request duration by pomerium service, target host, grpc service, grpc method, and status
 	GRPCClientRequestDurationView = &view.View{
-		Name:        "grpc_client_request_duration_ms",
+		Name:        "grpc/client/request_duration_ms",
 		Measure:     ocgrpc.ClientRoundtripLatency,
 		Description: "grpc Client Request duration in ms",
-		TagKeys:     []tag.Key{keyService, keyHost, keyGRPCMethod, keyGRPCService, ocgrpc.KeyClientStatus},
-		Aggregation: grcpLatencyDistribution,
+		TagKeys:     []tag.Key{TagKeyService, TagKeyHost, TagKeyGRPCMethod, TagKeyGRPCService, ocgrpc.KeyClientStatus},
+		Aggregation: DefaultMillisecondsDistribution,
 	}
 
 	// GRPCClientResponseSizeView  is an OpenCensus view which tracks GRPC Client
 	// response size by pomerium service, target host, grpc service, grpc method, and status
 	GRPCClientResponseSizeView = &view.View{
-		Name:        "grpc_client_response_size_bytes",
+		Name:        "grpc/client/response_size_bytes",
 		Measure:     ocgrpc.ClientReceivedBytesPerRPC,
 		Description: "grpc Client Response Size in bytes",
-		TagKeys:     []tag.Key{keyService, keyHost, keyGRPCMethod, keyGRPCService, ocgrpc.KeyClientStatus},
+		TagKeys:     []tag.Key{TagKeyService, TagKeyHost, TagKeyGRPCMethod, TagKeyGRPCService, ocgrpc.KeyClientStatus},
 		Aggregation: grpcSizeDistribution,
 	}
 
 	// GRPCClientRequestSizeView  is an OpenCensus view which tracks GRPC Client
 	// request size by pomerium service, target host, grpc service, grpc method, and status
 	GRPCClientRequestSizeView = &view.View{
-		Name:        "grpc_client_request_size_bytes",
+		Name:        "grpc/client/request_size_bytes",
 		Measure:     ocgrpc.ClientSentBytesPerRPC,
 		Description: "grpc Client Request Size in bytes",
-		TagKeys:     []tag.Key{keyService, keyHost, keyGRPCMethod, keyGRPCService, ocgrpc.KeyClientStatus},
+		TagKeys:     []tag.Key{TagKeyService, TagKeyHost, TagKeyGRPCMethod, TagKeyGRPCService, ocgrpc.KeyClientStatus},
 		Aggregation: grpcSizeDistribution,
 	}
 )
@@ -126,13 +131,13 @@ func GRPCClientInterceptor(service string) grpc.UnaryClientInterceptor {
 
 		taggedCtx, tagErr := tag.New(
 			ctx,
-			tag.Insert(keyService, service),
-			tag.Insert(keyHost, cc.Target()),
-			tag.Insert(keyGRPCMethod, rpcMethod),
-			tag.Insert(keyGRPCService, rpcService),
+			tag.Insert(TagKeyService, service),
+			tag.Insert(TagKeyHost, cc.Target()),
+			tag.Insert(TagKeyGRPCMethod, rpcMethod),
+			tag.Insert(TagKeyGRPCService, rpcService),
 		)
 		if tagErr != nil {
-			log.Warn().Err(tagErr).Str("context", "GRPCClientInterceptor").Msg("internal/metrics: Failed to create context")
+			log.Warn().Err(tagErr).Str("context", "GRPCClientInterceptor").Msg("internal/telemetry: Failed to create context")
 			return invoker(ctx, method, req, reply, cc, opts...)
 		}
 
@@ -165,12 +170,12 @@ func (h *GRPCServerStatsHandler) TagRPC(ctx context.Context, tagInfo *grpcstats.
 
 	taggedCtx, tagErr := tag.New(
 		handledCtx,
-		tag.Insert(keyService, h.service),
-		tag.Insert(keyGRPCMethod, rpcMethod),
-		tag.Insert(keyGRPCService, rpcService),
+		tag.Insert(TagKeyService, h.service),
+		tag.Insert(TagKeyGRPCMethod, rpcMethod),
+		tag.Insert(TagKeyGRPCService, rpcService),
 	)
 	if tagErr != nil {
-		log.Warn().Err(tagErr).Str("context", "GRPCServerStatsHandler").Msg("internal/metrics: Failed to create context")
+		log.Warn().Err(tagErr).Str("context", "GRPCServerStatsHandler").Msg("internal/telemetry: Failed to create context")
 		return handledCtx
 
 	}
@@ -180,6 +185,5 @@ func (h *GRPCServerStatsHandler) TagRPC(ctx context.Context, tagInfo *grpcstats.
 
 // NewGRPCServerStatsHandler creates a new GRPCServerStatsHandler for a pomerium service
 func NewGRPCServerStatsHandler(service string) grpcstats.Handler {
-
 	return &GRPCServerStatsHandler{service: service, Handler: &ocgrpc.ServerHandler{}}
 }

internal/telemetry/metrics/grpc_test.go

@@ -1,10 +1,11 @@
-package metrics
+package metrics // import "github.com/pomerium/pomerium/internal/telemetry/metrics"
 
 import (
 	"context"
 	"testing"
 
 	"go.opencensus.io/plugin/ocgrpc"
+	"go.opencensus.io/stats/view"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/stats"
 	"google.golang.org/grpc/status"
@@ -97,8 +98,8 @@ func Test_GRPCClientInterceptor(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 
-			UnRegisterView(GRPCClientViews)
-			RegisterView(GRPCClientViews)
+			view.Unregister(GRPCClientViews...)
+			view.Register(GRPCClientViews...)
 
 			invoker := testInvoker{
 				invokeResult: tt.errorCode,
@@ -167,8 +168,8 @@ func Test_GRPCServerStatsHandler(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 
-			UnRegisterView(GRPCServerViews)
-			RegisterView(GRPCServerViews)
+			view.Unregister(GRPCServerViews...)
+			view.Register(GRPCServerViews...)
 
 			statsHandler := NewGRPCServerStatsHandler("test_service")
 			mockServerRPCHandle(statsHandler, tt.method, tt.errorCode)

internal/telemetry/metrics/helpers_test.go

@@ -1,41 +1,12 @@
-package metrics
+package metrics // import "github.com/pomerium/pomerium/internal/telemetry/metrics"
 
 import (
-	"strings"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
 	"go.opencensus.io/metric/metricdata"
-	"go.opencensus.io/stats/view"
 )
 
-func testDataRetrieval(v *view.View, t *testing.T, want string) {
-	if v == nil {
-		t.Fatalf("%s: nil view passed", t.Name())
-	}
-	name := v.Name
-	data, err := view.RetrieveData(name)
-
-	if err != nil {
-		t.Fatalf("%s: failed to retrieve data line %s", name, err)
-	}
-
-	if want != "" && len(data) != 1 {
-		t.Fatalf("%s: received incorrect number of data rows: %d", name, len(data))
-	}
-	if want == "" && len(data) > 0 {
-		t.Fatalf("%s: received incorrect number of data rows: %d", name, len(data))
-	} else if want == "" {
-		return
-	}
-
-	dataString := data[0].String()
-
-	if want != "" && !strings.HasPrefix(dataString, want) {
-		t.Errorf("%s: Found unexpected data row: \nwant: %s\ngot: %s\n", name, want, dataString)
-	}
-}
-
 func testMetricRetrieval(metrics []*metricdata.Metric, t *testing.T, labels []metricdata.LabelValue, value interface{}, name string) {
 	switch value.(type) {
 	case int64:

internal/telemetry/metrics/http.go

@@ -0,0 +1,157 @@
+package metrics // import "github.com/pomerium/pomerium/internal/telemetry/metrics"
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/pomerium/pomerium/internal/log"
+	"github.com/pomerium/pomerium/internal/tripper"
+	"go.opencensus.io/plugin/ochttp"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
+)
+
+// HTTP Views
+var (
+	// HTTPClientViews contains opencensus views for HTTP Client metrics.
+	HTTPClientViews = []*view.View{
+		HTTPClientRequestCountView,
+		HTTPClientRequestDurationView,
+		HTTPClientResponseSizeView}
+	// HTTPServerViews contains opencensus views for HTTP Server metrics.
+	HTTPServerViews = []*view.View{
+		HTTPServerRequestCountView,
+		HTTPServerRequestDurationView,
+		HTTPServerRequestSizeView,
+		HTTPServerResponseSizeView}
+
+	// HTTPServerRequestCountView is an OpenCensus View that tracks HTTP server
+	// requests by pomerium service, host, method and status
+	HTTPServerRequestCountView = &view.View{
+		Name:        "http/server/requests_total",
+		Measure:     ochttp.ServerLatency,
+		Description: "Total HTTP Requests",
+		TagKeys:     []tag.Key{TagKeyService, TagKeyHost, TagKeyHTTPMethod, ochttp.StatusCode},
+		Aggregation: view.Count(),
+	}
+
+	// HTTPServerRequestDurationView is an OpenCensus view that tracks HTTP
+	// server request duration by pomerium service, host, method and status
+	HTTPServerRequestDurationView = &view.View{
+		Name:        "http/server/request_duration_ms",
+		Measure:     ochttp.ServerLatency,
+		Description: "HTTP Request duration in ms",
+		TagKeys:     []tag.Key{TagKeyService, TagKeyHost, TagKeyHTTPMethod, ochttp.StatusCode},
+		Aggregation: DefaultHTTPLatencyDistrubtion,
+	}
+
+	// HTTPServerRequestSizeView is an OpenCensus view that tracks HTTP server
+	// request size by pomerium service, host and method
+	HTTPServerRequestSizeView = &view.View{
+		Name:        "http/server/request_size_bytes",
+		Measure:     ochttp.ServerRequestBytes,
+		Description: "HTTP Server Request Size in bytes",
+		TagKeys:     []tag.Key{TagKeyService, TagKeyHost, TagKeyHTTPMethod},
+		Aggregation: DefaulHTTPSizeDistribution,
+	}
+
+	// HTTPServerResponseSizeView is an OpenCensus view that tracks HTTP server
+	// response size by pomerium service, host, method and status
+	HTTPServerResponseSizeView = &view.View{
+		Name:        "http/server/response_size_bytes",
+		Measure:     ochttp.ServerResponseBytes,
+		Description: "HTTP Server Response Size in bytes",
+		TagKeys:     []tag.Key{TagKeyService, TagKeyHost, TagKeyHTTPMethod, ochttp.StatusCode},
+		Aggregation: DefaulHTTPSizeDistribution,
+	}
+
+	// HTTPClientRequestCountView is an OpenCensus View that tracks HTTP client
+	// requests by pomerium service, destination, host, method and status
+	HTTPClientRequestCountView = &view.View{
+		Name:        "http/client/requests_total",
+		Measure:     ochttp.ClientRoundtripLatency,
+		Description: "Total HTTP Client Requests",
+		TagKeys:     []tag.Key{TagKeyService, TagKeyHost, TagKeyHTTPMethod, ochttp.StatusCode, TagKeyDestination},
+		Aggregation: view.Count(),
+	}
+
+	// HTTPClientRequestDurationView is an OpenCensus view that tracks HTTP
+	// client request duration by pomerium service, destination, host, method and status
+	HTTPClientRequestDurationView = &view.View{
+		Name:        "http/client/request_duration_ms",
+		Measure:     ochttp.ClientRoundtripLatency,
+		Description: "HTTP Client Request duration in ms",
+		TagKeys:     []tag.Key{TagKeyService, TagKeyHost, TagKeyHTTPMethod, ochttp.StatusCode, TagKeyDestination},
+		Aggregation: DefaultHTTPLatencyDistrubtion,
+	}
+
+	// HTTPClientResponseSizeView is an OpenCensus view that tracks HTTP client
+	// esponse size by pomerium service, destination, host, method and status
+	HTTPClientResponseSizeView = &view.View{
+		Name:        "http/client/response_size_bytes",
+		Measure:     ochttp.ClientReceivedBytes,
+		Description: "HTTP Client Response Size in bytes",
+		TagKeys:     []tag.Key{TagKeyService, TagKeyHost, TagKeyHTTPMethod, ochttp.StatusCode, TagKeyDestination},
+		Aggregation: DefaulHTTPSizeDistribution,
+	}
+
+	// HTTPClientRequestSizeView is an OpenCensus view that tracks HTTP client
+	//request size by pomerium service, destination, host and method
+	HTTPClientRequestSizeView = &view.View{
+		Name:        "http/client/response_size_bytes",
+		Measure:     ochttp.ClientSentBytes,
+		Description: "HTTP Client Response Size in bytes",
+		TagKeys:     []tag.Key{TagKeyService, TagKeyHost, TagKeyHTTPMethod, TagKeyDestination},
+		Aggregation: DefaulHTTPSizeDistribution,
+	}
+)
+
+// HTTPMetricsHandler creates a metrics middleware for incoming HTTP requests
+func HTTPMetricsHandler(service string) func(next http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			ctx, tagErr := tag.New(
+				r.Context(),
+				tag.Insert(TagKeyService, service),
+				tag.Insert(TagKeyHost, r.Host),
+				tag.Insert(TagKeyHTTPMethod, r.Method),
+			)
+			if tagErr != nil {
+				log.Warn().Err(tagErr).Str("context", "HTTPMetricsHandler").
+					Msg("telemetry/metrics: failed to create metrics tag")
+				next.ServeHTTP(w, r)
+				return
+			}
+
+			ocHandler := ochttp.Handler{
+				Handler: next,
+				FormatSpanName: func(r *http.Request) string {
+					return fmt.Sprintf("%s%s", r.Host, r.URL.Path)
+				},
+			}
+			ocHandler.ServeHTTP(w, r.WithContext(ctx))
+		})
+	}
+}
+
+// HTTPMetricsRoundTripper creates a metrics tracking tripper for outbound HTTP Requests
+func HTTPMetricsRoundTripper(service string, destination string) func(next http.RoundTripper) http.RoundTripper {
+	return func(next http.RoundTripper) http.RoundTripper {
+		return tripper.RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
+			ctx, tagErr := tag.New(
+				r.Context(),
+				tag.Insert(TagKeyService, service),
+				tag.Insert(TagKeyHost, r.Host),
+				tag.Insert(TagKeyHTTPMethod, r.Method),
+				tag.Insert(TagKeyDestination, destination),
+			)
+			if tagErr != nil {
+				log.Warn().Err(tagErr).Str("context", "HTTPMetricsRoundTripper").Msg("telemetry/metrics: failed to create metrics tag")
+				return next.RoundTrip(r)
+			}
+
+			ocTransport := ochttp.Transport{Base: next}
+			return ocTransport.RoundTrip(r.WithContext(ctx))
+		})
+	}
+}

internal/telemetry/metrics/http_test.go

@@ -1,4 +1,4 @@
-package metrics // import "github.com/pomerium/pomerium/internal/metrics"
+package metrics // import "github.com/pomerium/pomerium/internal/telemetry/metrics"
 
 import (
 	"bytes"
@@ -7,13 +7,40 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 
-	"github.com/pomerium/pomerium/internal/middleware"
 	"github.com/pomerium/pomerium/internal/tripper"
 	"go.opencensus.io/stats/view"
 )
 
+func testDataRetrieval(v *view.View, t *testing.T, want string) {
+	if v == nil {
+		t.Fatalf("%s: nil view passed", t.Name())
+	}
+	name := v.Name
+	data, err := view.RetrieveData(name)
+
+	if err != nil {
+		t.Fatalf("%s: failed to retrieve data line %s", name, err)
+	}
+
+	if want != "" && len(data) != 1 {
+		t.Fatalf("%s: received incorrect number of data rows: %d", name, len(data))
+	}
+	if want == "" && len(data) > 0 {
+		t.Fatalf("%s: received incorrect number of data rows: %d", name, len(data))
+	} else if want == "" {
+		return
+	}
+
+	dataString := data[0].String()
+
+	if want != "" && !strings.HasPrefix(dataString, want) {
+		t.Errorf("%s: Found unexpected data row: \nwant: %s\ngot: %s\n", name, want, dataString)
+	}
+}
+
 func newTestMux() http.Handler {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/good", func(w http.ResponseWriter, r *http.Request) {
@@ -25,10 +52,6 @@ func newTestMux() http.Handler {
 
 func Test_HTTPMetricsHandler(t *testing.T) {
 
-	chain := middleware.NewChain()
-	chain = chain.Append(HTTPMetricsHandler("test_service"))
-	chainHandler := chain.Then(newTestMux())
-
 	tests := []struct {
 		name                          string
 		url                           string
@@ -73,7 +96,9 @@ func Test_HTTPMetricsHandler(t *testing.T) {
 
 			req := httptest.NewRequest(tt.verb, tt.url, new(bytes.Buffer))
 			rec := httptest.NewRecorder()
-			chainHandler.ServeHTTP(rec, req)
+
+			h := HTTPMetricsHandler("test_service")(newTestMux())
+			h.ServeHTTP(rec, req)
 
 			testDataRetrieval(HTTPServerRequestSizeView, t, tt.wanthttpServerRequestSize)
 			testDataRetrieval(HTTPServerResponseSizeView, t, tt.wanthttpServerResponseSize)

internal/telemetry/metrics/info.go

@@ -1,4 +1,4 @@
-package metrics // import "github.com/pomerium/pomerium/internal/metrics"
+package metrics // import "github.com/pomerium/pomerium/internal/telemetry/metrics"
 
 import (
 	"context"
@@ -8,6 +8,7 @@ import (
 
 	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/internal/version"
+
 	"go.opencensus.io/metric"
 	"go.opencensus.io/metric/metricdata"
 	"go.opencensus.io/metric/metricproducer"
@@ -17,44 +18,53 @@ import (
 )
 
 var (
-	//buildInfo               = stats.Int64("build_info", "Build Metadata", "1")
-	configLastReload        = stats.Int64("config_last_reload_success_timestamp", "Timestamp of last successful config reload", "seconds")
-	configLastReloadSuccess = stats.Int64("config_last_reload_success", "Returns 1 if last reload was successful", "1")
-	registry                = newMetricRegistry()
+	// InfoViews contains opencensus views for informational metrics about
+	// pomerium itself.
+	InfoViews = []*view.View{ConfigLastReloadView, ConfigLastReloadSuccessView}
+
+	configLastReload = stats.Int64(
+		"config_last_reload_success_timestamp",
+		"Timestamp of last successful config reload",
+		"seconds")
+	configLastReloadSuccess = stats.Int64(
+		"config_last_reload_success",
+		"Returns 1 if last reload was successful",
+		"1")
+	registry = newMetricRegistry()
 
 	// ConfigLastReloadView contains the timestamp the configuration was last
-	// reloaded, labeled by service
+	// reloaded, labeled by service.
 	ConfigLastReloadView = &view.View{
 		Name:        configLastReload.Name(),
 		Description: configLastReload.Description(),
 		Measure:     configLastReload,
-		TagKeys:     []tag.Key{keyService},
+		TagKeys:     []tag.Key{TagKeyService},
 		Aggregation: view.LastValue(),
 	}
 
 	// ConfigLastReloadSuccessView contains the result of the last configuration
-	//  reload, labeled by service
+	// reload, labeled by service.
 	ConfigLastReloadSuccessView = &view.View{
 		Name:        configLastReloadSuccess.Name(),
 		Description: configLastReloadSuccess.Description(),
 		Measure:     configLastReloadSuccess,
-		TagKeys:     []tag.Key{keyService},
+		TagKeys:     []tag.Key{TagKeyService},
 		Aggregation: view.LastValue(),
 	}
 )
 
-// SetConfigInfo records the status, checksum and timestamp of a configuration reload.  You must register InfoViews or the related
-// config views before calling
+// SetConfigInfo records the status, checksum and timestamp of a configuration
+// reload. You must register InfoViews or the related config views before calling
 func SetConfigInfo(service string, success bool, checksum string) {
 
 	if success {
-		serviceTag := tag.Insert(keyService, service)
+		serviceTag := tag.Insert(TagKeyService, service)
 		if err := stats.RecordWithTags(
 			context.Background(),
 			[]tag.Mutator{serviceTag},
 			configLastReload.M(time.Now().Unix()),
 		); err != nil {
-			log.Error().Err(err).Msg("internal/metrics: failed to record config checksum timestamp")
+			log.Error().Err(err).Msg("internal/telemetry: failed to record config checksum timestamp")
 		}
 
 		if err := stats.RecordWithTags(
@@ -62,7 +72,7 @@ func SetConfigInfo(service string, success bool, checksum string) {
 			[]tag.Mutator{serviceTag},
 			configLastReloadSuccess.M(1),
 		); err != nil {
-			log.Error().Err(err).Msg("internal/metrics: failed to record config reload")
+			log.Error().Err(err).Msg("internal/telemetry: failed to record config reload")
 		}
 	} else {
 		stats.Record(context.Background(), configLastReloadSuccess.M(0))
@@ -96,7 +106,7 @@ func (r *metricRegistry) init() {
 				metric.WithLabelKeys("service", "version", "revision", "goversion"),
 			)
 			if err != nil {
-				log.Error().Err(err).Msg("internal/metrics: failed to register build info metric")
+				log.Error().Err(err).Msg("internal/telemetry: failed to register build info metric")
 			}
 
 			r.configChecksum, err = r.registry.AddFloat64Gauge("config_checksum_decimal",
@@ -104,7 +114,7 @@ func (r *metricRegistry) init() {
 				metric.WithLabelKeys("service"),
 			)
 			if err != nil {
-				log.Error().Err(err).Msg("internal/metrics: failed to register config checksum metric")
+				log.Error().Err(err).Msg("internal/telemetry: failed to register config checksum metric")
 			}
 
 			r.policyCount, err = r.registry.AddInt64DerivedGauge("policy_count_total",
@@ -112,7 +122,7 @@ func (r *metricRegistry) init() {
 				metric.WithLabelKeys("service"),
 			)
 			if err != nil {
-				log.Error().Err(err).Msg("internal/metrics: failed to register policy count metric")
+				log.Error().Err(err).Msg("internal/telemetry: failed to register policy count metric")
 			}
 		})
 }
@@ -130,7 +140,7 @@ func (r *metricRegistry) setBuildInfo(service string) {
 		metricdata.NewLabelValue((runtime.Version())),
 	)
 	if err != nil {
-		log.Error().Err(err).Msg("internal/metrics: failed to get build info metric")
+		log.Error().Err(err).Msg("internal/telemetry: failed to get build info metric")
 	}
 
 	// This sets our build_info metric to a constant 1 per
@@ -155,7 +165,7 @@ func (r *metricRegistry) setConfigChecksum(service string, checksum uint64) {
 	}
 	m, err := r.configChecksum.GetEntry(metricdata.NewLabelValue(service))
 	if err != nil {
-		log.Error().Err(err).Msg("internal/metrics: failed to get config checksum metric")
+		log.Error().Err(err).Msg("internal/telemetry: failed to get config checksum metric")
 	}
 	m.Set(float64(checksum))
 }
@@ -172,7 +182,7 @@ func (r *metricRegistry) addPolicyCountCallback(service string, f func() int64)
 	}
 	err := r.policyCount.UpsertEntry(f, metricdata.NewLabelValue(service))
 	if err != nil {
-		log.Error().Err(err).Msg("internal/metrics: failed to get policy count metric")
+		log.Error().Err(err).Msg("internal/telemetry: failed to get policy count metric")
 	}
 }
 

internal/telemetry/metrics/info_test.go

@@ -1,4 +1,4 @@
-package metrics // import "github.com/pomerium/pomerium/internal/metrics"
+package metrics // import "github.com/pomerium/pomerium/internal/telemetry/metrics"
 
 import (
 	"runtime"
@@ -8,6 +8,7 @@ import (
 
 	"go.opencensus.io/metric/metricdata"
 	"go.opencensus.io/metric/metricproducer"
+	"go.opencensus.io/stats/view"
 )
 
 func Test_SetConfigInfo(t *testing.T) {
@@ -24,9 +25,8 @@ func Test_SetConfigInfo(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			UnRegisterView(InfoViews)
-			RegisterView(InfoViews)
-
+			view.Unregister(InfoViews...)
+			view.Register(InfoViews...)
 			SetConfigInfo("test_service", tt.success, tt.checksum)
 
 			testDataRetrieval(ConfigLastReloadView, t, tt.wantLastReload)

internal/telemetry/metrics/providers.go

@@ -0,0 +1,39 @@
+package metrics // import "github.com/pomerium/pomerium/internal/telemetry/metrics"
+
+import (
+	"fmt"
+	"net/http"
+
+	ocprom "contrib.go.opencensus.io/exporter/prometheus"
+	prom "github.com/prometheus/client_golang/prometheus"
+	"go.opencensus.io/stats/view"
+)
+
+// PrometheusHandler creates an exporter that exports stats to Prometheus
+// and returns a handler suitable for exporting metrics.
+func PrometheusHandler() (http.Handler, error) {
+	if err := registerDefaultViews(); err != nil {
+		return nil, fmt.Errorf("internal/telemetry: failed registering views")
+	}
+	reg := prom.DefaultRegisterer.(*prom.Registry)
+	exporter, err := ocprom.NewExporter(
+		ocprom.Options{
+			Namespace: "pomerium",
+			Registry:  reg,
+		})
+	if err != nil {
+		return nil, fmt.Errorf("internal/telemetry: prometheus exporter: %v", err)
+	}
+	view.RegisterExporter(exporter)
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", exporter)
+	return mux, nil
+}
+
+func registerDefaultViews() error {
+	var views []*view.View
+	for _, v := range DefaultViews {
+		views = append(views, v...)
+	}
+	return view.Register(views...)
+}

internal/telemetry/metrics/providers_test.go

@@ -1,4 +1,4 @@
-package metrics // import "github.com/pomerium/pomerium/internal/metrics"
+package metrics // import "github.com/pomerium/pomerium/internal/telemetry/metrics"
 
 import (
 	"bytes"
@@ -8,9 +8,11 @@ import (
 	"testing"
 )
 
-func Test_newPromHTTPHandler(t *testing.T) {
-	h := newPromHTTPHandler()
-
+func Test_PrometheusHandler(t *testing.T) {
+	h, err := PrometheusHandler()
+	if err != nil {
+		t.Fatal(err)
+	}
 	req := httptest.NewRequest("GET", "http://test.local/metrics", new(bytes.Buffer))
 	rec := httptest.NewRecorder()
 	h.ServeHTTP(rec, req)

internal/telemetry/trace/trace.go

@@ -0,0 +1,74 @@
+package trace // import "github.com/pomerium/pomerium/internal/telemetry/trace"
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pomerium/pomerium/internal/log"
+
+	"contrib.go.opencensus.io/exporter/jaeger"
+	"go.opencensus.io/trace"
+)
+
+const (
+	JaegerTracingProviderName = "jaeger"
+)
+
+// TracingOptions contains the configurations settings for a http server.
+type TracingOptions struct {
+	// Shared
+	Provider string
+	Service  string
+	Debug    bool
+
+	// Jaeger
+
+	// CollectorEndpoint is the full url to the Jaeger HTTP Thrift collector.
+	// For example, http://localhost:14268/api/traces
+	JaegerCollectorEndpoint string `mapstructure:"tracing_jaeger_collector_endpoint"`
+	// AgentEndpoint instructs exporter to send spans to jaeger-agent at this address.
+	// For example, localhost:6831.
+	JaegerAgentEndpoint string `mapstructure:"tracing_jaeger_agent_endpoint"`
+}
+
+func RegisterTracing(opts *TracingOptions) error {
+	var err error
+	switch opts.Provider {
+	case JaegerTracingProviderName:
+		err = registerJaeger(opts)
+	default:
+		return fmt.Errorf("telemetry/trace: provider %s unknown", opts.Provider)
+	}
+	if err != nil {
+		return err
+	}
+	if opts.Debug {
+		log.Debug().Msg("telemetry/trace: debug on, sample everything")
+		trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
+	}
+	log.Debug().Interface("Opts", opts).Msg("telemetry/trace: exporter created")
+	return nil
+}
+
+func registerJaeger(opts *TracingOptions) error {
+	jex, err := jaeger.NewExporter(
+		jaeger.Options{
+			AgentEndpoint:     opts.JaegerAgentEndpoint,
+			CollectorEndpoint: opts.JaegerCollectorEndpoint,
+			ServiceName:       opts.Service,
+		})
+	if err != nil {
+		return err
+	}
+	trace.RegisterExporter(jex)
+	return nil
+}
+
+// StartSpan starts a new child span of the current span in the context. If
+// there is no span in the context, creates a new trace and span.
+//
+// Returned context contains the newly created span. You can use it to
+// propagate the returned span in process.
+func StartSpan(ctx context.Context, name string, o ...trace.StartOption) (context.Context, *trace.Span) {
+	return trace.StartSpan(ctx, name, o...)
+}

internal/telemetry/trace/trace_test.go

@@ -0,0 +1,23 @@
+package trace // import "github.com/pomerium/pomerium/internal/telemetry/trace"
+
+import "testing"
+
+func TestRegisterTracing(t *testing.T) {
+	tests := []struct {
+		name    string
+		opts    *TracingOptions
+		wantErr bool
+	}{
+		{"jaeger", &TracingOptions{JaegerAgentEndpoint: "localhost:6831", Service: "all", Provider: "jaeger"}, false},
+		{"jaeger with debug", &TracingOptions{JaegerAgentEndpoint: "localhost:6831", Service: "all", Provider: "jaeger", Debug: true}, false},
+		{"jaeger no endpoint", &TracingOptions{JaegerAgentEndpoint: "", Service: "all", Provider: "jaeger"}, true},
+		{"unknown provider", &TracingOptions{JaegerAgentEndpoint: "localhost:0", Service: "all", Provider: "Lucius Cornelius Sulla"}, true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := RegisterTracing(tt.opts); (err != nil) != tt.wantErr {
+				t.Errorf("RegisterTracing() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

proxy/clients/authenticate_client.go

@@ -3,12 +3,12 @@ package clients // import "github.com/pomerium/pomerium/proxy/clients"
 import (
 	"context"
 	"errors"
-	"time"
-
-	"google.golang.org/grpc"
 
 	"github.com/pomerium/pomerium/internal/sessions"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	pb "github.com/pomerium/pomerium/proto/authenticate"
+
+	"google.golang.org/grpc"
 )
 
 // Authenticator provides the authenticate service interface
@@ -48,11 +48,12 @@ type AuthenticateGRPC struct {
 // Redeem makes an RPC call to the authenticate service to creates a session state
 // from an encrypted code provided as a result of an oauth2 callback process.
 func (a *AuthenticateGRPC) Redeem(ctx context.Context, code string) (*sessions.SessionState, error) {
+	ctx, span := trace.StartSpan(ctx, "proxy.client.grpc.Redeem")
+	defer span.End()
+
 	if code == "" {
 		return nil, errors.New("missing code")
 	}
-	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
-	defer cancel()
 	protoSession, err := a.client.Authenticate(ctx, &pb.AuthenticateRequest{Code: code})
 	if err != nil {
 		return nil, err
@@ -68,6 +69,9 @@ func (a *AuthenticateGRPC) Redeem(ctx context.Context, code string) (*sessions.S
 // user's session. Requires a valid refresh token. Will return an error if the identity provider
 // has revoked the session or if the refresh token is no longer valid in this context.
 func (a *AuthenticateGRPC) Refresh(ctx context.Context, s *sessions.SessionState) (*sessions.SessionState, error) {
+	ctx, span := trace.StartSpan(ctx, "proxy.client.grpc.Refresh")
+	defer span.End()
+
 	if s.RefreshToken == "" {
 		return nil, errors.New("missing refresh token")
 	}
@@ -75,14 +79,7 @@ func (a *AuthenticateGRPC) Refresh(ctx context.Context, s *sessions.SessionState
 	if err != nil {
 		return nil, err
 	}
-	// todo(bdd): handle request id in grpc receiver and add to ctx logger
-	// reqID, ok := middleware.IDFromCtx(ctx)
-	// if ok {
-	// 	md := metadata.Pairs("req_id", reqID)
-	// 	ctx = metadata.NewOutgoingContext(ctx, md)
-	// }
-	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
-	defer cancel()
+
 	// todo(bdd): add grpc specific timeouts to main options
 	// todo(bdd): handle request id (metadata!?) in grpc receiver and add to ctx logger
 	reply, err := a.client.Refresh(ctx, req)
@@ -100,18 +97,13 @@ func (a *AuthenticateGRPC) Refresh(ctx context.Context, s *sessions.SessionState
 // does NOT do nonce or revokation validation.
 // https://openid.net/specs/openid-connect-core-1_0.html#IDTokenValidation
 func (a *AuthenticateGRPC) Validate(ctx context.Context, idToken string) (bool, error) {
+	ctx, span := trace.StartSpan(ctx, "proxy.client.grpc.Validate")
+	defer span.End()
+
 	if idToken == "" {
 		return false, errors.New("missing id token")
 	}
-	// todo(bdd): add grpc specific timeouts to main options
-	// todo(bdd): handle request id in grpc receiver and add to ctx logger
-	// reqID, ok := middleware.IDFromCtx(ctx)
-	// if ok {
-	// 	md := metadata.Pairs("req_id", reqID)
-	// 	ctx = metadata.NewOutgoingContext(ctx, md)
-	// }
-	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
-	defer cancel()
+
 	r, err := a.client.Validate(ctx, &pb.ValidateRequest{IdToken: idToken})
 	if err != nil {
 		return false, err

proxy/clients/authorize_client.go

@@ -3,12 +3,12 @@ package clients // import "github.com/pomerium/pomerium/proxy/clients"
 import (
 	"context"
 	"errors"
-	"time"
-
-	"google.golang.org/grpc"
 
 	"github.com/pomerium/pomerium/internal/sessions"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	pb "github.com/pomerium/pomerium/proto/authorize"
+
+	"google.golang.org/grpc"
 )
 
 // Authorizer provides the authorize service interface
@@ -47,11 +47,12 @@ type AuthorizeGRPC struct {
 // Authorize takes a route and user session and returns whether the
 // request is valid per access policy
 func (a *AuthorizeGRPC) Authorize(ctx context.Context, route string, s *sessions.SessionState) (bool, error) {
+	ctx, span := trace.StartSpan(ctx, "proxy.client.grpc.Authorize")
+	defer span.End()
+
 	if s == nil {
 		return false, errors.New("session cannot be nil")
 	}
-	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
-	defer cancel()
 	response, err := a.client.Authorize(ctx, &pb.Identity{
 		Route:             route,
 		User:              s.User,
@@ -65,11 +66,12 @@ func (a *AuthorizeGRPC) Authorize(ctx context.Context, route string, s *sessions
 
 // IsAdmin takes a session and returns whether the user is an administrator
 func (a *AuthorizeGRPC) IsAdmin(ctx context.Context, s *sessions.SessionState) (bool, error) {
+	ctx, span := trace.StartSpan(ctx, "proxy.client.grpc.IsAdmin")
+	defer span.End()
+
 	if s == nil {
 		return false, errors.New("session cannot be nil")
 	}
-	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
-	defer cancel()
 	response, err := a.client.IsAdmin(ctx, &pb.Identity{Email: s.Email, Groups: s.Groups})
 	return response.GetIsAdmin(), err
 }

proxy/clients/clients.go

@@ -11,8 +11,9 @@ import (
 	"strings"
 
 	"github.com/pomerium/pomerium/internal/log"
-	"github.com/pomerium/pomerium/internal/metrics"
 	"github.com/pomerium/pomerium/internal/middleware"
+	"github.com/pomerium/pomerium/internal/telemetry/metrics"
+
 	"go.opencensus.io/plugin/ocgrpc"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"

proxy/proxy.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"html/template"
 	stdlog "log"
+	"net"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
@@ -16,9 +17,10 @@ import (
 	"github.com/pomerium/pomerium/internal/config"
 	"github.com/pomerium/pomerium/internal/cryptutil"
 	"github.com/pomerium/pomerium/internal/log"
-	"github.com/pomerium/pomerium/internal/metrics"
 	"github.com/pomerium/pomerium/internal/middleware"
 	"github.com/pomerium/pomerium/internal/sessions"
+	"github.com/pomerium/pomerium/internal/telemetry/metrics"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 	"github.com/pomerium/pomerium/internal/templates"
 	"github.com/pomerium/pomerium/internal/tripper"
 	"github.com/pomerium/pomerium/proxy/clients"
@@ -196,11 +198,19 @@ func (p *Proxy) UpdatePolicies(opts *config.Options) error {
 		}
 		proxy := NewReverseProxy(policy.Destination)
 		// build http transport (roundtripper) middleware chain
-		// todo(bdd): this will make vet complain, it is safe
-		// and can be replaced with transport.Clone() in go 1.13
-		// https://go-review.googlesource.com/c/go/+/174597/
-		// https://github.com/golang/go/issues/26013#issuecomment-399481302
-		transport := *(http.DefaultTransport.(*http.Transport))
+		// todo(bdd): replace with transport.Clone() in go 1.13
+		transport := http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				Timeout:   30 * time.Second,
+				KeepAlive: 30 * time.Second,
+				DualStack: true,
+			}).DialContext,
+			MaxIdleConns:          100,
+			IdleConnTimeout:       90 * time.Second,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+		}
 		c := tripper.NewChain()
 		c = c.Append(metrics.HTTPMetricsRoundTripper("proxy", policy.Destination.Host))
 		if policy.TLSSkipVerify {
@@ -236,7 +246,9 @@ type UpstreamProxy struct {
 
 // ServeHTTP handles the second (reverse-proxying) leg of pomerium's request flow
 func (u *UpstreamProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	u.handler.ServeHTTP(w, r)
+	ctx, span := trace.StartSpan(r.Context(), fmt.Sprintf("%s%s", r.Host, r.URL.Path))
+	defer span.End()
+	u.handler.ServeHTTP(w, r.WithContext(ctx))
 }
 
 // NewReverseProxy returns a new ReverseProxy that routes URLs to the scheme, host, and